JMS datastore replication: performance with enterprise server

source: sync-msgs.db 5GB, GFS2

target: EXT4

serial: WALL TIME: 121sec 133msec

sendfile: WALL TIME: 113sec 264msec

default: WALL TIME: 64sec 7msec

mmap: WALL TIME: 142sec 57msec

aio: WALL TIME: 8sec 791msec

Replace XML element content with regexp

replaceAll
<([a-zA-Z]+[0-9]*:)?([^\s<]*[eE]lement[^\s]*)([^>]*>)([^<]*)(<\/\1\2)
<$1$2$3fixedContent$5

Make more friendly Tibco BW cmdline Validator

Override class name in validateproject.tra

package com.tibco.ae.tools.designer.cmdline;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;

import com.tibco.ae.designerapi.AEResource;
import com.tibco.ae.designerapi.DesignerError;
import com.tibco.ae.designerapi.DesignerFolder;
import com.tibco.ae.tools.designer.AEApplication;
import com.tibco.ae.tools.designer.AEDocument;
import com.tibco.ae.tools.designer.actions.ValidateProjectAction;
import com.tibco.ae.tools.vfileresourcestore.FileSystemVFileFactoryResourceStore;
import com.tibco.objectrepo.vfile.VFile;
import com.tibco.util.ListHashMap;
import com.tibco.util.ResourceManager;

public class ValidateBwProject extends DesignerCommandLineApp {

private TempAliases aliases = null;
private int errorCount = 0;
private int warningCount =0;

public void execute(String projectLocation, String aliasFileLocation) throws Exception {
initResources();
initializeApp();
TempAliases temp = new TempAliases() {
public void updateAliases(File file) throws FileNotFoundException, IOException {
int start = "tibco.alias.".length();
Properties p = new Properties();
p.load(new FileInputStream(file));
Properties modified = new Properties();
for (Iterator<Object> i = p.keySet().iterator(); i.hasNext();) {
String key = (String) i.next();
String value = p.getProperty(key);
if (key.length() != 0 && value.length() != 0) {
if (key.startsWith("tibco.alias."))
key = key.substring(start);
modified.setProperty(key, value);
value = value.replace("\\", "/");
File f = new File(value);
boolean fileOK = false;
try {
f = f.getCanonicalFile();
fileOK = f.exists();
}
catch (Exception exc) {}
if (!fileOK)
System.err.println("The file '"+f.getPath()+"' specified for alias '"+key+"' "+
" was not found.");
}
}
AEApplication.getAEApplication().setFileAliases(modified);
}
};
System.out.println("**** Processing aliases ****");
temp.updateAliases(new File(aliasFileLocation));
aliases = temp;
Properties props = createDocumentPropertiesForFile(projectLocation);
props.put("ae.forimport", "true");
props.put("ae.silent", "true");
System.out.println("**** Creating ActiveEnterprise document (serviceagents below may contain Java Instance instead of WebService with WSDL document) ****");
FileSystemVFileFactoryResourceStore storeFileFactory = new FileSystemVFileFactoryResourceStore(props) {
@Override
public void loadResourceFrom(DesignerFolder parent, VFile file)
throws Exception {
try {
String uri = file.getFullURI();
AEResource res = (AEResource) provider.getObject(uri, AEResource.class, true);
res.initializeNameFrom(file.getName());
parent.addResource(res);
}
catch (Exception e) { /* ignore unknown files */}
}
};
doc = new AEDocument(storeFileFactory, false);
doc.getResourceStore().insureFoldersAreInitialized();
ValidateProjectAction action = new ValidateProjectAction(doc);
System.out.println("**** Validating (you may see output from Java static initializers here) ****");
DesignerError errors[] = action.runValidation(false);

if (errors != null && errors.length > 0) {
printStatistics(errors);

ListHashMap errorTypeToErrorList = new ListHashMap();
int i = 0;
for (int max = errors.length; i < max; i++) {
String type = errors[i].getType();
if (type == null)
type = "ae.error.default.type";
errorTypeToErrorList.put(type, errors[i]);
}

for (@SuppressWarnings("rawtypes") Iterator iter = errorTypeToErrorList.keySet().iterator(); iter.hasNext(); ) {
String type = (String) iter.next();
@SuppressWarnings("unchecked")
List<DesignerError> errList = errorTypeToErrorList.getList(type);
StringBuffer sb = new StringBuffer();

if (!type.equals("ae.error.unused.gvardefn.type")) {
boolean ignoreXpathInGV = type.equals("ae.error.missing.gvardefn.type");
String categName = ResourceManager.manager.getString(type+".displayName");
sb.append("\n**** ["+type+"] "+categName+" ****\n\n");

int hits = 0;
for (int k=0; k < errList.size(); k++) {
DesignerError err = (DesignerError) errList.get(k);
String summ = err.getSummary();
String msg = err.getMessage();

if (ignoreXpathInGV && (msg.contains("[") || msg.contains("*")))
continue;
if (summ.contains("The resource") && summ.contains("is unknown"))
continue; /* Ignore not BW artifact files inside repository */
String uri = err.getAssociatedResource()!=null ? err.getAssociatedResource().getURI() : null;
if (uri!=null && uri.toLowerCase().startsWith("/test"))
continue; /* Ignore not valid unit tests */

sb.append("  "+summ+": "+msg+"\n");
if (uri != null)
sb.append("      <"+uri+">\r\n");
sb.append("\r\n");

if (type.contains("warning"))
warningCount++;
else
errorCount++;
hits++;
}
if (hits>0) {
System.err.println(sb);
}
}
}
if (errorCount==0)
System.out.println(ResourceManager.manager.getString("ae.validate.project.noerrors"));
}
cleanup();
}

@Override
public void cleanup() {
if (aliases != null)
aliases.resetAliases();
if (doc != null) {
try {
doc.close();
}
catch (Exception exp) {}
}
}

@Override
public void checkForErrors() {
if (DesignerError.getErrorCount() != 0) {
DesignerError errs[] = DesignerError.getErrorLog();
int i = 0;
for (int max = errs.length; i < max; i++) {
DesignerError err = errs[i];
System.err.println("[REPO ERROR] "+err.getSummary());
System.err.println((new StringBuilder()).append("\t")
.append(err.getMessage()).toString());
}
cleanup();
System.exit(1);
}
}

private void printStatistics(DesignerError errors[]) {
String s = ResourceManager.manager.getString(
"ae.validate.project.dialog.summary.status.text",
String.valueOf(errorCount), String.valueOf(warningCount));
System.out.println("**** Validation stats *****\n"+s);
}

public final static void main(String[] args) throws Exception {
String projectPath = null;
String aliasPath = null;
String[] cmdline = new String[args.length+3];
for (int k = 0; k < args.length; k++)
cmdline[k] = args[k];
for (int i=0; i < args.length; i++) {
if (args[i].equals("-a")) {
aliasPath = args[i+1];
if (projectPath==null)
projectPath = args[i+2];
}
else
projectPath = args[i];
}
if (projectPath==null || aliasPath==null ||
projectPath.trim().length()==0 || aliasPath.trim().length()==0) {
System.out.println("Usage: projectPath -a aliases.txt");
System.exit(0);
}
new ValidateBwProject().execute(projectPath, aliasPath);
System.exit(0);
}
}

How to add support for StartTls in Spring LDAP

import com.sun.jndi.ldap.LdapClient;

public class Binding extends LdapContextSource {

private SearchControls sc = new SearchControls();
private String base = null;
private static boolean requireServerCertInTrustedStore = false;
private static boolean traceAttributesModification = !true;
private boolean useStartTlsAuth = false;

private final static String[] CLEAR_PARAMS_FOR_STARTTLS = { ".security",
"java.naming.ldap.factory.socket", "com.sun.jndi.ldap.connect.pool"
};

private Hashtable<String,Object> prepareParamsForStartTls(@SuppressWarnings("rawtypes") Hashtable env) {
Hashtable<String,Object> props = new Hashtable<String, Object>();
for (Object k : env.keySet()) {
String key = k.toString();
int hitCount = 0;
for (int i = 0; i < CLEAR_PARAMS_FOR_STARTTLS.length; i++) {
if (key.contains(CLEAR_PARAMS_FOR_STARTTLS[i])) {
hitCount++;
break;
}
}
if (hitCount == 0)
props.put(key, env.get(key));
}
props.put("java.naming.ldap.version", "3");
props.put(Context.SECURITY_PROTOCOL, "plain");
return props;
}

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
protected DirContext getDirContextInstance(Hashtable env)
throws NamingException {

Hashtable<String,Object> params = prepareParamsForStartTls(env); /* strip pooling and security */
params.put("java.naming.ldap.attributes.binary", "objectSid objectGUID");
final LdapContext ctx = (LdapContext) NamingManager.getInitialContext(params);

StartTlsResponse tlsResp = null;
if (useStartTlsAuth) {
tlsResp = (StartTlsResponse) ctx.extendedOperation(new StartTlsRequest());
tlsResp.setHostnameVerifier(TolerantSSLSocketFactory.TOLERANT_HOSTNAME_VERIFIER);
try {
tlsResp.negotiate(TolerantSSLSocketFactory.INSTANCE);
}
catch (Throwable t) {
throw new NamingException("SSL negotiation for LDAP StartTLS failed: "+t);
}
}
ctx.getEnvironment().putAll(env); /* do not use addToEnvironment because it would reconnect without StartTls */
try {
Field fClient = ctx.getClass().getDeclaredField("clnt");
fClient.setAccessible(true);
LdapClient cli = (LdapClient) fClient.get(ctx);
/* We are over secure channel, need to authenticate now */
Method mAuthenticate = cli.getClass().getDeclaredMethod("authenticate", new Class[] {
boolean.class, String.class, Object.class, int.class, String.class, Control[].class, Hashtable.class
});
mAuthenticate.setAccessible(true);
mAuthenticate.invoke(cli, false, env.get(Context.SECURITY_PRINCIPAL).toString(),
env.get(Context.SECURITY_CREDENTIALS), 3, "simple", null, env);
}
catch (Exception e) {
Throwable t = e;
while (t.getCause() != null)
t = t.getCause();
throw new NamingException("Cannot authenticate due to error: "+t.getMessage());
}
return new InitialLdapContext(env, null) {
@Override
protected Context getDefaultInitCtx() throws NamingException {
/* cache context */
return ctx;
}
};
}

public static boolean isRequiredServerCertInTrustedStore() {
return requireServerCertInTrustedStore;
}

public static void setRequireServerCertInTrustedStore(
boolean requireServerCertInTrustedStore) {
Binding.requireServerCertInTrustedStore = requireServerCertInTrustedStore;
}

public static void setTraceAttributesModification(
boolean traceAttributesModification) {
Binding.traceAttributesModification = traceAttributesModification;
}

public static boolean getTraceAttributesModification() {
return traceAttributesModification;
}

public Binding(String userLogin, String userPassword, String serverUrl, String searchBase) throws Exception {

useStartTlsAuth = serverUrl.startsWith("ldap(s)");
if (useStartTlsAuth) {
serverUrl = "ldap"+serverUrl.substring(7);
setPooled(false);
}
setUserDn(userLogin);
setPassword(userPassword);
setBase(searchBase);
setReferral("follow");
setUrl(serverUrl);
setPooled(true);

HashMap<String,String> props = new HashMap<String, String>();
props.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
props.put(Context.SECURITY_AUTHENTICATION, "none");
props.put(Context.SECURITY_AUTHENTICATION, "simple");
props.put(Context.SECURITY_CREDENTIALS, userLogin);
props.put(Context.SECURITY_PRINCIPAL, userPassword);

if (!requireServerCertInTrustedStore) {
props.put("java.naming.ldap.factory.socket", TolerantSSLSocketFactory.class.getName());
}
setCacheEnvironmentProperties(true);
setBaseEnvironmentProperties(props);
setAuthenticationStrategy(new SimpleDirContextAuthenticationStrategy());
afterPropertiesSet();
sc.setSearchScope(SearchControls.SUBTREE_SCOPE);
sc.setCountLimit(0);
sc.setReturningAttributes(null);
sc.setReturningObjFlag(true);
}

public SearchControls getSearchControls() {
return sc;
}

public String getSearchBase() {
return this.base != null ? base : "";
}
}

Unescape Unicode (for Tibco dealing with international data)

ByteArrayOutputStream repacked = new ByteArrayOutputStream();
byte[] buff = (in+"\0\0\0\0\0\0").getBytes(); //padding to make McCabe happy
for (int i=0; i < buff.length-6; ) {
    if (buff[i] == '\\' && buff[i+1] == 'u') {
        String t = new String(new int[] { Integer.decode("0x"+new String(buff, i+2, 4)) }, 0, 1);
        for (int k=0; k < t.length(); k++) {
            int cp = t.codePointAt(k);
            if (!Character.isIdentifierIgnorable(cp))
                repacked.write(t.getBytes("utf-8"));
        }
        i+=6;
    }
    else {
        repacked.write(buff, i, 1);
        i++;
    }
}
in = repacked.toString(encoding);