Could someone give a better example of how to extend [1] to create an ensemble from multiple machine learning classifiers to improve the classification performance? [1] http://heuristically.wordpress.com/2009/12/23/compare-performance-machine-learning-classifiers-r/ Berry and Linoff ("Mastering Data Mining", p217) suggest the approach below, but the performance is similar to the individual models. # create ensemble (per Berry and Linoff) evidence_a <- attr(x.svm.prob, "probabilities")[,2] * x.ip.prob[,2] * x.cf.prob evidence_b <- ( 1-x.ip.prob[,2]) * (1-attr(x.svm.prob, "probabilities")[,2] * (1-x.cf.prob)) x.en.prob <- (evidence_a) / (evidence_a + evidence_b) # bagging x.ip.prob.rocr <- prediction(x.ip.prob[,2], BreastCancer[ind == 2,'Class']) x.ip.perf <- performance(x.ip.prob.rocr, "tpr","fpr") plot(x.ip.perf, col=2) # svm x.svm.prob.rocr <- prediction(attr(x.svm.prob, "probabilities")[,2], BreastCancer[ind == 2,'Class']) x.svm.perf <- performance(x.svm.prob.rocr, "tpr","fpr") plot(x.svm.perf, col=3, add=TRUE) # cforest x.cf.prob.rocr <- prediction(x.cf.prob, BreastCancer[ind == 2,'Class']) x.cf.perf <- performance(x.cf.prob.rocr, "tpr","fpr") plot(x.cf.perf, col=4, add=TRUE) # ensemble #x.en.rocr <- prediction(rowMeans(cbind(x1=x.cf.prob, x2=x.ip.prob[,2])), BreastCancer[ind == 2,'Class']) x.en.rocr <- prediction(x.en.prob, BreastCancer[ind == 2,'Class']) x.en.perf <- performance(x.en.rocr, "tpr","fpr") plot(x.en.perf, col=5, add=TRUE) # Draw a legend. legend(0.6, 0.6, c('bagging','svm', 'cforest', 'ensemble'), 2:5) Jack