Hello everyone
I am using nested resampling in caret (5-fold outer and bootstrap inner
resampling) and by default, it shows the "Accuracy" metric. How can I
use
it for the ROC/AUC metric?
My code is:
d=readARFF("apns.arff")
index <- createDataPartition(d$isKilled , p = .70,list = FALSE)
tr <- d[index, ]
ts <- d[-index, ]
boot <- trainControl(method = "boot", number=100,
search="random",
classProbs = TRUE, summaryFunction = twoClassSummary)
outer_folds <- createFolds(d$isKilled, k = 5)
boot <- trainControl(method = "boot", number=10)
CV1 <- lapply(outer_folds, function(index){
tr <- d[-index, ]
ts <- d[index,]
cart1 <-train(isKilled ~ ., data = tr,
method = "rpart",
tuneLength = 20,
metric = "Accuracy",
preProc = c("center", "scale",
"nzv"),
trControl = boot)
postResample(predict(cart1, ts), ts$isKilled)
})
sapply(CV1, function(x) x[3]) -> CV_MAE1
CV_MAE1
[[alternative HTML version deleted]]