MLR Tidyverse Chapter 03 Notes - kNN

ML Home | MLR Book Home

library(tidyverse)
library(mlr)

kNN - k Nearest Neighbors

Can use mean or median for regression problems (see ch12)

For ties choose random or decrease k (p59)

Diabetes Example

p60

Tibble Summary

data(diabetes, package = "mclust")

diabetesTib <- as_tibble(diabetes)
summary(diabetesTib)

##       class       glucose       insulin            sspg      
##  Chemical:36   Min.   : 70   Min.   :  45.0   Min.   : 10.0  
##  Normal  :76   1st Qu.: 90   1st Qu.: 352.0   1st Qu.:118.0  
##  Overt   :33   Median : 97   Median : 403.0   Median :156.0  
##                Mean   :122   Mean   : 540.8   Mean   :186.1  
##                3rd Qu.:112   3rd Qu.: 558.0   3rd Qu.:221.0  
##                Max.   :353   Max.   :1568.0   Max.   :748.0

diabetesTib

diabetesTib

## # A tibble: 145 x 4
##    class  glucose insulin  sspg
##    <fct>    <dbl>   <dbl> <dbl>
##  1 Normal      80     356   124
##  2 Normal      97     289   117
##  3 Normal     105     319   143
##  4 Normal      90     356   199
##  5 Normal      90     323   240
##  6 Normal      86     381   157
##  7 Normal     100     350   221
##  8 Normal      85     301   186
##  9 Normal      97     379   142
## 10 Normal      97     296   131
## # … with 135 more rows

Plot glucose vs insulin

ggplot(diabetesTib, aes(glucose, insulin, col = class)) +
  geom_point()  +
  theme_bw()

Plot sspg vs insulin

ggplot(diabetesTib, aes(sspg, insulin, col = class)) +
  geom_point() +
  theme_bw()

Plot sspg vs glucose

ggplot(diabetesTib, aes(sspg, glucose, col = class)) +
  geom_point() +
  theme_bw()

Exercise 1

It’s also very important for kNN (and many machine learning algorithms) to scale the predictor variables by dividing them by their standard deviation.

mlr3 library has 3 stages

p62

• Define the task
• Define the learner
• Train the model

Diabetes Example

• Must scale data for kNN
• Doesn’t handle categorical variables natively

makeClassifTask

diabetesTask <- makeClassifTask(data = diabetesTib, target = "class")

## Warning in makeTask(type = type, data = data, weights = weights, blocking =
## blocking, : Provided data is not a pure data.frame but from class tbl_df, hence
## it will be converted.

makeLearner

knn <- makeLearner("classif.knn", par.vals = list("k" = 2))

listLearners

listLearners()$class

## Warning in listLearners.character(): The following learners could not be constructed, probably because their packages are not installed:
## surv.CoxBoost,surv.cv.CoxBoost
## Check ?learners to see which packages you need or install mlr with all suggestions.

##   [1] "classif.ada"                    "classif.adaboostm1"            
##   [3] "classif.bartMachine"            "classif.binomial"              
##   [5] "classif.boosting"               "classif.bst"                   
##   [7] "classif.C50"                    "classif.cforest"               
##   [9] "classif.clusterSVM"             "classif.ctree"                 
##  [11] "classif.cvglmnet"               "classif.dbnDNN"                
##  [13] "classif.dcSVM"                  "classif.earth"                 
##  [15] "classif.evtree"                 "classif.extraTrees"            
##  [17] "classif.fdausc.glm"             "classif.fdausc.kernel"         
##  [19] "classif.fdausc.knn"             "classif.fdausc.np"             
##  [21] "classif.FDboost"                "classif.featureless"           
##  [23] "classif.fgam"                   "classif.fnn"                   
##  [25] "classif.gamboost"               "classif.gaterSVM"              
##  [27] "classif.gausspr"                "classif.gbm"                   
##  [29] "classif.geoDA"                  "classif.glmboost"              
##  [31] "classif.glmnet"                 "classif.h2o.deeplearning"      
##  [33] "classif.h2o.gbm"                "classif.h2o.glm"               
##  [35] "classif.h2o.randomForest"       "classif.IBk"                   
##  [37] "classif.J48"                    "classif.JRip"                  
##  [39] "classif.kknn"                   "classif.knn"                   
##  [41] "classif.ksvm"                   "classif.lda"                   
##  [43] "classif.LiblineaRL1L2SVC"       "classif.LiblineaRL1LogReg"     
##  [45] "classif.LiblineaRL2L1SVC"       "classif.LiblineaRL2LogReg"     
##  [47] "classif.LiblineaRL2SVC"         "classif.LiblineaRMultiClassSVC"
##  [49] "classif.linDA"                  "classif.logreg"                
##  [51] "classif.lssvm"                  "classif.lvq1"                  
##  [53] "classif.mda"                    "classif.mlp"                   
##  [55] "classif.multinom"               "classif.naiveBayes"            
##  [57] "classif.neuralnet"              "classif.nnet"                  
##  [59] "classif.nnTrain"                "classif.nodeHarvest"           
##  [61] "classif.OneR"                   "classif.pamr"                  
##  [63] "classif.PART"                   "classif.penalized"             
##  [65] "classif.plr"                    "classif.plsdaCaret"            
##  [67] "classif.probit"                 "classif.qda"                   
##  [69] "classif.quaDA"                  "classif.randomForest"          
##  [71] "classif.randomForestSRC"        "classif.ranger"                
##  [73] "classif.rda"                    "classif.rFerns"                
##  [75] "classif.rknn"                   "classif.rotationForest"        
##  [77] "classif.rpart"                  "classif.RRF"                   
##  [79] "classif.rrlda"                  "classif.saeDNN"                
##  [81] "classif.sda"                    "classif.sparseLDA"             
##  [83] "classif.svm"                    "classif.xgboost"               
##  [85] "cluster.cmeans"                 "cluster.Cobweb"                
##  [87] "cluster.dbscan"                 "cluster.EM"                    
##  [89] "cluster.FarthestFirst"          "cluster.kkmeans"               
##  [91] "cluster.kmeans"                 "cluster.MiniBatchKmeans"       
##  [93] "cluster.SimpleKMeans"           "cluster.XMeans"                
##  [95] "multilabel.cforest"             "multilabel.randomForestSRC"    
##  [97] "multilabel.rFerns"              "regr.bartMachine"              
##  [99] "regr.bcart"                     "regr.bgp"                      
## [101] "regr.bgpllm"                    "regr.blm"                      
## [103] "regr.brnn"                      "regr.bst"                      
## [105] "regr.btgp"                      "regr.btgpllm"                  
## [107] "regr.btlm"                      "regr.cforest"                  
## [109] "regr.crs"                       "regr.ctree"                    
## [111] "regr.cubist"                    "regr.cvglmnet"                 
## [113] "regr.earth"                     "regr.evtree"                   
## [115] "regr.extraTrees"                "regr.FDboost"                  
## [117] "regr.featureless"               "regr.fgam"                     
## [119] "regr.fnn"                       "regr.frbs"                     
## [121] "regr.gamboost"                  "regr.gausspr"                  
## [123] "regr.gbm"                       "regr.glm"                      
## [125] "regr.glmboost"                  "regr.glmnet"                   
## [127] "regr.GPfit"                     "regr.h2o.deeplearning"         
## [129] "regr.h2o.gbm"                   "regr.h2o.glm"                  
## [131] "regr.h2o.randomForest"          "regr.IBk"                      
## [133] "regr.kknn"                      "regr.km"                       
## [135] "regr.ksvm"                      "regr.laGP"                     
## [137] "regr.LiblineaRL2L1SVR"          "regr.LiblineaRL2L2SVR"         
## [139] "regr.lm"                        "regr.mars"                     
## [141] "regr.mob"                       "regr.nnet"                     
## [143] "regr.nodeHarvest"               "regr.pcr"                      
## [145] "regr.penalized"                 "regr.plsr"                     
## [147] "regr.randomForest"              "regr.randomForestSRC"          
## [149] "regr.ranger"                    "regr.rknn"                     
## [151] "regr.rpart"                     "regr.RRF"                      
## [153] "regr.rsm"                       "regr.rvm"                      
## [155] "regr.svm"                       "regr.xgboost"                  
## [157] "surv.cforest"                   "surv.CoxBoost"                 
## [159] "surv.coxph"                     "surv.cv.CoxBoost"              
## [161] "surv.cvglmnet"                  "surv.gamboost"                 
## [163] "surv.gbm"                       "surv.glmboost"                 
## [165] "surv.glmnet"                    "surv.randomForestSRC"          
## [167] "surv.ranger"                    "surv.rpart"

Train

knnModel <- train(knn, diabetesTask)

Predict

knnPred <- predict(knnModel, newdata = diabetesTib)

## Warning in predict.WrappedModel(knnModel, newdata = diabetesTib): Provided data
## for prediction is not a pure data.frame but from class tbl_df, hence it will be
## converted.

Performance

• mmce - mean misclassification error
• acc - accuracy

performance(knnPred, measures = list(mmce, acc))

##       mmce        acc 
## 0.03448276 0.96551724

3.3 Bias-Variance Tradeoff

• Bias - Underfit. Use too few variables or a simple model.
• Variance - Overfit the training data. Won’t handle new data well.

3.4 Cross-Validation

p69

• Holdout
• K-fold
• LOOCV - Leave One Out Cross-Validation

3.5 Cross-validating our kNN Model

Cross-validation: method = Holdout

stratify - True means hold proportion

#diabetesTask <- makeClassifTask(data = diabetesTib, target = "class")

#knn <- makeLearner("classif.knn", par.vals = list("k" = 2))
holdout <- makeResampleDesc(method = "Holdout", split = 2/3, stratify = TRUE)

holdoutCV <- resample(learner = knn, task = diabetesTask,
                      resampling = holdout, measures = list(mmce, acc))

## Resampling: holdout

## Measures:             mmce      acc

## [Resample] iter 1:    0.1020408 0.8979592

## 

## Aggregated Result: mmce.test.mean=0.1020408,acc.test.mean=0.8979592

## 

Accuracy goes down.

Directly access value

holdoutCV$aggr

## mmce.test.mean  acc.test.mean 
##      0.1020408      0.8979592

Exercise 2

• Use 10% of data for testing
• Don’t stratify

holdout2 <- makeResampleDesc(method = "Holdout", split = 9/10, stratify = FALSE)
holdoutCV2 <- resample(learner = knn, task = diabetesTask,
                      resampling = holdout, measures = list(mmce, acc))

## Resampling: holdout

## Measures:             mmce      acc

## [Resample] iter 1:    0.0612245 0.9387755

## 

## Aggregated Result: mmce.test.mean=0.0612245,acc.test.mean=0.9387755

## 

Confusion Matrix

calculateConfusionMatrix(holdoutCV$pred, relative = TRUE)

## Relative confusion matrix (normalized by row/column):
##           predicted
## true       Chemical  Normal    Overt     -err.-   
##   Chemical 0.67/0.89 0.25/0.11 0.08/0.08 0.33     
##   Normal   0.04/0.11 0.96/0.89 0.00/0.00 0.04     
##   Overt    0.00/0.00 0.00/0.00 1.00/0.92 0.00     
##   -err.-        0.11      0.11      0.08 0.10     
## 
## 
## Absolute confusion matrix:
##           predicted
## true       Chemical Normal Overt -err.-
##   Chemical        8      3     1      4
##   Normal          1     25     0      1
##   Overt           0      0    11      0
##   -err.-          1      3     1      5

Cross-validation: method = RepCV/repeated cross-validation/(k-fold)

• 10 folds
• 50 reps
• stratify = TRUE
• measures = list(mmce, acc)

kFold <- makeResampleDesc(method = "RepCV", 
                          folds = 10, 
                          reps = 50,
                          stratify = TRUE)

kFoldCV <- resample(learner = knn, task = diabetesTask,
                    resampling = kFold, 
                    measures = list(mmce, acc)) # Performance Measures

## Resampling: repeated cross-validation

## Measures:             mmce      acc

## [Resample] iter 1:    0.2142857 0.7857143

## [Resample] iter 2:    0.0625000 0.9375000

## [Resample] iter 3:    0.2000000 0.8000000

## [Resample] iter 4:    0.0000000 1.0000000

## [Resample] iter 5:    0.0666667 0.9333333

## [Resample] iter 6:    0.1428571 0.8571429

## [Resample] iter 7:    0.1333333 0.8666667

## [Resample] iter 8:    0.0000000 1.0000000

## [Resample] iter 9:    0.1428571 0.8571429

## [Resample] iter 10:   0.0000000 1.0000000

## [Resample] iter 11:   0.2307692 0.7692308

## [Resample] iter 12:   0.1333333 0.8666667

## [Resample] iter 13:   0.1428571 0.8571429

## [Resample] iter 14:   0.0625000 0.9375000

## [Resample] iter 15:   0.2000000 0.8000000

## [Resample] iter 16:   0.1538462 0.8461538

## [Resample] iter 17:   0.0000000 1.0000000

## [Resample] iter 18:   0.0000000 1.0000000

## [Resample] iter 19:   0.2142857 0.7857143

## [Resample] iter 20:   0.0714286 0.9285714

## [Resample] iter 21:   0.0714286 0.9285714

## [Resample] iter 22:   0.1428571 0.8571429

## [Resample] iter 23:   0.2142857 0.7857143

## [Resample] iter 24:   0.0625000 0.9375000

## [Resample] iter 25:   0.1428571 0.8571429

## [Resample] iter 26:   0.0666667 0.9333333

## [Resample] iter 27:   0.0714286 0.9285714

## [Resample] iter 28:   0.2000000 0.8000000

## [Resample] iter 29:   0.0714286 0.9285714

## [Resample] iter 30:   0.0666667 0.9333333

## [Resample] iter 31:   0.3571429 0.6428571

## [Resample] iter 32:   0.0666667 0.9333333

## [Resample] iter 33:   0.0625000 0.9375000

## [Resample] iter 34:   0.0714286 0.9285714

## [Resample] iter 35:   0.0000000 1.0000000

## [Resample] iter 36:   0.2307692 0.7692308

## [Resample] iter 37:   0.0666667 0.9333333

## [Resample] iter 38:   0.1428571 0.8571429

## [Resample] iter 39:   0.0714286 0.9285714

## [Resample] iter 40:   0.0666667 0.9333333

## [Resample] iter 41:   0.1538462 0.8461538

## [Resample] iter 42:   0.0000000 1.0000000

## [Resample] iter 43:   0.2000000 0.8000000

## [Resample] iter 44:   0.0625000 0.9375000

## [Resample] iter 45:   0.1428571 0.8571429

## [Resample] iter 46:   0.2666667 0.7333333

## [Resample] iter 47:   0.2307692 0.7692308

## [Resample] iter 48:   0.0000000 1.0000000

## [Resample] iter 49:   0.0666667 0.9333333

## [Resample] iter 50:   0.1333333 0.8666667

## [Resample] iter 51:   0.0000000 1.0000000

## [Resample] iter 52:   0.1250000 0.8750000

## [Resample] iter 53:   0.0625000 0.9375000

## [Resample] iter 54:   0.2000000 0.8000000

## [Resample] iter 55:   0.1538462 0.8461538

## [Resample] iter 56:   0.1428571 0.8571429

## [Resample] iter 57:   0.0000000 1.0000000

## [Resample] iter 58:   0.1538462 0.8461538

## [Resample] iter 59:   0.2000000 0.8000000

## [Resample] iter 60:   0.1250000 0.8750000

## [Resample] iter 61:   0.2142857 0.7857143

## [Resample] iter 62:   0.2000000 0.8000000

## [Resample] iter 63:   0.2307692 0.7692308

## [Resample] iter 64:   0.2000000 0.8000000

## [Resample] iter 65:   0.0666667 0.9333333

## [Resample] iter 66:   0.0000000 1.0000000

## [Resample] iter 67:   0.2307692 0.7692308

## [Resample] iter 68:   0.0000000 1.0000000

## [Resample] iter 69:   0.0666667 0.9333333

## [Resample] iter 70:   0.0000000 1.0000000

## [Resample] iter 71:   0.2857143 0.7142857

## [Resample] iter 72:   0.1428571 0.8571429

## [Resample] iter 73:   0.0714286 0.9285714

## [Resample] iter 74:   0.1333333 0.8666667

## [Resample] iter 75:   0.2500000 0.7500000

## [Resample] iter 76:   0.0714286 0.9285714

## [Resample] iter 77:   0.0625000 0.9375000

## [Resample] iter 78:   0.0000000 1.0000000

## [Resample] iter 79:   0.0000000 1.0000000

## [Resample] iter 80:   0.2307692 0.7692308

## [Resample] iter 81:   0.2000000 0.8000000

## [Resample] iter 82:   0.2666667 0.7333333

## [Resample] iter 83:   0.0666667 0.9333333

## [Resample] iter 84:   0.0769231 0.9230769

## [Resample] iter 85:   0.1428571 0.8571429

## [Resample] iter 86:   0.2142857 0.7857143

## [Resample] iter 87:   0.0000000 1.0000000

## [Resample] iter 88:   0.0000000 1.0000000

## [Resample] iter 89:   0.1428571 0.8571429

## [Resample] iter 90:   0.0625000 0.9375000

## [Resample] iter 91:   0.0666667 0.9333333

## [Resample] iter 92:   0.0000000 1.0000000

## [Resample] iter 93:   0.1428571 0.8571429

## [Resample] iter 94:   0.0000000 1.0000000

## [Resample] iter 95:   0.1333333 0.8666667

## [Resample] iter 96:   0.1428571 0.8571429

## [Resample] iter 97:   0.0769231 0.9230769

## [Resample] iter 98:   0.0666667 0.9333333

## [Resample] iter 99:   0.2000000 0.8000000

## [Resample] iter 100:  0.0000000 1.0000000

## [Resample] iter 101:  0.2000000 0.8000000

## [Resample] iter 102:  0.0666667 0.9333333

## [Resample] iter 103:  0.1333333 0.8666667

## [Resample] iter 104:  0.1538462 0.8461538

## [Resample] iter 105:  0.0000000 1.0000000

## [Resample] iter 106:  0.2142857 0.7857143

## [Resample] iter 107:  0.1538462 0.8461538

## [Resample] iter 108:  0.1333333 0.8666667

## [Resample] iter 109:  0.0000000 1.0000000

## [Resample] iter 110:  0.0666667 0.9333333

## [Resample] iter 111:  0.0714286 0.9285714

## [Resample] iter 112:  0.0666667 0.9333333

## [Resample] iter 113:  0.0714286 0.9285714

## [Resample] iter 114:  0.0714286 0.9285714

## [Resample] iter 115:  0.1428571 0.8571429

## [Resample] iter 116:  0.0714286 0.9285714

## [Resample] iter 117:  0.0714286 0.9285714

## [Resample] iter 118:  0.1250000 0.8750000

## [Resample] iter 119:  0.0666667 0.9333333

## [Resample] iter 120:  0.1333333 0.8666667

## [Resample] iter 121:  0.1428571 0.8571429

## [Resample] iter 122:  0.1250000 0.8750000

## [Resample] iter 123:  0.1428571 0.8571429

## [Resample] iter 124:  0.0625000 0.9375000

## [Resample] iter 125:  0.0769231 0.9230769

## [Resample] iter 126:  0.1428571 0.8571429

## [Resample] iter 127:  0.1333333 0.8666667

## [Resample] iter 128:  0.0000000 1.0000000

## [Resample] iter 129:  0.0625000 0.9375000

## [Resample] iter 130:  0.1428571 0.8571429

## [Resample] iter 131:  0.1538462 0.8461538

## [Resample] iter 132:  0.0000000 1.0000000

## [Resample] iter 133:  0.2666667 0.7333333

## [Resample] iter 134:  0.2000000 0.8000000

## [Resample] iter 135:  0.0666667 0.9333333

## [Resample] iter 136:  0.1333333 0.8666667

## [Resample] iter 137:  0.0000000 1.0000000

## [Resample] iter 138:  0.0714286 0.9285714

## [Resample] iter 139:  0.0666667 0.9333333

## [Resample] iter 140:  0.1428571 0.8571429

## [Resample] iter 141:  0.2666667 0.7333333

## [Resample] iter 142:  0.1538462 0.8461538

## [Resample] iter 143:  0.0000000 1.0000000

## [Resample] iter 144:  0.1333333 0.8666667

## [Resample] iter 145:  0.2000000 0.8000000

## [Resample] iter 146:  0.0714286 0.9285714

## [Resample] iter 147:  0.0714286 0.9285714

## [Resample] iter 148:  0.0000000 1.0000000

## [Resample] iter 149:  0.0000000 1.0000000

## [Resample] iter 150:  0.1333333 0.8666667

## [Resample] iter 151:  0.0000000 1.0000000

## [Resample] iter 152:  0.2142857 0.7857143

## [Resample] iter 153:  0.0714286 0.9285714

## [Resample] iter 154:  0.0000000 1.0000000

## [Resample] iter 155:  0.0625000 0.9375000

## [Resample] iter 156:  0.1428571 0.8571429

## [Resample] iter 157:  0.0000000 1.0000000

## [Resample] iter 158:  0.2000000 0.8000000

## [Resample] iter 159:  0.1333333 0.8666667

## [Resample] iter 160:  0.0666667 0.9333333

## [Resample] iter 161:  0.0714286 0.9285714

## [Resample] iter 162:  0.1250000 0.8750000

## [Resample] iter 163:  0.0714286 0.9285714

## [Resample] iter 164:  0.1428571 0.8571429

## [Resample] iter 165:  0.1428571 0.8571429

## [Resample] iter 166:  0.0714286 0.9285714

## [Resample] iter 167:  0.0666667 0.9333333

## [Resample] iter 168:  0.0666667 0.9333333

## [Resample] iter 169:  0.2000000 0.8000000

## [Resample] iter 170:  0.0000000 1.0000000

## [Resample] iter 171:  0.0714286 0.9285714

## [Resample] iter 172:  0.0714286 0.9285714

## [Resample] iter 173:  0.0000000 1.0000000

## [Resample] iter 174:  0.1428571 0.8571429

## [Resample] iter 175:  0.0666667 0.9333333

## [Resample] iter 176:  0.2142857 0.7857143

## [Resample] iter 177:  0.0625000 0.9375000

## [Resample] iter 178:  0.0000000 1.0000000

## [Resample] iter 179:  0.1428571 0.8571429

## [Resample] iter 180:  0.1333333 0.8666667

## [Resample] iter 181:  0.0666667 0.9333333

## [Resample] iter 182:  0.1538462 0.8461538

## [Resample] iter 183:  0.0769231 0.9230769

## [Resample] iter 184:  0.1333333 0.8666667

## [Resample] iter 185:  0.2142857 0.7857143

## [Resample] iter 186:  0.1250000 0.8750000

## [Resample] iter 187:  0.1428571 0.8571429

## [Resample] iter 188:  0.1333333 0.8666667

## [Resample] iter 189:  0.2142857 0.7857143

## [Resample] iter 190:  0.0625000 0.9375000

## [Resample] iter 191:  0.1333333 0.8666667

## [Resample] iter 192:  0.1333333 0.8666667

## [Resample] iter 193:  0.0769231 0.9230769

## [Resample] iter 194:  0.2000000 0.8000000

## [Resample] iter 195:  0.1538462 0.8461538

## [Resample] iter 196:  0.1333333 0.8666667

## [Resample] iter 197:  0.2000000 0.8000000

## [Resample] iter 198:  0.1428571 0.8571429

## [Resample] iter 199:  0.0666667 0.9333333

## [Resample] iter 200:  0.0000000 1.0000000

## [Resample] iter 201:  0.0666667 0.9333333

## [Resample] iter 202:  0.0714286 0.9285714

## [Resample] iter 203:  0.1428571 0.8571429

## [Resample] iter 204:  0.2000000 0.8000000

## [Resample] iter 205:  0.0666667 0.9333333

## [Resample] iter 206:  0.0714286 0.9285714

## [Resample] iter 207:  0.0666667 0.9333333

## [Resample] iter 208:  0.1333333 0.8666667

## [Resample] iter 209:  0.2857143 0.7142857

## [Resample] iter 210:  0.0000000 1.0000000

## [Resample] iter 211:  0.0714286 0.9285714

## [Resample] iter 212:  0.0000000 1.0000000

## [Resample] iter 213:  0.2142857 0.7857143

## [Resample] iter 214:  0.1333333 0.8666667

## [Resample] iter 215:  0.2000000 0.8000000

## [Resample] iter 216:  0.0000000 1.0000000

## [Resample] iter 217:  0.0000000 1.0000000

## [Resample] iter 218:  0.0000000 1.0000000

## [Resample] iter 219:  0.1428571 0.8571429

## [Resample] iter 220:  0.0666667 0.9333333

## [Resample] iter 221:  0.1428571 0.8571429

## [Resample] iter 222:  0.1250000 0.8750000

## [Resample] iter 223:  0.0769231 0.9230769

## [Resample] iter 224:  0.1333333 0.8666667

## [Resample] iter 225:  0.2857143 0.7142857

## [Resample] iter 226:  0.0666667 0.9333333

## [Resample] iter 227:  0.1250000 0.8750000

## [Resample] iter 228:  0.0714286 0.9285714

## [Resample] iter 229:  0.0000000 1.0000000

## [Resample] iter 230:  0.0714286 0.9285714

## [Resample] iter 231:  0.0769231 0.9230769

## [Resample] iter 232:  0.0714286 0.9285714

## [Resample] iter 233:  0.2000000 0.8000000

## [Resample] iter 234:  0.0000000 1.0000000

## [Resample] iter 235:  0.2307692 0.7692308

## [Resample] iter 236:  0.0666667 0.9333333

## [Resample] iter 237:  0.0000000 1.0000000

## [Resample] iter 238:  0.2142857 0.7857143

## [Resample] iter 239:  0.0666667 0.9333333

## [Resample] iter 240:  0.0666667 0.9333333

## [Resample] iter 241:  0.2142857 0.7857143

## [Resample] iter 242:  0.0000000 1.0000000

## [Resample] iter 243:  0.0625000 0.9375000

## [Resample] iter 244:  0.2000000 0.8000000

## [Resample] iter 245:  0.0000000 1.0000000

## [Resample] iter 246:  0.0714286 0.9285714

## [Resample] iter 247:  0.0714286 0.9285714

## [Resample] iter 248:  0.1333333 0.8666667

## [Resample] iter 249:  0.0714286 0.9285714

## [Resample] iter 250:  0.1250000 0.8750000

## [Resample] iter 251:  0.0714286 0.9285714

## [Resample] iter 252:  0.0714286 0.9285714

## [Resample] iter 253:  0.0000000 1.0000000

## [Resample] iter 254:  0.1333333 0.8666667

## [Resample] iter 255:  0.0666667 0.9333333

## [Resample] iter 256:  0.2142857 0.7857143

## [Resample] iter 257:  0.0714286 0.9285714

## [Resample] iter 258:  0.1250000 0.8750000

## [Resample] iter 259:  0.0714286 0.9285714

## [Resample] iter 260:  0.0000000 1.0000000

## [Resample] iter 261:  0.0714286 0.9285714

## [Resample] iter 262:  0.0625000 0.9375000

## [Resample] iter 263:  0.0666667 0.9333333

## [Resample] iter 264:  0.0666667 0.9333333

## [Resample] iter 265:  0.0714286 0.9285714

## [Resample] iter 266:  0.1333333 0.8666667

## [Resample] iter 267:  0.0769231 0.9230769

## [Resample] iter 268:  0.2666667 0.7333333

## [Resample] iter 269:  0.1428571 0.8571429

## [Resample] iter 270:  0.0714286 0.9285714

## [Resample] iter 271:  0.2142857 0.7857143

## [Resample] iter 272:  0.0000000 1.0000000

## [Resample] iter 273:  0.0714286 0.9285714

## [Resample] iter 274:  0.0000000 1.0000000

## [Resample] iter 275:  0.2000000 0.8000000

## [Resample] iter 276:  0.0666667 0.9333333

## [Resample] iter 277:  0.1428571 0.8571429

## [Resample] iter 278:  0.1333333 0.8666667

## [Resample] iter 279:  0.0769231 0.9230769

## [Resample] iter 280:  0.0625000 0.9375000

## [Resample] iter 281:  0.0666667 0.9333333

## [Resample] iter 282:  0.1538462 0.8461538

## [Resample] iter 283:  0.1333333 0.8666667

## [Resample] iter 284:  0.0714286 0.9285714

## [Resample] iter 285:  0.1428571 0.8571429

## [Resample] iter 286:  0.1333333 0.8666667

## [Resample] iter 287:  0.1428571 0.8571429

## [Resample] iter 288:  0.0625000 0.9375000

## [Resample] iter 289:  0.2142857 0.7857143

## [Resample] iter 290:  0.2000000 0.8000000

## [Resample] iter 291:  0.1250000 0.8750000

## [Resample] iter 292:  0.0666667 0.9333333

## [Resample] iter 293:  0.0666667 0.9333333

## [Resample] iter 294:  0.2666667 0.7333333

## [Resample] iter 295:  0.2307692 0.7692308

## [Resample] iter 296:  0.0769231 0.9230769

## [Resample] iter 297:  0.0000000 1.0000000

## [Resample] iter 298:  0.0666667 0.9333333

## [Resample] iter 299:  0.1428571 0.8571429

## [Resample] iter 300:  0.0000000 1.0000000

## [Resample] iter 301:  0.1333333 0.8666667

## [Resample] iter 302:  0.0666667 0.9333333

## [Resample] iter 303:  0.0666667 0.9333333

## [Resample] iter 304:  0.2000000 0.8000000

## [Resample] iter 305:  0.0000000 1.0000000

## [Resample] iter 306:  0.2857143 0.7142857

## [Resample] iter 307:  0.0000000 1.0000000

## [Resample] iter 308:  0.2666667 0.7333333

## [Resample] iter 309:  0.0769231 0.9230769

## [Resample] iter 310:  0.2142857 0.7857143

## [Resample] iter 311:  0.0666667 0.9333333

## [Resample] iter 312:  0.1428571 0.8571429

## [Resample] iter 313:  0.1428571 0.8571429

## [Resample] iter 314:  0.4000000 0.6000000

## [Resample] iter 315:  0.0666667 0.9333333

## [Resample] iter 316:  0.0000000 1.0000000

## [Resample] iter 317:  0.0000000 1.0000000

## [Resample] iter 318:  0.0000000 1.0000000

## [Resample] iter 319:  0.0666667 0.9333333

## [Resample] iter 320:  0.0000000 1.0000000

## [Resample] iter 321:  0.1428571 0.8571429

## [Resample] iter 322:  0.0666667 0.9333333

## [Resample] iter 323:  0.0000000 1.0000000

## [Resample] iter 324:  0.2142857 0.7857143

## [Resample] iter 325:  0.0000000 1.0000000

## [Resample] iter 326:  0.0714286 0.9285714

## [Resample] iter 327:  0.1875000 0.8125000

## [Resample] iter 328:  0.1333333 0.8666667

## [Resample] iter 329:  0.1333333 0.8666667

## [Resample] iter 330:  0.1428571 0.8571429

## [Resample] iter 331:  0.1333333 0.8666667

## [Resample] iter 332:  0.0714286 0.9285714

## [Resample] iter 333:  0.2857143 0.7142857

## [Resample] iter 334:  0.0714286 0.9285714

## [Resample] iter 335:  0.1333333 0.8666667

## [Resample] iter 336:  0.0000000 1.0000000

## [Resample] iter 337:  0.2142857 0.7857143

## [Resample] iter 338:  0.0666667 0.9333333

## [Resample] iter 339:  0.2142857 0.7857143

## [Resample] iter 340:  0.0666667 0.9333333

## [Resample] iter 341:  0.0666667 0.9333333

## [Resample] iter 342:  0.1428571 0.8571429

## [Resample] iter 343:  0.1250000 0.8750000

## [Resample] iter 344:  0.1333333 0.8666667

## [Resample] iter 345:  0.0666667 0.9333333

## [Resample] iter 346:  0.0000000 1.0000000

## [Resample] iter 347:  0.2666667 0.7333333

## [Resample] iter 348:  0.0769231 0.9230769

## [Resample] iter 349:  0.0714286 0.9285714

## [Resample] iter 350:  0.0714286 0.9285714

## [Resample] iter 351:  0.0714286 0.9285714

## [Resample] iter 352:  0.1333333 0.8666667

## [Resample] iter 353:  0.0769231 0.9230769

## [Resample] iter 354:  0.0666667 0.9333333

## [Resample] iter 355:  0.1875000 0.8125000

## [Resample] iter 356:  0.2000000 0.8000000

## [Resample] iter 357:  0.1333333 0.8666667

## [Resample] iter 358:  0.1333333 0.8666667

## [Resample] iter 359:  0.0000000 1.0000000

## [Resample] iter 360:  0.1538462 0.8461538

## [Resample] iter 361:  0.0666667 0.9333333

## [Resample] iter 362:  0.0666667 0.9333333

## [Resample] iter 363:  0.1875000 0.8125000

## [Resample] iter 364:  0.0000000 1.0000000

## [Resample] iter 365:  0.2857143 0.7142857

## [Resample] iter 366:  0.0714286 0.9285714

## [Resample] iter 367:  0.0666667 0.9333333

## [Resample] iter 368:  0.0666667 0.9333333

## [Resample] iter 369:  0.0769231 0.9230769

## [Resample] iter 370:  0.0000000 1.0000000

## [Resample] iter 371:  0.0714286 0.9285714

## [Resample] iter 372:  0.0000000 1.0000000

## [Resample] iter 373:  0.2142857 0.7857143

## [Resample] iter 374:  0.2000000 0.8000000

## [Resample] iter 375:  0.3125000 0.6875000

## [Resample] iter 376:  0.1333333 0.8666667

## [Resample] iter 377:  0.1428571 0.8571429

## [Resample] iter 378:  0.0666667 0.9333333

## [Resample] iter 379:  0.0666667 0.9333333

## [Resample] iter 380:  0.1538462 0.8461538

## [Resample] iter 381:  0.2857143 0.7142857

## [Resample] iter 382:  0.2000000 0.8000000

## [Resample] iter 383:  0.0714286 0.9285714

## [Resample] iter 384:  0.1538462 0.8461538

## [Resample] iter 385:  0.0666667 0.9333333

## [Resample] iter 386:  0.0000000 1.0000000

## [Resample] iter 387:  0.0714286 0.9285714

## [Resample] iter 388:  0.2666667 0.7333333

## [Resample] iter 389:  0.0000000 1.0000000

## [Resample] iter 390:  0.2000000 0.8000000

## [Resample] iter 391:  0.0714286 0.9285714

## [Resample] iter 392:  0.1428571 0.8571429

## [Resample] iter 393:  0.3571429 0.6428571

## [Resample] iter 394:  0.1333333 0.8666667

## [Resample] iter 395:  0.0000000 1.0000000

## [Resample] iter 396:  0.0000000 1.0000000

## [Resample] iter 397:  0.2666667 0.7333333

## [Resample] iter 398:  0.1333333 0.8666667

## [Resample] iter 399:  0.0000000 1.0000000

## [Resample] iter 400:  0.0000000 1.0000000

## [Resample] iter 401:  0.2142857 0.7857143

## [Resample] iter 402:  0.0666667 0.9333333

## [Resample] iter 403:  0.0714286 0.9285714

## [Resample] iter 404:  0.1428571 0.8571429

## [Resample] iter 405:  0.0000000 1.0000000

## [Resample] iter 406:  0.0000000 1.0000000

## [Resample] iter 407:  0.0666667 0.9333333

## [Resample] iter 408:  0.1428571 0.8571429

## [Resample] iter 409:  0.0714286 0.9285714

## [Resample] iter 410:  0.1538462 0.8461538

## [Resample] iter 411:  0.0000000 1.0000000

## [Resample] iter 412:  0.0000000 1.0000000

## [Resample] iter 413:  0.1333333 0.8666667

## [Resample] iter 414:  0.1538462 0.8461538

## [Resample] iter 415:  0.0666667 0.9333333

## [Resample] iter 416:  0.0666667 0.9333333

## [Resample] iter 417:  0.2307692 0.7692308

## [Resample] iter 418:  0.2666667 0.7333333

## [Resample] iter 419:  0.0714286 0.9285714

## [Resample] iter 420:  0.0625000 0.9375000

## [Resample] iter 421:  0.0666667 0.9333333

## [Resample] iter 422:  0.2500000 0.7500000

## [Resample] iter 423:  0.0000000 1.0000000

## [Resample] iter 424:  0.2000000 0.8000000

## [Resample] iter 425:  0.1333333 0.8666667

## [Resample] iter 426:  0.0000000 1.0000000

## [Resample] iter 427:  0.0666667 0.9333333

## [Resample] iter 428:  0.1538462 0.8461538

## [Resample] iter 429:  0.2142857 0.7857143

## [Resample] iter 430:  0.0666667 0.9333333

## [Resample] iter 431:  0.1250000 0.8750000

## [Resample] iter 432:  0.1428571 0.8571429

## [Resample] iter 433:  0.0666667 0.9333333

## [Resample] iter 434:  0.0000000 1.0000000

## [Resample] iter 435:  0.2000000 0.8000000

## [Resample] iter 436:  0.0000000 1.0000000

## [Resample] iter 437:  0.1428571 0.8571429

## [Resample] iter 438:  0.1333333 0.8666667

## [Resample] iter 439:  0.0769231 0.9230769

## [Resample] iter 440:  0.0714286 0.9285714

## [Resample] iter 441:  0.0000000 1.0000000

## [Resample] iter 442:  0.2666667 0.7333333

## [Resample] iter 443:  0.0000000 1.0000000

## [Resample] iter 444:  0.0666667 0.9333333

## [Resample] iter 445:  0.1333333 0.8666667

## [Resample] iter 446:  0.2857143 0.7142857

## [Resample] iter 447:  0.1428571 0.8571429

## [Resample] iter 448:  0.0714286 0.9285714

## [Resample] iter 449:  0.2142857 0.7857143

## [Resample] iter 450:  0.0666667 0.9333333

## [Resample] iter 451:  0.0714286 0.9285714

## [Resample] iter 452:  0.0000000 1.0000000

## [Resample] iter 453:  0.2000000 0.8000000

## [Resample] iter 454:  0.0714286 0.9285714

## [Resample] iter 455:  0.0666667 0.9333333

## [Resample] iter 456:  0.1538462 0.8461538

## [Resample] iter 457:  0.1333333 0.8666667

## [Resample] iter 458:  0.0714286 0.9285714

## [Resample] iter 459:  0.1875000 0.8125000

## [Resample] iter 460:  0.1538462 0.8461538

## [Resample] iter 461:  0.1333333 0.8666667

## [Resample] iter 462:  0.1428571 0.8571429

## [Resample] iter 463:  0.0625000 0.9375000

## [Resample] iter 464:  0.0714286 0.9285714

## [Resample] iter 465:  0.0000000 1.0000000

## [Resample] iter 466:  0.1250000 0.8750000

## [Resample] iter 467:  0.0769231 0.9230769

## [Resample] iter 468:  0.0714286 0.9285714

## [Resample] iter 469:  0.0666667 0.9333333

## [Resample] iter 470:  0.1428571 0.8571429

## [Resample] iter 471:  0.0714286 0.9285714

## [Resample] iter 472:  0.0714286 0.9285714

## [Resample] iter 473:  0.1333333 0.8666667

## [Resample] iter 474:  0.3076923 0.6923077

## [Resample] iter 475:  0.1333333 0.8666667

## [Resample] iter 476:  0.2000000 0.8000000

## [Resample] iter 477:  0.0000000 1.0000000

## [Resample] iter 478:  0.0000000 1.0000000

## [Resample] iter 479:  0.0666667 0.9333333

## [Resample] iter 480:  0.0666667 0.9333333

## [Resample] iter 481:  0.0625000 0.9375000

## [Resample] iter 482:  0.0000000 1.0000000

## [Resample] iter 483:  0.0666667 0.9333333

## [Resample] iter 484:  0.0000000 1.0000000

## [Resample] iter 485:  0.0000000 1.0000000

## [Resample] iter 486:  0.1428571 0.8571429

## [Resample] iter 487:  0.2000000 0.8000000

## [Resample] iter 488:  0.1538462 0.8461538

## [Resample] iter 489:  0.1428571 0.8571429

## [Resample] iter 490:  0.0769231 0.9230769

## [Resample] iter 491:  0.0666667 0.9333333

## [Resample] iter 492:  0.0000000 1.0000000

## [Resample] iter 493:  0.0714286 0.9285714

## [Resample] iter 494:  0.1428571 0.8571429

## [Resample] iter 495:  0.1428571 0.8571429

## [Resample] iter 496:  0.0666667 0.9333333

## [Resample] iter 497:  0.0000000 1.0000000

## [Resample] iter 498:  0.0714286 0.9285714

## [Resample] iter 499:  0.2000000 0.8000000

## [Resample] iter 500:  0.0000000 1.0000000

## 

## Aggregated Result: mmce.test.mean=0.1066920,acc.test.mean=0.8933080

## 

average performance measures:

kFoldCV$aggr

## mmce.test.mean  acc.test.mean 
##       0.106692       0.893308

Exercise 3

Define two new resampling descriptions: one that performs 3-fold CV repeated 5 times, and one that performs 3-fold CV repeated 500 times (don’t overwrite your existing description). Use the resample() function to cross-validate the kNN algorithm using both of these resampling descriptions. Repeat the resampling five times for each method, and see which one gives more stable results.

Confusion Matrix

calculateConfusionMatrix(kFoldCV$pred, relative = TRUE)

## Relative confusion matrix (normalized by row/column):
##           predicted
## true       Chemical  Normal    Overt     -err.-   
##   Chemical 0.81/0.78 0.10/0.05 0.09/0.11 0.19     
##   Normal   0.04/0.08 0.96/0.95 0.00/0.00 0.04     
##   Overt    0.16/0.14 0.00/0.00 0.84/0.89 0.16     
##   -err.-        0.22      0.05      0.11 0.11     
## 
## 
## Absolute confusion matrix:
##           predicted
## true       Chemical Normal Overt -err.-
##   Chemical     1449    186   165    351
##   Normal        156   3644     0    156
##   Overt         264      0  1386    264
##   -err.-        420    186   165    771

Cross-validation: method = LOO

LOO <- makeResampleDesc(method = "LOO")

Exercise 4

Try to create two new leave-one-out resampling descriptions: one that uses stratified sampling, and one that repeats the procedure five times. What happens?

LOO2 <- makeResampleDesc(method = "LOO", 
#                          folds = 10, 
#                          reps = 5,
                          stratify = FALSE)

Answer: Errors

Resample LOO

LOOCV <- resample(learner = knn, task = diabetesTask,
                    resampling = LOO, 
                    measures = list(mmce, acc)) # Performance Measures

## Resampling: LOO

## Measures:             mmce      acc

## [Resample] iter 1:    0.0000000 1.0000000

## [Resample] iter 2:    0.0000000 1.0000000

## [Resample] iter 3:    0.0000000 1.0000000

## [Resample] iter 4:    0.0000000 1.0000000

## [Resample] iter 5:    0.0000000 1.0000000

## [Resample] iter 6:    0.0000000 1.0000000

## [Resample] iter 7:    0.0000000 1.0000000

## [Resample] iter 8:    0.0000000 1.0000000

## [Resample] iter 9:    0.0000000 1.0000000

## [Resample] iter 10:   0.0000000 1.0000000

## [Resample] iter 11:   0.0000000 1.0000000

## [Resample] iter 12:   0.0000000 1.0000000

## [Resample] iter 13:   0.0000000 1.0000000

## [Resample] iter 14:   0.0000000 1.0000000

## [Resample] iter 15:   0.0000000 1.0000000

## [Resample] iter 16:   0.0000000 1.0000000

## [Resample] iter 17:   0.0000000 1.0000000

## [Resample] iter 18:   0.0000000 1.0000000

## [Resample] iter 19:   0.0000000 1.0000000

## [Resample] iter 20:   0.0000000 1.0000000

## [Resample] iter 21:   0.0000000 1.0000000

## [Resample] iter 22:   0.0000000 1.0000000

## [Resample] iter 23:   0.0000000 1.0000000

## [Resample] iter 24:   0.0000000 1.0000000

## [Resample] iter 25:   0.0000000 1.0000000

## [Resample] iter 26:   0.0000000 1.0000000

## [Resample] iter 27:   0.0000000 1.0000000

## [Resample] iter 28:   0.0000000 1.0000000

## [Resample] iter 29:   0.0000000 1.0000000

## [Resample] iter 30:   0.0000000 1.0000000

## [Resample] iter 31:   0.0000000 1.0000000

## [Resample] iter 32:   0.0000000 1.0000000

## [Resample] iter 33:   0.0000000 1.0000000

## [Resample] iter 34:   0.0000000 1.0000000

## [Resample] iter 35:   0.0000000 1.0000000

## [Resample] iter 36:   0.0000000 1.0000000

## [Resample] iter 37:   0.0000000 1.0000000

## [Resample] iter 38:   0.0000000 1.0000000

## [Resample] iter 39:   0.0000000 1.0000000

## [Resample] iter 40:   0.0000000 1.0000000

## [Resample] iter 41:   0.0000000 1.0000000

## [Resample] iter 42:   0.0000000 1.0000000

## [Resample] iter 43:   0.0000000 1.0000000

## [Resample] iter 44:   0.0000000 1.0000000

## [Resample] iter 45:   0.0000000 1.0000000

## [Resample] iter 46:   0.0000000 1.0000000

## [Resample] iter 47:   0.0000000 1.0000000

## [Resample] iter 48:   0.0000000 1.0000000

## [Resample] iter 49:   0.0000000 1.0000000

## [Resample] iter 50:   0.0000000 1.0000000

## [Resample] iter 51:   0.0000000 1.0000000

## [Resample] iter 52:   0.0000000 1.0000000

## [Resample] iter 53:   0.0000000 1.0000000

## [Resample] iter 54:   0.0000000 1.0000000

## [Resample] iter 55:   0.0000000 1.0000000

## [Resample] iter 56:   0.0000000 1.0000000

## [Resample] iter 57:   0.0000000 1.0000000

## [Resample] iter 58:   0.0000000 1.0000000

## [Resample] iter 59:   0.0000000 1.0000000

## [Resample] iter 60:   0.0000000 1.0000000

## [Resample] iter 61:   0.0000000 1.0000000

## [Resample] iter 62:   0.0000000 1.0000000

## [Resample] iter 63:   0.0000000 1.0000000

## [Resample] iter 64:   0.0000000 1.0000000

## [Resample] iter 65:   0.0000000 1.0000000

## [Resample] iter 66:   1.0000000 0.0000000

## [Resample] iter 67:   1.0000000 0.0000000

## [Resample] iter 68:   0.0000000 1.0000000

## [Resample] iter 69:   1.0000000 0.0000000

## [Resample] iter 70:   0.0000000 1.0000000

## [Resample] iter 71:   0.0000000 1.0000000

## [Resample] iter 72:   0.0000000 1.0000000

## [Resample] iter 73:   0.0000000 1.0000000

## [Resample] iter 74:   0.0000000 1.0000000

## [Resample] iter 75:   0.0000000 1.0000000

## [Resample] iter 76:   0.0000000 1.0000000

## [Resample] iter 77:   0.0000000 1.0000000

## [Resample] iter 78:   0.0000000 1.0000000

## [Resample] iter 79:   0.0000000 1.0000000

## [Resample] iter 80:   0.0000000 1.0000000

## [Resample] iter 81:   0.0000000 1.0000000

## [Resample] iter 82:   0.0000000 1.0000000

## [Resample] iter 83:   1.0000000 0.0000000

## [Resample] iter 84:   0.0000000 1.0000000

## [Resample] iter 85:   0.0000000 1.0000000

## [Resample] iter 86:   0.0000000 1.0000000

## [Resample] iter 87:   0.0000000 1.0000000

## [Resample] iter 88:   0.0000000 1.0000000

## [Resample] iter 89:   0.0000000 1.0000000

## [Resample] iter 90:   0.0000000 1.0000000

## [Resample] iter 91:   0.0000000 1.0000000

## [Resample] iter 92:   1.0000000 0.0000000

## [Resample] iter 93:   0.0000000 1.0000000

## [Resample] iter 94:   0.0000000 1.0000000

## [Resample] iter 95:   0.0000000 1.0000000

## [Resample] iter 96:   0.0000000 1.0000000

## [Resample] iter 97:   0.0000000 1.0000000

## [Resample] iter 98:   0.0000000 1.0000000

## [Resample] iter 99:   0.0000000 1.0000000

## [Resample] iter 100:  0.0000000 1.0000000

## [Resample] iter 101:  0.0000000 1.0000000

## [Resample] iter 102:  0.0000000 1.0000000

## [Resample] iter 103:  0.0000000 1.0000000

## [Resample] iter 104:  1.0000000 0.0000000

## [Resample] iter 105:  1.0000000 0.0000000

## [Resample] iter 106:  0.0000000 1.0000000

## [Resample] iter 107:  0.0000000 1.0000000

## [Resample] iter 108:  0.0000000 1.0000000

## [Resample] iter 109:  1.0000000 0.0000000

## [Resample] iter 110:  0.0000000 1.0000000

## [Resample] iter 111:  0.0000000 1.0000000

## [Resample] iter 112:  1.0000000 0.0000000

## [Resample] iter 113:  0.0000000 1.0000000

## [Resample] iter 114:  0.0000000 1.0000000

## [Resample] iter 115:  1.0000000 0.0000000

## [Resample] iter 116:  0.0000000 1.0000000

## [Resample] iter 117:  0.0000000 1.0000000

## [Resample] iter 118:  0.0000000 1.0000000

## [Resample] iter 119:  0.0000000 1.0000000

## [Resample] iter 120:  0.0000000 1.0000000

## [Resample] iter 121:  0.0000000 1.0000000

## [Resample] iter 122:  0.0000000 1.0000000

## [Resample] iter 123:  0.0000000 1.0000000

## [Resample] iter 124:  1.0000000 0.0000000

## [Resample] iter 125:  0.0000000 1.0000000

## [Resample] iter 126:  0.0000000 1.0000000

## [Resample] iter 127:  0.0000000 1.0000000

## [Resample] iter 128:  0.0000000 1.0000000

## [Resample] iter 129:  0.0000000 1.0000000

## [Resample] iter 130:  0.0000000 1.0000000

## [Resample] iter 131:  1.0000000 0.0000000

## [Resample] iter 132:  0.0000000 1.0000000

## [Resample] iter 133:  0.0000000 1.0000000

## [Resample] iter 134:  1.0000000 0.0000000

## [Resample] iter 135:  0.0000000 1.0000000

## [Resample] iter 136:  1.0000000 0.0000000

## [Resample] iter 137:  0.0000000 1.0000000

## [Resample] iter 138:  0.0000000 1.0000000

## [Resample] iter 139:  0.0000000 1.0000000

## [Resample] iter 140:  0.0000000 1.0000000

## [Resample] iter 141:  0.0000000 1.0000000

## [Resample] iter 142:  0.0000000 1.0000000

## [Resample] iter 143:  0.0000000 1.0000000

## [Resample] iter 144:  0.0000000 1.0000000

## [Resample] iter 145:  0.0000000 1.0000000

## 

## Aggregated Result: mmce.test.mean=0.0965517,acc.test.mean=0.9034483

## 

LOOCV$aggr

LOOCV$aggr

## mmce.test.mean  acc.test.mean 
##     0.09655172     0.90344828

Confusion Matrix

calculateConfusionMatrix(LOOCV$pred, relative = TRUE)

## Relative confusion matrix (normalized by row/column):
##           predicted
## true       Chemical  Normal    Overt     -err.-   
##   Chemical 0.81/0.81 0.11/0.05 0.08/0.10 0.19     
##   Normal   0.03/0.06 0.97/0.95 0.00/0.00 0.03     
##   Overt    0.15/0.14 0.00/0.00 0.85/0.90 0.15     
##   -err.-        0.19      0.05      0.10 0.10     
## 
## 
## Absolute confusion matrix:
##           predicted
## true       Chemical Normal Overt -err.-
##   Chemical       29      4     3      7
##   Normal          2     74     0      2
##   Overt           5      0    28      5
##   -err.-          7      4     3     14

3.6 Hyperparameters