## ----setup-------------------------------------------------------------------- # nolint start library(mlexperiments) ## ----------------------------------------------------------------------------- library(mlbench) data("BreastCancer") dataset <- BreastCancer |> data.table::as.data.table() |> na.omit() seed <- 123 feature_cols <- colnames(dataset)[2:10] target_col <- "Class" to_num <- c( "Cl.thickness", "Cell.size", "Cell.shape", "Marg.adhesion", "Epith.c.size" ) dataset[, (to_num) := lapply(.SD, as.numeric), .SDcols = to_num] ## ----------------------------------------------------------------------------- seed <- 123 if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) { # on cran ncores <- 2L } else { ncores <- ifelse( test = parallel::detectCores() > 4, yes = 4L, no = ifelse( test = parallel::detectCores() < 2L, yes = 1L, no = parallel::detectCores() ) ) } options("mlexperiments.bayesian.max_init" = 4L) ## ----------------------------------------------------------------------------- data_split <- splitTools::partition( y = dataset[, get(target_col)], p = c(train = 0.7, test = 0.3), type = "stratified", seed = seed ) train_x <- model.matrix( ~ -1 + ., dataset[data_split$train, .SD, .SDcols = feature_cols] ) train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L test_x <- model.matrix( ~ -1 + ., dataset[data_split$test, .SD, .SDcols = feature_cols] ) test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L ## ----------------------------------------------------------------------------- fold_list <- splitTools::create_folds( y = train_y, k = 3, type = "stratified", seed = seed ) ## ----------------------------------------------------------------------------- # required learner arguments, not optimized learner_args <- list( l = 2, test = parse(text = "fold_test$x"), use.all = FALSE ) # set arguments for predict function and performance metric, # required for mlexperiments::MLCrossValidation and # mlexperiments::MLNestedCV predict_args <- list(type = "response") performance_metric <- metric("ACC") performance_metric_args <- NULL return_models <- FALSE # required for grid search and initialization of bayesian optimization parameter_grid <- expand.grid( k = seq(4, 68, 6) ) # reduce to a maximum of 10 rows if (nrow(parameter_grid) > 10) { set.seed(123) sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE) parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows) } # required for bayesian optimization parameter_bounds <- list(k = c(2L, 80L)) optim_args <- list( n_iter = ncores, kappa = 3.5, acq = "ucb" ) ## ----------------------------------------------------------------------------- tuner <- mlexperiments::MLTuneParameters$new( learner = LearnerKnn$new(), strategy = "grid", ncores = ncores, seed = seed ) tuner$parameter_grid <- parameter_grid tuner$learner_args <- learner_args tuner$split_type <- "stratified" tuner$set_data( x = train_x, y = train_y ) tuner_results_grid <- tuner$execute(k = 3) head(tuner_results_grid) #> setting_id metric_optim_mean k l use.all #> #> 1: 1 0.03979364 16 2 FALSE #> 2: 2 0.06281546 64 2 FALSE #> 3: 3 0.03771031 10 2 FALSE #> 4: 4 0.04393394 34 2 FALSE #> 5: 5 0.05862242 58 2 FALSE #> 6: 6 0.04393394 28 2 FALSE ## ----------------------------------------------------------------------------- tuner <- mlexperiments::MLTuneParameters$new( learner = LearnerKnn$new(), strategy = "bayesian", ncores = ncores, seed = seed ) tuner$parameter_grid <- parameter_grid tuner$parameter_bounds <- parameter_bounds tuner$learner_args <- learner_args tuner$optim_args <- optim_args tuner$split_type <- "stratified" tuner$set_data( x = train_x, y = train_y ) tuner_results_bayesian <- tuner$execute(k = 3) #> #> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'... #> ... reducing initialization grid to 4 rows. #> elapsed = 0.016 Round = 1 k = 10.0000 Value = -0.03771031 #> elapsed = 0.02 Round = 2 k = 4.0000 Value = -0.04607001 #> elapsed = 0.014 Round = 3 k = 64.0000 Value = -0.06281546 #> elapsed = 0.013 Round = 4 k = 52.0000 Value = -0.06070576 #> elapsed = 0.012 Round = 5 k = 25.0000 Value = -0.0418506 #> elapsed = 0.015 Round = 6 k = 80.0000 Value = -0.05860932 #> elapsed = 0.011 Round = 7 k = 17.0000 Value = -0.03976727 #> elapsed = 0.013 Round = 8 k = 37.0000 Value = -0.04811371 #> #> Best Parameters Found: #> Round = 1 k = 10.0000 Value = -0.03771031 head(tuner_results_bayesian) #> setting_id k Value l use.all metric_optim_mean #> #> 1: 1 10 -0.03771031 2 FALSE 0.03771031 #> 2: 2 4 -0.04607001 2 FALSE 0.04607001 #> 3: 3 64 -0.06281546 2 FALSE 0.06281546 #> 4: 4 52 -0.06070576 2 FALSE 0.06070576 #> 5: 5 25 -0.04185060 2 FALSE 0.04185060 #> 6: 6 80 -0.05860932 2 FALSE 0.05860932 ## ----------------------------------------------------------------------------- validator <- mlexperiments::MLCrossValidation$new( learner = LearnerKnn$new(), fold_list = fold_list, ncores = ncores, seed = seed ) validator$learner_args <- tuner$results$best.setting validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- return_models validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> CV fold: Fold2 #> #> CV fold: Fold3 head(validator_results) #> fold performance k l use.all #> #> 1: Fold1 0.9746835 10 2 FALSE #> 2: Fold2 0.9496855 10 2 FALSE #> 3: Fold3 0.9625000 10 2 FALSE ## ----------------------------------------------------------------------------- validator <- mlexperiments::MLNestedCV$new( learner = LearnerKnn$new(), strategy = "grid", fold_list = fold_list, k_tuning = 3L, ncores = ncores, seed = seed ) validator$parameter_grid <- parameter_grid validator$learner_args <- learner_args validator$split_type <- "stratified" validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- return_models validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> CV fold: Fold2 #> #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> head(validator_results) #> fold performance k l use.all #> #> 1: Fold1 0.9746835 10 2 FALSE #> 2: Fold2 0.9496855 10 2 FALSE #> 3: Fold3 0.9625000 10 2 FALSE ## ----------------------------------------------------------------------------- validator <- mlexperiments::MLNestedCV$new( learner = LearnerKnn$new(), strategy = "bayesian", fold_list = fold_list, k_tuning = 3L, ncores = ncores, seed = seed ) validator$parameter_grid <- parameter_grid validator$learner_args <- learner_args validator$split_type <- "stratified" validator$parameter_bounds <- parameter_bounds validator$optim_args <- optim_args validator$predict_args <- predict_args validator$performance_metric <- performance_metric validator$performance_metric_args <- performance_metric_args validator$return_models <- return_models validator$set_data( x = train_x, y = train_y ) validator_results <- validator$execute() #> #> CV fold: Fold1 #> #> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'... #> ... reducing initialization grid to 4 rows. #> elapsed = 0.009 Round = 1 k = 10.0000 Value = -0.04699348 #> elapsed = 0.008 Round = 2 k = 4.0000 Value = -0.05639805 #> elapsed = 0.01 Round = 3 k = 64.0000 Value = -0.07523658 #> elapsed = 0.01 Round = 4 k = 52.0000 Value = -0.07209193 #> elapsed = 0.009 Round = 5 k = 25.0000 Value = -0.05331217 #> elapsed = 0.011 Round = 6 k = 80.0000 Value = -0.08152589 #> elapsed = 0.009 Round = 7 k = 17.0000 Value = -0.05016752 #> elapsed = 0.011 Round = 8 k = 37.0000 Value = -0.06271675 #> #> Best Parameters Found: #> Round = 1 k = 10.0000 Value = -0.04699348 #> #> CV fold: Fold2 #> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%) #> #> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'... #> ... reducing initialization grid to 4 rows. #> elapsed = 0.009 Round = 1 k = 10.0000 Value = -0.02830189 #> elapsed = 0.009 Round = 2 k = 4.0000 Value = -0.0408805 #> elapsed = 0.01 Round = 3 k = 64.0000 Value = -0.07232704 #> elapsed = 0.01 Round = 4 k = 52.0000 Value = -0.05974843 #> elapsed = 0.01 Round = 5 k = 25.0000 Value = -0.03773585 #> elapsed = 0.01 Round = 6 k = 80.0000 Value = -0.08490566 #> elapsed = 0.01 Round = 7 k = 17.0000 Value = -0.02830189 #> elapsed = 0.01 Round = 8 k = 39.0000 Value = -0.05031447 #> #> Best Parameters Found: #> Round = 1 k = 10.0000 Value = -0.02830189 #> #> CV fold: Fold3 #> CV progress [========================================================================================================] 3/3 (100%) #> #> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'... #> ... reducing initialization grid to 4 rows. #> elapsed = 0.01 Round = 1 k = 10.0000 Value = -0.04117999 #> elapsed = 0.008 Round = 2 k = 4.0000 Value = -0.0442947 #> elapsed = 0.01 Round = 3 k = 64.0000 Value = -0.06004792 #> elapsed = 0.01 Round = 4 k = 52.0000 Value = -0.05687332 #> elapsed = 0.009 Round = 5 k = 27.0000 Value = -0.04432465 #> elapsed = 0.009 Round = 6 k = 17.0000 Value = -0.04432465 #> elapsed = 0.011 Round = 7 k = 80.0000 Value = -0.06951183 #> elapsed = 0.01 Round = 8 k = 38.0000 Value = -0.05372866 #> #> Best Parameters Found: #> Round = 1 k = 10.0000 Value = -0.04117999 head(validator_results) #> fold performance k l use.all #> #> 1: Fold1 0.9746835 10 2 FALSE #> 2: Fold2 0.9496855 10 2 FALSE #> 3: Fold3 0.9625000 10 2 FALSE ## ----include=FALSE------------------------------------------------------------ # nolint end .