From e7e94d3590d4dfcaaeb0cbf41fca5a8d2150776b Mon Sep 17 00:00:00 2001 From: qiushi Date: Fri, 9 Sep 2022 13:54:50 -0500 Subject: [PATCH 1/3] set curl options --- tests/testthat.R | 6 +- tests/testthat/_snaps/tune_grid.md | 110 ++++++++++++++++++++++++ tests/testthat/test-case-weights.R | 9 +- tests/testthat/test-misc.R | 12 --- tests/testthat/test-parsnip-execution.R | 34 -------- tests/testthat/test-tune_grid.R | 15 ---- 6 files changed, 115 insertions(+), 71 deletions(-) create mode 100644 tests/testthat/_snaps/tune_grid.md diff --git a/tests/testthat.R b/tests/testthat.R index 84fc996..23f26c9 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,9 +1,5 @@ library(testthat) library(agua) -# Working on figuring out an issue where the first test runs but others fail with -# Unexpected CURL error: Failed to connect to localhost port 54321 after 1 ms: Connection refused -# [1] "Job request failed Unexpected CURL error: Failed to connect to localhost port 54321 after 1 ms: Connection refused, will retry after 3s." -agua::h2o_start() test_check("agua") -h2o::h2o.shutdown(prompt = FALSE) +option("prefer_RCurl") == FALSE diff --git a/tests/testthat/_snaps/tune_grid.md b/tests/testthat/_snaps/tune_grid.md new file mode 100644 index 0000000..a962536 --- /dev/null +++ b/tests/testthat/_snaps/tune_grid.md @@ -0,0 +1,110 @@ +# tune model only (with id) + + Code + res + Output + # Tuning results + # 5-fold cross-validation + # A tibble: 5 x 5 + splits id .metrics .notes .predictions + + 1 Fold1 + 2 Fold2 + 3 Fold3 + 4 Fold4 + 5 Fold5 + + There were issues with some computations: + + - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... + + Run `show_notes(.Last.tune.result)` for more information. + +# tune model only (without id) + + Code + res + Output + # Tuning results + # 5-fold cross-validation + # A tibble: 5 x 5 + splits id .metrics .notes .predictions + + 1 Fold1 + 2 Fold2 + 3 Fold3 + 4 Fold4 + 5 Fold5 + + There were issues with some computations: + + - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... + + Run `show_notes(.Last.tune.result)` for more information. + +# tune model only (with id and recipe) + + Code + res + Output + # Tuning results + # 5-fold cross-validation + # A tibble: 5 x 5 + splits id .metrics .notes .predictions + + 1 Fold1 + 2 Fold2 + 3 Fold3 + 4 Fold4 + 5 Fold5 + + There were issues with some computations: + + - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... + + Run `show_notes(.Last.tune.result)` for more information. + +# tune model and recipe + + Code + res + Output + # Tuning results + # 5-fold cross-validation + # A tibble: 5 x 5 + splits id .metrics .notes .predictions + + 1 Fold1 + 2 Fold2 + 3 Fold3 + 4 Fold4 + 5 Fold5 + + There were issues with some computations: + + - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... + + Run `show_notes(.Last.tune.result)` for more information. + +# tune with backend options parallelism + + Code + res + Output + # Tuning results + # 5-fold cross-validation + # A tibble: 5 x 5 + splits id .metrics .notes .predictions + + 1 Fold1 + 2 Fold2 + 3 Fold3 + 4 Fold4 + 5 Fold5 + + There were issues with some computations: + + - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... + + Run `show_notes(.Last.tune.result)` for more information. + diff --git a/tests/testthat/test-case-weights.R b/tests/testthat/test-case-weights.R index a1fb91e..41e1edc 100644 --- a/tests/testthat/test-case-weights.R +++ b/tests/testthat/test-case-weights.R @@ -1,5 +1,4 @@ test_that("case weights works", { - skip_if(!interactive()) skip_if_not_installed("modeldata") h2o_start() @@ -18,10 +17,10 @@ test_that("case weights works", { set.seed(1) fit_res <- spec %>% fit(Class ~ ., data = two_class_dat, case_weights = wts) - wf_res <- workflow() %>% - add_model(spec) %>% - add_formula(Class ~ .) %>% - add_case_weights(wts) %>% + wf_res <- workflows::workflow() %>% + workflows::add_model(spec) %>% + workflows::add_formula(Class ~ .) %>% + workflows::add_case_weights(wts) %>% fit(data = two_class_dat %>% dplyr::mutate(wts = wts)) diff --git a/tests/testthat/test-misc.R b/tests/testthat/test-misc.R index 82c3713..9d1c420 100644 --- a/tests/testthat/test-misc.R +++ b/tests/testthat/test-misc.R @@ -1,5 +1,4 @@ test_that("data conversion", { - skip_if(!interactive()) h2o_start() expect_silent(cars_1 <- as_h2o(mtcars)) @@ -7,14 +6,3 @@ test_that("data conversion", { expect_equal(class(cars_1$id), "character") expect_equal(as_tibble(cars_1$data), tibble::as_tibble(mtcars)) }) - - -test_that("server functions", { - # skip("h2o server issues with testthat") - skip_if(!interactive()) - - expect_silent(agua::h2o_start()) - expect_true(h2o_running()) - h2o::h2o.shutdown(prompt = FALSE) - expect_false(h2o_running()) -}) diff --git a/tests/testthat/test-parsnip-execution.R b/tests/testthat/test-parsnip-execution.R index 77bfdf6..163ce8d 100644 --- a/tests/testthat/test-parsnip-execution.R +++ b/tests/testthat/test-parsnip-execution.R @@ -1,7 +1,4 @@ test_that("random forest execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(rand_forest(mtry = 2, trees = 5) %>% set_mode("regression")) @@ -11,23 +8,14 @@ test_that("random forest execution", { test_that("linear regression execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(linear_reg(penalty = 0.1)) }) test_that("logistic regression execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(logistic_reg(mixture = 1)) }) test_that("poisson regression execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(poisson_reg(engine = "h2o"), data = as.data.frame(Titanic), formula = Freq ~ . @@ -35,9 +23,6 @@ test_that("poisson regression execution", { }) test_that("multinomial regression execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(multinom_reg(), data = iris, formula = Species ~ . @@ -45,16 +30,10 @@ test_that("multinomial regression execution", { }) test_that("naive bayes execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(naive_Bayes(engine = "h2o", Laplace = 1)) }) test_that("mlp execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(mlp(hidden_units = 100) %>% set_mode("regression")) expect_h2o_fit(mlp(hidden_units = 100) %>% @@ -62,9 +41,6 @@ test_that("mlp execution", { }) test_that("rule fit execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(rule_fit(engine = "h2o", trees = 10, tree_depth = 3) %>% set_mode("regression")) expect_h2o_fit(rule_fit(engine = "h2o", trees = 10, tree_depth = 3) %>% @@ -72,9 +48,6 @@ test_that("rule fit execution", { }) test_that("xgboost execution", { - skip_if(!interactive()) - h2o_start() - skip_if(!h2o_xgboost_available()) expect_h2o_fit(boost_tree(learn_rate = .1, trees = 5) %>% set_mode("regression")) @@ -83,9 +56,6 @@ test_that("xgboost execution", { }) test_that("gbm execution", { - skip_if(!interactive()) - h2o_start() - expect_h2o_fit(boost_tree(learn_rate = .1, trees = 5) %>% set_mode("regression"), engine = "h2o_gbm") expect_h2o_fit(boost_tree(learn_rate = .1, trees = 5) %>% @@ -93,8 +63,6 @@ test_that("gbm execution", { }) test_that("automl execution", { - skip_if(!interactive()) - h2o_start() data(two_class_dat, package = "modeldata") set.seed(1) @@ -116,8 +84,6 @@ test_that("automl execution", { }) test_that("automl tools", { - skip_if(!interactive()) - h2o_start() set.seed(1) spec <- auto_ml() %>% diff --git a/tests/testthat/test-tune_grid.R b/tests/testthat/test-tune_grid.R index 46f1697..ef2b553 100644 --- a/tests/testthat/test-tune_grid.R +++ b/tests/testthat/test-tune_grid.R @@ -1,9 +1,6 @@ data(two_class_dat, package = "modeldata") test_that("tune model only (with id)", { - skip_if(!interactive()) - h2o_start() - helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_label) %>% @@ -19,9 +16,6 @@ test_that("tune model only (with id)", { }) test_that("tune model only (without id)", { - skip_if(!interactive()) - h2o_start() - helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -37,9 +31,6 @@ test_that("tune model only (without id)", { }) test_that("tune model only (with id and recipe)", { - skip_if(!interactive()) - h2o_start() - helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -57,9 +48,6 @@ test_that("tune model only (with id and recipe)", { test_that("tune model and recipe", { - skip_if(!interactive()) - h2o_start() - helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -78,9 +66,6 @@ test_that("tune model and recipe", { }) test_that("tune with backend options parallelism", { - skip_if(!interactive()) - h2o_start() - helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% From 89fd1c749ca8cac63f3a51e7a78e999c6359f497 Mon Sep 17 00:00:00 2001 From: qiushi Date: Fri, 9 Sep 2022 14:06:31 -0500 Subject: [PATCH 2/3] start h2o once --- tests/testthat.R | 3 ++- tests/testthat/test-misc.R | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/testthat.R b/tests/testthat.R index 23f26c9..6d131e2 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,5 +1,6 @@ library(testthat) library(agua) +options("prefer_RCurl" = FALSE) +h2o_start() test_check("agua") -option("prefer_RCurl") == FALSE diff --git a/tests/testthat/test-misc.R b/tests/testthat/test-misc.R index 9d1c420..7f1ef30 100644 --- a/tests/testthat/test-misc.R +++ b/tests/testthat/test-misc.R @@ -1,6 +1,4 @@ test_that("data conversion", { - h2o_start() - expect_silent(cars_1 <- as_h2o(mtcars)) expect_equal(class(cars_1$data), "H2OFrame") expect_equal(class(cars_1$id), "character") From 917ab309145d543aa060437caaa3d58dff29d121 Mon Sep 17 00:00:00 2001 From: qiushi Date: Fri, 9 Sep 2022 15:30:09 -0500 Subject: [PATCH 3/3] start server in each tune chunk --- DESCRIPTION | 2 +- NAMESPACE | 1 + R/server.R | 16 ++++++++++++++ man/h2o-server.Rd | 4 ++++ tests/testthat.R | 2 -- tests/testthat/_snaps/tune_grid.md | 22 ------------------- tests/testthat/test-case-weights.R | 5 +++-- tests/testthat/test-misc.R | 3 +++ tests/testthat/test-parsnip-execution.R | 3 +++ tests/testthat/test-tune_grid.R | 29 +++++++++---------------- 10 files changed, 41 insertions(+), 46 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3106a05..f6d330d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -55,5 +55,5 @@ Config/testthat/edition: 3 Config/testthat/parallel: false Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.0.9000 +RoxygenNote: 7.2.1 VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index 6ab55d4..8ab4d44 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -28,6 +28,7 @@ export(extract_fit_engine) export(extract_fit_parsnip) export(get_leaderboard) export(h2o_activation) +export(h2o_end) export(h2o_get_frame) export(h2o_get_model) export(h2o_predict) diff --git a/R/server.R b/R/server.R index f980052..3cfdeda 100644 --- a/R/server.R +++ b/R/server.R @@ -7,6 +7,7 @@ #' h2o_start() #' } #' @rdname h2o-server +#' @keywords internal #' @export h2o_start <- function() { res <- utils::capture.output(h2o:::with_no_h2o_progress( @@ -16,6 +17,16 @@ h2o_start <- function() { } #' @rdname h2o-server +#' @keywords internal +#' @export +h2o_end <- function() { + if (h2o_running()) { + h2o::h2o.shutdown(prompt = FALSE) + } +} + +#' @rdname h2o-server +#' @keywords internal #' @export h2o_running <- function(verbose = FALSE) { res <- try(h2o::h2o.clusterIsUp(), silent = TRUE) @@ -30,18 +41,21 @@ h2o_running <- function(verbose = FALSE) { } #' @rdname h2o-server +#' @keywords internal #' @export h2o_remove <- function(id) { h2o::h2o.rm(id) } #' @rdname h2o-server +#' @keywords internal #' @export h2o_remove_all <- function() { h2o::h2o.removeAll() } #' @rdname h2o-server +#' @keywords internal #' @export h2o_get_model <- function(id) { res <- eval_silently(h2o:::with_no_h2o_progress(h2o::h2o.getModel(id))) @@ -52,6 +66,7 @@ h2o_get_model <- function(id) { } #' @rdname h2o-server +#' @keywords internal #' @export h2o_get_frame <- function(id) { res <- eval_silently(h2o:::with_no_h2o_progress(h2o::h2o.getFrame(id))) @@ -61,6 +76,7 @@ h2o_get_frame <- function(id) { } #' @rdname h2o-server +#' @keywords internal #' @export h2o_xgboost_available <- function() { "XGBoost" %in% h2o::h2o.list_core_extensions() diff --git a/man/h2o-server.Rd b/man/h2o-server.Rd index 42878d5..c07e100 100644 --- a/man/h2o-server.Rd +++ b/man/h2o-server.Rd @@ -2,6 +2,7 @@ % Please edit documentation in R/server.R \name{h2o_start} \alias{h2o_start} +\alias{h2o_end} \alias{h2o_running} \alias{h2o_remove} \alias{h2o_remove_all} @@ -12,6 +13,8 @@ \usage{ h2o_start() +h2o_end() + h2o_running(verbose = FALSE) h2o_remove(id) @@ -37,3 +40,4 @@ if (!h2o_running()) { h2o_start() } } +\keyword{internal} diff --git a/tests/testthat.R b/tests/testthat.R index 6d131e2..7a1095b 100644 --- a/tests/testthat.R +++ b/tests/testthat.R @@ -1,6 +1,4 @@ library(testthat) library(agua) -options("prefer_RCurl" = FALSE) -h2o_start() test_check("agua") diff --git a/tests/testthat/_snaps/tune_grid.md b/tests/testthat/_snaps/tune_grid.md index a962536..608d75a 100644 --- a/tests/testthat/_snaps/tune_grid.md +++ b/tests/testthat/_snaps/tune_grid.md @@ -86,25 +86,3 @@ Run `show_notes(.Last.tune.result)` for more information. -# tune with backend options parallelism - - Code - res - Output - # Tuning results - # 5-fold cross-validation - # A tibble: 5 x 5 - splits id .metrics .notes .predictions - - 1 Fold1 - 2 Fold2 - 3 Fold3 - 4 Fold4 - 5 Fold5 - - There were issues with some computations: - - - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - Error(s) x1: Error in .h2o.doSafeREST(h2oRestApiVersion = h2oRestApiVersion, u... - - Run `show_notes(.Last.tune.result)` for more information. - diff --git a/tests/testthat/test-case-weights.R b/tests/testthat/test-case-weights.R index 41e1edc..2b18e46 100644 --- a/tests/testthat/test-case-weights.R +++ b/tests/testthat/test-case-weights.R @@ -1,7 +1,8 @@ +options("prefer_RCurl" = FALSE) +h2o_start() + test_that("case weights works", { skip_if_not_installed("modeldata") - - h2o_start() data("two_class_dat", package = "modeldata") wts <- runif(nrow(two_class_dat)) diff --git a/tests/testthat/test-misc.R b/tests/testthat/test-misc.R index 7f1ef30..cdf85a7 100644 --- a/tests/testthat/test-misc.R +++ b/tests/testthat/test-misc.R @@ -1,3 +1,6 @@ +options("prefer_RCurl" = FALSE) +h2o_start() + test_that("data conversion", { expect_silent(cars_1 <- as_h2o(mtcars)) expect_equal(class(cars_1$data), "H2OFrame") diff --git a/tests/testthat/test-parsnip-execution.R b/tests/testthat/test-parsnip-execution.R index 163ce8d..00cfabe 100644 --- a/tests/testthat/test-parsnip-execution.R +++ b/tests/testthat/test-parsnip-execution.R @@ -1,3 +1,6 @@ +options("prefer_RCurl" = FALSE) +h2o_start() + test_that("random forest execution", { expect_h2o_fit(rand_forest(mtry = 2, trees = 5) %>% set_mode("regression")) diff --git a/tests/testthat/test-tune_grid.R b/tests/testthat/test-tune_grid.R index ef2b553..080840c 100644 --- a/tests/testthat/test-tune_grid.R +++ b/tests/testthat/test-tune_grid.R @@ -1,6 +1,10 @@ data(two_class_dat, package = "modeldata") +options("prefer_RCurl" = FALSE) +h2o_start() test_that("tune model only (with id)", { + h2o_start() + helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_label) %>% @@ -16,6 +20,8 @@ test_that("tune model only (with id)", { }) test_that("tune model only (without id)", { + h2o_start() + helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -31,6 +37,8 @@ test_that("tune model only (without id)", { }) test_that("tune model only (with id and recipe)", { + h2o_start() + helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -48,6 +56,8 @@ test_that("tune model only (with id and recipe)", { test_that("tune model and recipe", { + h2o_start() + helper_objects <- helper_objects_tune() wflow <- workflows::workflow() %>% workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% @@ -64,22 +74,3 @@ test_that("tune model and recipe", { ) expect_snapshot(res) }) - -test_that("tune with backend options parallelism", { - helper_objects <- helper_objects_tune() - wflow <- workflows::workflow() %>% - workflows::add_model(helper_objects$glm_spec_tune_no_label) %>% - workflows::add_recipe(helper_objects$rec_tune) - param_grid <- expand.grid( - penalty = 10^seq(-10, 1, length = 5), - deg_free = c(3, 4, 5) - ) - control <- tune::control_grid(save_pred = TRUE) - res <- tune::tune_grid(wflow, - resamples = helper_objects$folds, - control = control, - grid = param_grid, - backend_options = list(parallelism = 20) - ) - expect_snapshot(res) -})