diff --git a/NEWS.md b/NEWS.md index 69d8476..704cb8b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,13 @@ ### learner version 1.1.0 (unreleased) +* Fix blank CV heatmaps in RStudio when followed by `plot_objective()` by + avoiding explicit graphics-parameter changes in the objective plot. The plot + now uses the current margins, so very small margins can cut off the title and + axis labels. + +* Return per-candidate, per-fold stopping codes and update counts in + `cv.learner()$cv_diagnostics`, so CV checks do not require refitting. + * Add an optional relative stopping rule to `learner()` and `cv.learner()`. Use `control = list(stopping_rule = "relative")` to stop once the relative change in the objective stays below `threshold` (default `1e-6`) for `window` updates in a diff --git a/R/learner.R b/R/learner.R index 340583a..2a8614f 100644 --- a/R/learner.R +++ b/R/learner.R @@ -43,8 +43,9 @@ #' #' To obtain the final estimate, apply \code{\link{learner}} with the selected #' penalties to the full data (see Examples). Fits that reach the iteration -#' limit still contribute to the held out error, so check the settings in -#' \code{control} before interpreting the selection. +#' limit still contribute to the held out error. Inspect \code{cv_diagnostics} +#' before interpreting the selection; a successful final fit does not imply +#' that all CV fits met the stopping rule. #' #' @return #' A list containing: @@ -62,6 +63,17 @@ #' \item{lambda_1_all, lambda_2_all, lambda_1_row_all, lambda_1_col_all}{The #' candidate grids in their original order. Only grids used in the search are #' returned. \code{\link{plot_cv}} reads these automatically.} +#' \item{cv_diagnostics}{Data frame with one row per candidate and fold, containing +#' \code{candidate}, \code{fold}, \code{lambda_1_row}, \code{lambda_1_col}, +#' \code{lambda_2}, \code{updates}, \code{stop_code}, and \code{selected}. +#' Candidate indices follow the order of \code{as.vector(mse_all)}; rows are +#' ordered by candidate, then fold. Common penalties appear in both space columns. +#' Stopping codes have the same meaning as in \code{\link{learner}}: +#' 1, the stopping rule was met; 2, the iteration limit was reached; +#' 3, excessive growth or non-finite values were detected. Updates count recorded +#' objective values. If a candidate fails with non-finite values, its remaining +#' folds are skipped and have \code{NA} for updates and stopping codes. +#' These diagnostics are saved during CV; no additional fits are performed.} #' \item{n_observed}{Number of observed target entries, used to convert the #' summed validation errors to mean squared errors when plotting.} #' @@ -153,26 +165,34 @@ cv.learner <- function(Y_source, Y_target, r, lambda_1_all = NULL, lambda_2_all, index_set[[fold]] <- indices[seq.int(start, end)] } - scores <- cv_learner_cpp(sources$matrices, Y_target, sources$weights, + cv_result <- cv_learner_cpp(sources$matrices, Y_target, sources$weights, grid$row, grid$col, grid$balance, step_size, control$max_iter, control$threshold, n_cores, r, control$max_value, index_set, control$stopping_rule == "relative", control$window) + scores <- cv_result$scores if (!any(is.finite(scores))) stop('No candidate produced finite validation errors.') if (any(!is.finite(scores))) warning('Some candidates produced non-finite validation errors and were excluded.') scores[!is.finite(scores)] <- Inf best <- which.min(scores) # Keep the historical mse_all scale: summed held-out squared errors. mse_all <- array(scores, dim = grid_dim) + candidate <- rep(seq_len(nrow(grid)), each = n_folds) + cv_diagnostics <- data.frame( + candidate = candidate, fold = rep(seq_len(n_folds), times = nrow(grid)), + lambda_1_row = grid$row[candidate], lambda_1_col = grid$col[candidate], + lambda_2 = grid$balance[candidate], updates = cv_result$updates, + stop_code = cv_result$stop_code, selected = candidate == best) if (!separate) { return(list(lambda_1_min = grid$row[best], lambda_2_min = grid$balance[best], mse_all = mse_all, r = r, lambda_1_all = lambda_1_all, - lambda_2_all = lambda_2_all, n_observed = n_indices)) + lambda_2_all = lambda_2_all, n_observed = n_indices, + cv_diagnostics = cv_diagnostics)) } list(lambda_1_min = NA_real_, lambda_2_min = grid$balance[best], lambda_1_row_min = grid$row[best], lambda_1_col_min = grid$col[best], mse_all = mse_all, r = r, lambda_1_row_all = lambda_1_row_all, lambda_1_col_all = lambda_1_col_all, lambda_2_all = lambda_2_all, - n_observed = n_indices) + n_observed = n_indices, cv_diagnostics = cv_diagnostics) } #' Latent space-based transfer learning diff --git a/R/plotting.R b/R/plotting.R index ff964cd..94c5d45 100644 --- a/R/plotting.R +++ b/R/plotting.R @@ -22,6 +22,9 @@ #' instead of settling; a flat-looking curve alone does not establish #' convergence. #' +#' The plot is drawn with the current graphics margins, so very small margins +#' can cut off the title and axis labels. +#' #' @examples #' fit <- learner(dat_highsim$Y_source, dat_highsim$Y_target, #' r = 3, lambda_1 = 1, lambda_2 = 1, step_size = 0.003, @@ -65,8 +68,7 @@ plot_objective <- function(x, iterations = "all", main = NULL) { } values <- replace(objective[shown], !finite[shown], NA_real_) - old <- graphics::par(mar = c(5, 4.5, 4, 1.5)) - on.exit(graphics::par(old), add = TRUE) + # Use the current margins without recording explicit par() changes. xlim <- range(shown) graphics::plot(shown, values, type = 'n', xlim = xlim, xlab = 'Iteration', ylab = 'Objective value', bty = 'l') diff --git a/inst/examples/advanced-learner.R b/inst/examples/advanced-learner.R index dcd1330..1c17bbe 100644 --- a/inst/examples/advanced-learner.R +++ b/inst/examples/advanced-learner.R @@ -37,7 +37,8 @@ compatible <- local({ target <- diag(c(4, 0, 0)) cv <- cv.learner(y, target, r = 1, lambda_1_all = 1, lambda_2_all = 1, step_size = 0.001, n_folds = 2, control = list(max_iter = 2)) - all(c("lambda_1_all", "lambda_2_all", "n_observed") %in% names(cv)) + all(c("lambda_1_all", "lambda_2_all", "n_observed", + "cv_diagnostics") %in% names(cv)) }) if (!compatible) { stop(paste("This learner installation lacks the CV metadata used here.", @@ -105,10 +106,10 @@ fit_multi$convergence_criterion # grid-boundaries needed <- c("asym", "asym_space_grid", "asym_balance_grid", - "cv_separate", "fit_separate") + "cv_separate", "fit_separate", "cv_multi") if (!all(vapply(needed, exists, logical(1), inherits = TRUE))) { - stop("Run Section 2 through the final refit before starting Section 4.", - call. = FALSE) + stop("Run Section 2 through the final refit, and the cross-validation in ", + "Section 3, before starting Section 4.", call. = FALSE) } selected <- c(U = cv_separate$lambda_1_row_min, V = cv_separate$lambda_1_col_min, @@ -155,22 +156,11 @@ sensitivity <- rbind( print(sensitivity, row.names = FALSE) # inner-cv-stopping -set.seed(815) -indices <- sample(which(!is.na(asym$Y_target))) -n <- length(indices) -folds <- lapply(1:3, function(k) { - indices[seq.int(floor((k - 1) * n / 3) + 1, floor(k * n / 3))] -}) -inner_fits <- do.call(rbind, lapply(1:3, function(k) { - training <- asym$Y_target - training[folds[[k]]] <- NA - fit <- learner( - asym$Y_source, training, r = cv_separate$r, - lambda_1_row = cv_separate$lambda_1_row_min, - lambda_1_col = cv_separate$lambda_1_col_min, - lambda_2 = cv_separate$lambda_2_min, - step_size = 0.003, control = list(max_iter = 2000)) - data.frame(fold = k, updates = length(fit$objective_values), - stop_code = fit$convergence_criterion) -})) +inner_fits <- subset(cv_separate$cv_diagnostics, selected, + select = c(fold, updates, stop_code)) print(inner_fits, row.names = FALSE) + +# multiple-cv-stopping +with(cv_multi$cv_diagnostics, table(stop_code, useNA = "ifany")) +subset(cv_multi$cv_diagnostics, selected, + select = c(fold, updates, stop_code)) diff --git a/man/cv.learner.Rd b/man/cv.learner.Rd index dcbc4d8..2051773 100644 --- a/man/cv.learner.Rd +++ b/man/cv.learner.Rd @@ -66,6 +66,17 @@ of observed target entries to obtain mean squared errors.} \item{lambda_1_all, lambda_2_all, lambda_1_row_all, lambda_1_col_all}{The candidate grids in their original order. Only grids used in the search are returned. \code{\link{plot_cv}} reads these automatically.} +\item{cv_diagnostics}{Data frame with one row per candidate and fold, containing +\code{candidate}, \code{fold}, \code{lambda_1_row}, \code{lambda_1_col}, +\code{lambda_2}, \code{updates}, \code{stop_code}, and \code{selected}. +Candidate indices follow the order of \code{as.vector(mse_all)}; rows are +ordered by candidate, then fold. Common penalties appear in both space columns. +Stopping codes have the same meaning as in \code{\link{learner}}: +1, the stopping rule was met; 2, the iteration limit was reached; +3, excessive growth or non-finite values were detected. Updates count recorded +objective values. If a candidate fails with non-finite values, its remaining +folds are skipped and have \code{NA} for updates and stopping codes. +These diagnostics are saved during CV; no additional fits are performed.} \item{n_observed}{Number of observed target entries, used to convert the summed validation errors to mean squared errors when plotting.} } @@ -92,8 +103,9 @@ penalties together. Both modes can be used with multiple sources. Call To obtain the final estimate, apply \code{\link{learner}} with the selected penalties to the full data (see Examples). Fits that reach the iteration -limit still contribute to the held out error, so check the settings in -\code{control} before interpreting the selection. +limit still contribute to the held out error. Inspect \code{cv_diagnostics} +before interpreting the selection; a successful final fit does not imply +that all CV fits met the stopping rule. } \examples{ res <- cv.learner(Y_source = dat_highsim$Y_source, diff --git a/man/plot_objective.Rd b/man/plot_objective.Rd index 1b07be6..a148199 100644 --- a/man/plot_objective.Rd +++ b/man/plot_objective.Rd @@ -33,6 +33,9 @@ Initialization is not included in this history. A curve that zigzags between two levels means that the objective alternates from one iteration to the next instead of settling; a flat-looking curve alone does not establish convergence. + +The plot is drawn with the current graphics margins, so very small margins +can cut off the title and axis labels. } \examples{ fit <- learner(dat_highsim$Y_source, dat_highsim$Y_target, diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index d7465c6..6cb1d4e 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -35,7 +35,7 @@ BEGIN_RCPP END_RCPP } // cv_learner_cpp -std::vector cv_learner_cpp(const Rcpp::List& Y_source, const Eigen::MatrixXd& Y_target, const std::vector& source_weights, const std::vector& row_grid, const std::vector& col_grid, const std::vector& balance_grid, double step_size, int max_iter, double threshold, int n_cores, int r, double max_value, const std::vector>& index_set, bool relative, int window); +List cv_learner_cpp(const Rcpp::List& Y_source, const Eigen::MatrixXd& Y_target, const std::vector& source_weights, const std::vector& row_grid, const std::vector& col_grid, const std::vector& balance_grid, double step_size, int max_iter, double threshold, int n_cores, int r, double max_value, const std::vector>& index_set, bool relative, int window); RcppExport SEXP _learner_cv_learner_cpp(SEXP Y_sourceSEXP, SEXP Y_targetSEXP, SEXP source_weightsSEXP, SEXP row_gridSEXP, SEXP col_gridSEXP, SEXP balance_gridSEXP, SEXP step_sizeSEXP, SEXP max_iterSEXP, SEXP thresholdSEXP, SEXP n_coresSEXP, SEXP rSEXP, SEXP max_valueSEXP, SEXP index_setSEXP, SEXP relativeSEXP, SEXP windowSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; diff --git a/src/kernels.cpp b/src/kernels.cpp index 9779dca..df58a57 100644 --- a/src/kernels.cpp +++ b/src/kernels.cpp @@ -189,28 +189,35 @@ List learner_cpp(const Rcpp::List &Y_source, const Eigen::MatrixXd &Y_target, // All three vectors contain one entry per candidate combination, in R array order. // [[Rcpp::export]] -std::vector cv_learner_cpp(const Rcpp::List &Y_source, - const Eigen::MatrixXd &Y_target, - const std::vector &source_weights, - const std::vector &row_grid, - const std::vector &col_grid, - const std::vector &balance_grid, - double step_size, int max_iter, double threshold, - int n_cores, int r, double max_value, - const std::vector> &index_set, bool relative, int window) { +List cv_learner_cpp(const Rcpp::List &Y_source, + const Eigen::MatrixXd &Y_target, + const std::vector &source_weights, + const std::vector &row_grid, + const std::vector &col_grid, + const std::vector &balance_grid, + double step_size, int max_iter, double threshold, + int n_cores, int r, double max_value, + const std::vector> &index_set, bool relative, int window) { SourceSpaces spaces = prepare_spaces(Y_source, source_weights, r); int p = Y_target.rows(); int n_grid = row_grid.size(); std::vector scores(n_grid, 0.0); + const int n_folds = index_set.size(); + std::vector stopping_codes(n_grid * n_folds, NA_INTEGER); + std::vector updates(n_grid * n_folds, NA_INTEGER); #pragma omp parallel for schedule(dynamic) num_threads(n_cores) for (int g = 0; g < n_grid; ++g) { double score = 0.0; - for (const auto &fold : index_set) { + for (int f = 0; f < n_folds; ++f) { + const auto &fold = index_set[f]; Eigen::MatrixXd training = Y_target; for (int idx : fold) training(idx % p, idx / p) = std::numeric_limits::quiet_NaN(); FitResult fit = learner_worker(spaces, training, row_grid[g], col_grid[g], balance_grid[g], step_size, max_iter, threshold, max_value, relative, window); + const int diagnostic_index = g * n_folds + f; + stopping_codes[diagnostic_index] = fit.criterion; + updates[diagnostic_index] = fit.objective.size(); if (!fit.estimate.allFinite() || !std::all_of(fit.objective.begin(), fit.objective.end(), [](double x) { return std::isfinite(x); })) { @@ -226,7 +233,9 @@ std::vector cv_learner_cpp(const Rcpp::List &Y_source, } scores[g] = score; } - return scores; + return List::create(Named("scores") = scores, + Named("stop_code") = stopping_codes, + Named("updates") = updates); } // -------------------------------------------------------------- diff --git a/tests/testthat/test-cv-diagnostics.R b/tests/testthat/test-cv-diagnostics.R new file mode 100644 index 0000000..41fb24e --- /dev/null +++ b/tests/testthat/test-cv-diagnostics.R @@ -0,0 +1,113 @@ +test_that('saved diagnostics and scores agree with independent fold refits', { + source <- dat_highsim$Y_source[1:8, 1:6] + target <- dat_highsim$Y_target[1:8, 1:6] + target[c(2, 9, 18)] <- NA_real_ + n_folds <- 2L + for (separate in c(FALSE, TRUE)) { + for (relative in c(FALSE, TRUE)) { + control <- list(max_iter = 12, threshold = if (relative) .1 else 0, + stopping_rule = if (relative) 'relative' else 'absolute', + window = if (relative) 3 else 1) + args <- if (separate) { + list(lambda_1_row_all = c(2, 0), lambda_1_col_all = c(1, 0)) + } else list(lambda_1_all = c(2, 0)) + set.seed(91) + cv <- do.call(cv.learner, c(list(Y_source = list(source, source * .9), + Y_target = target, source_weights = c(.7, .3), r = 2, + lambda_2_all = c(.1, .01), n_folds = n_folds, + step_size = .001, control = control), args)) + d <- cv$cv_diagnostics + expect_equal(nrow(d), length(cv$mse_all) * n_folds) + expect_identical(d$candidate, rep(seq_along(cv$mse_all), each = n_folds)) + expect_identical(d$fold, rep(seq_len(n_folds), length(cv$mse_all))) + expect_identical(d$selected, d$candidate == which.min(cv$mse_all)) + set.seed(91) + idx <- sample(which(!is.na(target))) + n <- length(idx) + folds <- lapply(seq_len(n_folds), function(k) { + idx[seq.int(floor((k - 1) * n / n_folds) + 1, floor(k * n / n_folds))] + }) + expected_scores <- numeric(length(cv$mse_all)) + for (i in seq_len(nrow(d))) { + held_out <- folds[[d$fold[i]]] + training <- target + training[held_out] <- NA_real_ + fit <- learner(list(source, source * .9), training, + source_weights = c(.7, .3), r = 2, + lambda_1_row = d$lambda_1_row[i], lambda_1_col = d$lambda_1_col[i], + lambda_2 = d$lambda_2[i], step_size = .001, control = control) + expect_equal(d$stop_code[i], fit$convergence_criterion) + expect_equal(d$updates[i], length(fit$objective_values)) + g <- d$candidate[i] + expected_scores[g] <- expected_scores[g] + + sum((target[held_out] - fit$learner_estimate[held_out])^2) + } + expect_equal(as.vector(cv$mse_all), expected_scores, tolerance = 1e-12) + if (!separate) expect_equal(d$lambda_1_row, d$lambda_1_col) + } + } +}) + +test_that('non-finite candidates record failure and mark skipped folds as missing', { + source <- diag(c(4, 3, 2, 1)) + target <- diag(c(1, 2, 3, 4)) + set.seed(42) + expect_warning(cv <- cv.learner(source, target, r = 2, + lambda_1_all = c(1e308, 0), lambda_2_all = 1, step_size = .001, + n_folds = 3, control = list(max_iter = 3)), 'non-finite validation errors') + d <- cv$cv_diagnostics + expect_equal(d$stop_code[1:3], c(3L, NA_integer_, NA_integer_)) + expect_equal(d$updates[1:3], c(1L, NA_integer_, NA_integer_)) + expect_false(any(d$selected[1:3])) + expect_false(anyNA(d$stop_code[4:6])) + expect_true(all(d$selected[4:6])) +}) + +test_that('CV records tolerance stopping including the singleton grid', { + source <- matrix(0, 4, 3) + set.seed(12) + cv <- cv.learner(source, source, r = 1, lambda_1_all = 0, + lambda_2_all = 0, step_size = .001, n_folds = 2, + control = list(max_iter = 10)) + expect_equal(cv$cv_diagnostics$stop_code, c(1L, 1L)) + expect_equal(cv$cv_diagnostics$updates, c(2L, 2L)) + expect_true(all(cv$cv_diagnostics$selected)) +}) + +test_that('diagnostics match refits when stopping differs across candidates and folds', { + sources <- dat_multisource$Y_source + target <- dat_multisource$Y_target + weights <- c(0.7, 0.3) + settings <- list( + list(n_folds = 3L, args = list(lambda_1_all = c(0, 1, 10))), + list(n_folds = 2L, args = list(lambda_1_row_all = c(0, 1, 10), + lambda_1_col_all = c(0, 1, 10)))) + for (setting in settings) { + n_folds <- setting$n_folds + set.seed(815) + cv <- do.call(cv.learner, c(list(Y_source = sources, Y_target = target, + r = 3, source_weights = weights, lambda_2_all = c(0.1, 1), + n_folds = n_folds, step_size = 0.0003, control = list(max_iter = 2000)), + setting$args)) + d <- cv$cv_diagnostics + # The rows must differ, or a row stored in the wrong place would go unnoticed. + expect_gt(nrow(unique(d[c('stop_code', 'updates')])), 3L) + expect_gt(length(unique(d$stop_code)), 1L) + set.seed(815) + idx <- sample(which(!is.na(target))) + n <- length(idx) + folds <- lapply(seq_len(n_folds), function(k) { + idx[seq.int(floor((k - 1) * n / n_folds) + 1, floor(k * n / n_folds))] + }) + for (i in seq_len(nrow(d))) { + training <- target + training[folds[[d$fold[i]]]] <- NA_real_ + fit <- learner(sources, training, r = 3, source_weights = weights, + lambda_1_row = d$lambda_1_row[i], lambda_1_col = d$lambda_1_col[i], + lambda_2 = d$lambda_2[i], step_size = 0.0003, + control = list(max_iter = 2000)) + expect_equal(d$stop_code[i], fit$convergence_criterion) + expect_equal(d$updates[i], length(fit$objective_values)) + } + } +}) diff --git a/tests/testthat/test-cv-multisource.R b/tests/testthat/test-cv-multisource.R index 5b77b0e..72bc262 100644 --- a/tests/testthat/test-cv-multisource.R +++ b/tests/testthat/test-cv-multisource.R @@ -64,7 +64,7 @@ test_that("tied CV is the diagonal of the independent grid", { tied <- fit(lambda_1_all = c(1, 3)) separate <- fit(lambda_1_row_all = c(1, 3), lambda_1_col_all = c(1, 3)) expect_named(tied, c('lambda_1_min', 'lambda_2_min', 'mse_all', 'r', - 'lambda_1_all', 'lambda_2_all', 'n_observed')) + 'lambda_1_all', 'lambda_2_all', 'n_observed', 'cv_diagnostics')) expect_equal(dim(tied$mse_all), c(2L, 2L)) for (i in 1:2) expect_equal(tied$mse_all[i, ], separate$mse_all[i, i, ], tolerance = 1e-10) single <- fit(lambda_1_row_all = 1, lambda_1_col_all = 3) diff --git a/tests/testthat/test-plotting.R b/tests/testthat/test-plotting.R index 191f27b..9e0643a 100644 --- a/tests/testthat/test-plotting.R +++ b/tests/testthat/test-plotting.R @@ -74,6 +74,23 @@ test_that('near-optimum plots handle alternating and missing values and keep gra expect_equal(graphics::par('mar'), old) }) +test_that('objective plots make no par() calls and leave device settings alone', { + # Check explicit parameter changes separately from mixed grid/base replay. + path <- plot_device() + on.exit({grDevices::dev.off(); unlink(path)}, add = TRUE) + grDevices::dev.control('enable') + settings <- graphics::par(c('mar', 'new')) + plot_objective(list(objective_values = c(5, 4, 3))) + recorded <- vapply(grDevices::recordPlot()[[1]], function(step) { + symbol <- step[[2]][[1]] + if (is.list(symbol) && is.character(symbol$name)) symbol$name else NA_character_ + }, character(1)) + # The structure of a recorded plot is internal to R; skip if it looks different. + skip_if(length(recorded) == 0L || anyNA(recorded)) + expect_false('C_par' %in% recorded) + expect_equal(graphics::par(c('mar', 'new')), settings) +}) + test_that('objective plots handle flat, singleton, and non-finite histories', { path <- plot_device() on.exit({grDevices::dev.off(); unlink(path)}, add = TRUE) @@ -226,3 +243,26 @@ test_that('objective plots reject ranges with no finite values before drawing', expect_equal(plot_objective(fit, iterations = 2:5)$objective, fit$objective_values[2:5]) }) + + +test_that('switching from a CV heatmap to an objective plot preserves the heatmap record', { + skip_if_not_installed('ggplot2', minimum_version = '3.4.0') + path <- plot_device() + on.exit({grDevices::dev.off(); unlink(path)}, add = TRUE) + grDevices::dev.control('enable') + cv <- list(mse_all = matrix(c(4, 3, 2, 1), 2), + lambda_1_all = c(0, 1), lambda_2_all = c(.1, 1), + lambda_1_min = 1, lambda_2_min = 1, n_observed = 10) + plot_cv(cv) + plot_cv(cv) + before <- grDevices::recordPlot() + saved <- NULL + old_hooks <- getHook('before.plot.new') + on.exit(setHook('before.plot.new', old_hooks, 'replace'), add = TRUE) + setHook('before.plot.new', function(...) { + if (is.null(saved)) saved <<- grDevices::recordPlot() + }, 'append') + plot_objective(list(objective_values = c(5, 4, 3))) + expect_identical(saved, before) + expect_no_error(grDevices::replayPlot(saved)) +}) diff --git a/vignettes/advanced-learner.Rmd b/vignettes/advanced-learner.Rmd index 35b6b9f..100f00b 100644 --- a/vignettes/advanced-learner.Rmd +++ b/vignettes/advanced-learner.Rmd @@ -50,7 +50,8 @@ cat('