Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
### learner version 1.1.0 (unreleased)

* Fix blank CV heatmaps in RStudio when followed by `plot_objective()` by
avoiding explicit graphics-parameter changes in the objective plot. The plot
now uses the current margins, so very small margins can cut off the title and
axis labels.

* Return per-candidate, per-fold stopping codes and update counts in
`cv.learner()$cv_diagnostics`, so CV checks do not require refitting.

* Add an optional relative stopping rule to `learner()` and `cv.learner()`. Use
`control = list(stopping_rule = "relative")` to stop once the relative change in
the objective stays below `threshold` (default `1e-6`) for `window` updates in a
Expand Down
30 changes: 25 additions & 5 deletions R/learner.R
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@
#'
#' To obtain the final estimate, apply \code{\link{learner}} with the selected
#' penalties to the full data (see Examples). Fits that reach the iteration
#' limit still contribute to the held out error, so check the settings in
#' \code{control} before interpreting the selection.
#' limit still contribute to the held out error. Inspect \code{cv_diagnostics}
#' before interpreting the selection; a successful final fit does not imply
#' that all CV fits met the stopping rule.
#'
#' @return
#' A list containing:
Expand All @@ -62,6 +63,17 @@
#' \item{lambda_1_all, lambda_2_all, lambda_1_row_all, lambda_1_col_all}{The
#' candidate grids in their original order. Only grids used in the search are
#' returned. \code{\link{plot_cv}} reads these automatically.}
#' \item{cv_diagnostics}{Data frame with one row per candidate and fold, containing
#' \code{candidate}, \code{fold}, \code{lambda_1_row}, \code{lambda_1_col},
#' \code{lambda_2}, \code{updates}, \code{stop_code}, and \code{selected}.
#' Candidate indices follow the order of \code{as.vector(mse_all)}; rows are
#' ordered by candidate, then fold. Common penalties appear in both space columns.
#' Stopping codes have the same meaning as in \code{\link{learner}}:
#' 1, the stopping rule was met; 2, the iteration limit was reached;
#' 3, excessive growth or non-finite values were detected. Updates count recorded
#' objective values. If a candidate fails with non-finite values, its remaining
#' folds are skipped and have \code{NA} for updates and stopping codes.
#' These diagnostics are saved during CV; no additional fits are performed.}
#' \item{n_observed}{Number of observed target entries, used to convert the
#' summed validation errors to mean squared errors when plotting.}
#'
Expand Down Expand Up @@ -153,26 +165,34 @@ cv.learner <- function(Y_source, Y_target, r, lambda_1_all = NULL, lambda_2_all,
index_set[[fold]] <- indices[seq.int(start, end)]
}

scores <- cv_learner_cpp(sources$matrices, Y_target, sources$weights,
cv_result <- cv_learner_cpp(sources$matrices, Y_target, sources$weights,
grid$row, grid$col, grid$balance, step_size,
control$max_iter, control$threshold, n_cores, r,
control$max_value, index_set, control$stopping_rule == "relative", control$window)
scores <- cv_result$scores
if (!any(is.finite(scores))) stop('No candidate produced finite validation errors.')
if (any(!is.finite(scores))) warning('Some candidates produced non-finite validation errors and were excluded.')
scores[!is.finite(scores)] <- Inf
best <- which.min(scores)
# Keep the historical mse_all scale: summed held-out squared errors.
mse_all <- array(scores, dim = grid_dim)
candidate <- rep(seq_len(nrow(grid)), each = n_folds)
cv_diagnostics <- data.frame(
candidate = candidate, fold = rep(seq_len(n_folds), times = nrow(grid)),
lambda_1_row = grid$row[candidate], lambda_1_col = grid$col[candidate],
lambda_2 = grid$balance[candidate], updates = cv_result$updates,
stop_code = cv_result$stop_code, selected = candidate == best)
if (!separate) {
return(list(lambda_1_min = grid$row[best], lambda_2_min = grid$balance[best],
mse_all = mse_all, r = r, lambda_1_all = lambda_1_all,
lambda_2_all = lambda_2_all, n_observed = n_indices))
lambda_2_all = lambda_2_all, n_observed = n_indices,
cv_diagnostics = cv_diagnostics))
}
list(lambda_1_min = NA_real_, lambda_2_min = grid$balance[best],
lambda_1_row_min = grid$row[best], lambda_1_col_min = grid$col[best],
mse_all = mse_all, r = r, lambda_1_row_all = lambda_1_row_all,
lambda_1_col_all = lambda_1_col_all, lambda_2_all = lambda_2_all,
n_observed = n_indices)
n_observed = n_indices, cv_diagnostics = cv_diagnostics)
}

#' Latent space-based transfer learning
Expand Down
6 changes: 4 additions & 2 deletions R/plotting.R
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
#' instead of settling; a flat-looking curve alone does not establish
#' convergence.
#'
#' The plot is drawn with the current graphics margins, so very small margins
#' can cut off the title and axis labels.
#'
#' @examples
#' fit <- learner(dat_highsim$Y_source, dat_highsim$Y_target,
#' r = 3, lambda_1 = 1, lambda_2 = 1, step_size = 0.003,
Expand Down Expand Up @@ -65,8 +68,7 @@ plot_objective <- function(x, iterations = "all", main = NULL) {
}
values <- replace(objective[shown], !finite[shown], NA_real_)

old <- graphics::par(mar = c(5, 4.5, 4, 1.5))
on.exit(graphics::par(old), add = TRUE)
# Use the current margins without recording explicit par() changes.
xlim <- range(shown)
graphics::plot(shown, values, type = 'n', xlim = xlim, xlab = 'Iteration',
ylab = 'Objective value', bty = 'l')
Expand Down
34 changes: 12 additions & 22 deletions inst/examples/advanced-learner.R
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ compatible <- local({
target <- diag(c(4, 0, 0))
cv <- cv.learner(y, target, r = 1, lambda_1_all = 1, lambda_2_all = 1,
step_size = 0.001, n_folds = 2, control = list(max_iter = 2))
all(c("lambda_1_all", "lambda_2_all", "n_observed") %in% names(cv))
all(c("lambda_1_all", "lambda_2_all", "n_observed",
"cv_diagnostics") %in% names(cv))
})
if (!compatible) {
stop(paste("This learner installation lacks the CV metadata used here.",
Expand Down Expand Up @@ -105,10 +106,10 @@ fit_multi$convergence_criterion

# grid-boundaries
needed <- c("asym", "asym_space_grid", "asym_balance_grid",
"cv_separate", "fit_separate")
"cv_separate", "fit_separate", "cv_multi")
if (!all(vapply(needed, exists, logical(1), inherits = TRUE))) {
stop("Run Section 2 through the final refit before starting Section 4.",
call. = FALSE)
stop("Run Section 2 through the final refit, and the cross-validation in ",
"Section 3, before starting Section 4.", call. = FALSE)
}
selected <- c(U = cv_separate$lambda_1_row_min,
V = cv_separate$lambda_1_col_min,
Expand Down Expand Up @@ -155,22 +156,11 @@ sensitivity <- rbind(
print(sensitivity, row.names = FALSE)

# inner-cv-stopping
set.seed(815)
indices <- sample(which(!is.na(asym$Y_target)))
n <- length(indices)
folds <- lapply(1:3, function(k) {
indices[seq.int(floor((k - 1) * n / 3) + 1, floor(k * n / 3))]
})
inner_fits <- do.call(rbind, lapply(1:3, function(k) {
training <- asym$Y_target
training[folds[[k]]] <- NA
fit <- learner(
asym$Y_source, training, r = cv_separate$r,
lambda_1_row = cv_separate$lambda_1_row_min,
lambda_1_col = cv_separate$lambda_1_col_min,
lambda_2 = cv_separate$lambda_2_min,
step_size = 0.003, control = list(max_iter = 2000))
data.frame(fold = k, updates = length(fit$objective_values),
stop_code = fit$convergence_criterion)
}))
inner_fits <- subset(cv_separate$cv_diagnostics, selected,
select = c(fold, updates, stop_code))
print(inner_fits, row.names = FALSE)

# multiple-cv-stopping
with(cv_multi$cv_diagnostics, table(stop_code, useNA = "ifany"))
subset(cv_multi$cv_diagnostics, selected,
select = c(fold, updates, stop_code))
16 changes: 14 additions & 2 deletions man/cv.learner.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions man/plot_objective.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion src/RcppExports.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ BEGIN_RCPP
END_RCPP
}
// cv_learner_cpp
std::vector<double> cv_learner_cpp(const Rcpp::List& Y_source, const Eigen::MatrixXd& Y_target, const std::vector<double>& source_weights, const std::vector<double>& row_grid, const std::vector<double>& col_grid, const std::vector<double>& balance_grid, double step_size, int max_iter, double threshold, int n_cores, int r, double max_value, const std::vector<std::vector<int>>& index_set, bool relative, int window);
List cv_learner_cpp(const Rcpp::List& Y_source, const Eigen::MatrixXd& Y_target, const std::vector<double>& source_weights, const std::vector<double>& row_grid, const std::vector<double>& col_grid, const std::vector<double>& balance_grid, double step_size, int max_iter, double threshold, int n_cores, int r, double max_value, const std::vector<std::vector<int>>& index_set, bool relative, int window);
RcppExport SEXP _learner_cv_learner_cpp(SEXP Y_sourceSEXP, SEXP Y_targetSEXP, SEXP source_weightsSEXP, SEXP row_gridSEXP, SEXP col_gridSEXP, SEXP balance_gridSEXP, SEXP step_sizeSEXP, SEXP max_iterSEXP, SEXP thresholdSEXP, SEXP n_coresSEXP, SEXP rSEXP, SEXP max_valueSEXP, SEXP index_setSEXP, SEXP relativeSEXP, SEXP windowSEXP) {
BEGIN_RCPP
Rcpp::RObject rcpp_result_gen;
Expand Down
31 changes: 20 additions & 11 deletions src/kernels.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -189,28 +189,35 @@ List learner_cpp(const Rcpp::List &Y_source, const Eigen::MatrixXd &Y_target,

// All three vectors contain one entry per candidate combination, in R array order.
// [[Rcpp::export]]
std::vector<double> cv_learner_cpp(const Rcpp::List &Y_source,
const Eigen::MatrixXd &Y_target,
const std::vector<double> &source_weights,
const std::vector<double> &row_grid,
const std::vector<double> &col_grid,
const std::vector<double> &balance_grid,
double step_size, int max_iter, double threshold,
int n_cores, int r, double max_value,
const std::vector<std::vector<int>> &index_set, bool relative, int window) {
List cv_learner_cpp(const Rcpp::List &Y_source,
const Eigen::MatrixXd &Y_target,
const std::vector<double> &source_weights,
const std::vector<double> &row_grid,
const std::vector<double> &col_grid,
const std::vector<double> &balance_grid,
double step_size, int max_iter, double threshold,
int n_cores, int r, double max_value,
const std::vector<std::vector<int>> &index_set, bool relative, int window) {
SourceSpaces spaces = prepare_spaces(Y_source, source_weights, r);
int p = Y_target.rows();
int n_grid = row_grid.size();
std::vector<double> scores(n_grid, 0.0);
const int n_folds = index_set.size();
std::vector<int> stopping_codes(n_grid * n_folds, NA_INTEGER);
std::vector<int> updates(n_grid * n_folds, NA_INTEGER);

#pragma omp parallel for schedule(dynamic) num_threads(n_cores)
for (int g = 0; g < n_grid; ++g) {
double score = 0.0;
for (const auto &fold : index_set) {
for (int f = 0; f < n_folds; ++f) {
const auto &fold = index_set[f];
Eigen::MatrixXd training = Y_target;
for (int idx : fold) training(idx % p, idx / p) = std::numeric_limits<double>::quiet_NaN();
FitResult fit = learner_worker(spaces, training, row_grid[g], col_grid[g],
balance_grid[g], step_size, max_iter, threshold, max_value, relative, window);
const int diagnostic_index = g * n_folds + f;
stopping_codes[diagnostic_index] = fit.criterion;
updates[diagnostic_index] = fit.objective.size();
if (!fit.estimate.allFinite() ||
!std::all_of(fit.objective.begin(), fit.objective.end(),
[](double x) { return std::isfinite(x); })) {
Expand All @@ -226,7 +233,9 @@ std::vector<double> cv_learner_cpp(const Rcpp::List &Y_source,
}
scores[g] = score;
}
return scores;
return List::create(Named("scores") = scores,
Named("stop_code") = stopping_codes,
Named("updates") = updates);
}

// --------------------------------------------------------------
Expand Down
113 changes: 113 additions & 0 deletions tests/testthat/test-cv-diagnostics.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
test_that('saved diagnostics and scores agree with independent fold refits', {
source <- dat_highsim$Y_source[1:8, 1:6]
target <- dat_highsim$Y_target[1:8, 1:6]
target[c(2, 9, 18)] <- NA_real_
n_folds <- 2L
for (separate in c(FALSE, TRUE)) {
for (relative in c(FALSE, TRUE)) {
control <- list(max_iter = 12, threshold = if (relative) .1 else 0,
stopping_rule = if (relative) 'relative' else 'absolute',
window = if (relative) 3 else 1)
args <- if (separate) {
list(lambda_1_row_all = c(2, 0), lambda_1_col_all = c(1, 0))
} else list(lambda_1_all = c(2, 0))
set.seed(91)
cv <- do.call(cv.learner, c(list(Y_source = list(source, source * .9),
Y_target = target, source_weights = c(.7, .3), r = 2,
lambda_2_all = c(.1, .01), n_folds = n_folds,
step_size = .001, control = control), args))
d <- cv$cv_diagnostics
expect_equal(nrow(d), length(cv$mse_all) * n_folds)
expect_identical(d$candidate, rep(seq_along(cv$mse_all), each = n_folds))
expect_identical(d$fold, rep(seq_len(n_folds), length(cv$mse_all)))
expect_identical(d$selected, d$candidate == which.min(cv$mse_all))
set.seed(91)
idx <- sample(which(!is.na(target)))
n <- length(idx)
folds <- lapply(seq_len(n_folds), function(k) {
idx[seq.int(floor((k - 1) * n / n_folds) + 1, floor(k * n / n_folds))]
})
expected_scores <- numeric(length(cv$mse_all))
for (i in seq_len(nrow(d))) {
held_out <- folds[[d$fold[i]]]
training <- target
training[held_out] <- NA_real_
fit <- learner(list(source, source * .9), training,
source_weights = c(.7, .3), r = 2,
lambda_1_row = d$lambda_1_row[i], lambda_1_col = d$lambda_1_col[i],
lambda_2 = d$lambda_2[i], step_size = .001, control = control)
expect_equal(d$stop_code[i], fit$convergence_criterion)
expect_equal(d$updates[i], length(fit$objective_values))
g <- d$candidate[i]
expected_scores[g] <- expected_scores[g] +
sum((target[held_out] - fit$learner_estimate[held_out])^2)
}
expect_equal(as.vector(cv$mse_all), expected_scores, tolerance = 1e-12)
if (!separate) expect_equal(d$lambda_1_row, d$lambda_1_col)
}
}
})

test_that('non-finite candidates record failure and mark skipped folds as missing', {
source <- diag(c(4, 3, 2, 1))
target <- diag(c(1, 2, 3, 4))
set.seed(42)
expect_warning(cv <- cv.learner(source, target, r = 2,
lambda_1_all = c(1e308, 0), lambda_2_all = 1, step_size = .001,
n_folds = 3, control = list(max_iter = 3)), 'non-finite validation errors')
d <- cv$cv_diagnostics
expect_equal(d$stop_code[1:3], c(3L, NA_integer_, NA_integer_))
expect_equal(d$updates[1:3], c(1L, NA_integer_, NA_integer_))
expect_false(any(d$selected[1:3]))
expect_false(anyNA(d$stop_code[4:6]))
expect_true(all(d$selected[4:6]))
})

test_that('CV records tolerance stopping including the singleton grid', {
source <- matrix(0, 4, 3)
set.seed(12)
cv <- cv.learner(source, source, r = 1, lambda_1_all = 0,
lambda_2_all = 0, step_size = .001, n_folds = 2,
control = list(max_iter = 10))
expect_equal(cv$cv_diagnostics$stop_code, c(1L, 1L))
expect_equal(cv$cv_diagnostics$updates, c(2L, 2L))
expect_true(all(cv$cv_diagnostics$selected))
})

test_that('diagnostics match refits when stopping differs across candidates and folds', {
sources <- dat_multisource$Y_source
target <- dat_multisource$Y_target
weights <- c(0.7, 0.3)
settings <- list(
list(n_folds = 3L, args = list(lambda_1_all = c(0, 1, 10))),
list(n_folds = 2L, args = list(lambda_1_row_all = c(0, 1, 10),
lambda_1_col_all = c(0, 1, 10))))
for (setting in settings) {
n_folds <- setting$n_folds
set.seed(815)
cv <- do.call(cv.learner, c(list(Y_source = sources, Y_target = target,
r = 3, source_weights = weights, lambda_2_all = c(0.1, 1),
n_folds = n_folds, step_size = 0.0003, control = list(max_iter = 2000)),
setting$args))
d <- cv$cv_diagnostics
# The rows must differ, or a row stored in the wrong place would go unnoticed.
expect_gt(nrow(unique(d[c('stop_code', 'updates')])), 3L)
expect_gt(length(unique(d$stop_code)), 1L)
set.seed(815)
idx <- sample(which(!is.na(target)))
n <- length(idx)
folds <- lapply(seq_len(n_folds), function(k) {
idx[seq.int(floor((k - 1) * n / n_folds) + 1, floor(k * n / n_folds))]
})
for (i in seq_len(nrow(d))) {
training <- target
training[folds[[d$fold[i]]]] <- NA_real_
fit <- learner(sources, training, r = 3, source_weights = weights,
lambda_1_row = d$lambda_1_row[i], lambda_1_col = d$lambda_1_col[i],
lambda_2 = d$lambda_2[i], step_size = 0.0003,
control = list(max_iter = 2000))
expect_equal(d$stop_code[i], fit$convergence_criterion)
expect_equal(d$updates[i], length(fit$objective_values))
}
}
})
2 changes: 1 addition & 1 deletion tests/testthat/test-cv-multisource.R
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ test_that("tied CV is the diagonal of the independent grid", {
tied <- fit(lambda_1_all = c(1, 3))
separate <- fit(lambda_1_row_all = c(1, 3), lambda_1_col_all = c(1, 3))
expect_named(tied, c('lambda_1_min', 'lambda_2_min', 'mse_all', 'r',
'lambda_1_all', 'lambda_2_all', 'n_observed'))
'lambda_1_all', 'lambda_2_all', 'n_observed', 'cv_diagnostics'))
expect_equal(dim(tied$mse_all), c(2L, 2L))
for (i in 1:2) expect_equal(tied$mse_all[i, ], separate$mse_all[i, i, ], tolerance = 1e-10)
single <- fit(lambda_1_row_all = 1, lambda_1_col_all = 3)
Expand Down
Loading
Loading