library(cvms)
context("cross_validate()")

# NOTICE:
# Numbers tested are the results I got and not "what should be"
# This will allow me to see if something changes, but it shouldn't give false confidence.


test_that("binomial models work with cross_validate()", {

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  CVbinomlist <- cross_validate(dat,
    formulas = c("diagnosis~score", "diagnosis~age"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  expect_equal(CVbinomlist$AUC, c(0.7615741, 0.1666667), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Lower CI`, c(0.58511535, 0.01748744), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Upper CI`, c(0.9380328, 0.3158459), tolerance = 1e-3)
  expect_equal(CVbinomlist$Kappa, c(0.4927536, -0.3636364), tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, c(0.5833333, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$Specificity, c(0.8888889, 0.6666667), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Pos Pred Value`, c(0.7777778, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Neg Pred Value`, c(0.7619048, 0.5), tolerance = 1e-3)
  expect_equal(CVbinomlist$F1, c(0.6666667, NA), tolerance = 1e-3)
  expect_equal(CVbinomlist$Prevalence, c(0.4, 0.4), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Rate`, c(0.2333333, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Prevalence`, c(0.3, 0.2), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Balanced Accuracy`, c(0.7361111, 0.3333333), tolerance = 1e-3)
  expect_equal(CVbinomlist$MCC, c(0.5048268, -0.4082483), tolerance = 1e-3)
  expect_equal(CVbinomlist$Folds, c(4, 4))
  expect_equal(CVbinomlist$`Fold Columns`, c(1, 1))
  expect_equal(CVbinomlist$`Convergence Warnings`, c(0, 0))
  expect_equal(CVbinomlist$Dependent, c("diagnosis", "diagnosis"))
  expect_equal(CVbinomlist$Fixed, c("score", "age"))

  # Enter sub tibbles
  expect_is(CVbinomlist$Predictions[[1]], "tbl_df")
  expect_is(CVbinomlist$ROC[[1]]$.folds, "roc")
  expect_equal(
    colnames(CVbinomlist$Predictions[[1]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction", "Predicted Class"
    )
  )
  expect_equal(
    names(CVbinomlist$ROC[[1]]$.folds),
    c(
      "percent", "sensitivities", "specificities", "thresholds",
      "direction", "cases", "controls", "fun.sesp", "auc", "call",
      "original.predictor", "original.response", "predictor", "response",
      "levels"
    )
  )
  expect_equal(CVbinomlist$ROC[[1]]$.folds$levels, c("1", "0"))
  expect_equal(as.numeric(CVbinomlist$ROC[[1]]$.folds$auc), 0.761574074074074, tolerance = 1e-5)
  expect_equal(CVbinomlist$ROC[[1]]$.folds$direction, ">")
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$sensitivities,
    c(1, 1, 1, 1, 1, 1, 1, 0.916666666666667, 0.833333333333333,
    0.833333333333333, 0.833333333333333, 0.75, 0.75, 0.75, 0.666666666666667,
    0.583333333333333, 0.583333333333333, 0.583333333333333, 0.583333333333333,
    0.583333333333333, 0.5, 0.416666666666667, 0.333333333333333,
    0.333333333333333, 0.25, 0.166666666666667, 0.166666666666667,
    0.0833333333333333, 0),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$specificities,
    c(0, 0.0555555555555556, 0.111111111111111, 0.166666666666667,
    0.222222222222222, 0.277777777777778, 0.333333333333333, 0.388888888888889,
    0.388888888888889, 0.444444444444444, 0.5, 0.5, 0.611111111111111,
    0.666666666666667, 0.666666666666667, 0.666666666666667, 0.722222222222222,
    0.777777777777778, 0.833333333333333, 0.888888888888889, 0.888888888888889,
    0.888888888888889, 0.888888888888889, 0.944444444444444, 0.944444444444444,
    0.944444444444444, 1, 1, 1),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$thresholds,
    c(Inf, 0.929306616108083, 0.909484002715809, 0.87591323408248,
    0.855631927354967, 0.848041386220925, 0.837689984778296, 0.802294462575759,
    0.76725510377552, 0.748457739195168, 0.72744063532645, 0.713631833116223,
    0.698028863110633, 0.668895524575313, 0.627303515132141, 0.59573874641954,
    0.586948588473742, 0.56976858247031, 0.533712764066178, 0.484386733644502,
    0.411849382147996, 0.318495485800134, 0.258156534624716, 0.212151384771614,
    0.165941018105731, 0.144237359545136, 0.112757724455647, 0.0923605619969366,
    -Inf),
    tolerance = 1e-5
  )
  expect_equal(nrow(CVbinomlist$Predictions[[1]]), 30)
  expect_equal(
    CVbinomlist$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )

  expect_true(
    as.character(CVbinomlist$Process[[1]]) %in%
    paste0("---\nProcess Information\n---\nTarget column: target\nPredi",
           "ction column: prediction\nFamily / type: Binomial\nClasses: ",
           "0, 1\nPositive class: 0\nCutoff: 0.5\nProbabilities are of c",
           "lass: 1\nProbabilities < 0.5 are considered: 0\nProbabilitie",
           "s >= 0.5 are considered: 1\nLocale used when sorting class l",
           "evels (LC_ALL): \n  ",
           c("en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8",
             "C/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8",
             Sys.getlocale()),
           "\nTarget counts: total=30, 0=12, 1=18\n",
           "Probability summary: mean: 0.585, median: 0.669, range: [0.0",
           "91, 0.932], SD: 0.272, IQR: 0.426\n---")
  )



})

test_that("binomial models checks that dependent variable is numeric with cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(7)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  ) %>%
    dplyr::mutate(diagnosis = factor(diagnosis))

  CVbinomlist <- cross_validate(dat,
    formulas = c("diagnosis~score", "diagnosis~age"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  expect_equal(CVbinomlist$AUC, c(0.7476852, 0.4583333), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Lower CI`, c(0.5621978, 0.2309283), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Upper CI`, c(0.9331726, 0.6857384), tolerance = 1e-3)
  expect_equal(CVbinomlist$Kappa, c(0.4285714, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, c(0.5833333, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$Specificity, c(0.8333333, 1.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Pos Pred Value`, c(0.7, NaN), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Neg Pred Value`, c(0.75, 0.60), tolerance = 1e-3)
  expect_equal(CVbinomlist$F1, c(0.6363636, NA), tolerance = 1e-3)
  expect_equal(CVbinomlist$Prevalence, c(0.4, 0.4), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Rate`, c(0.2333333, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Prevalence`, c(0.3333, 0.0), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Balanced Accuracy`, c(0.7083333, 0.5000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$MCC, c(0.4330127, 0.0000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$Folds, c(4, 4))
  expect_equal(CVbinomlist$`Fold Columns`, c(1, 1))
  expect_equal(CVbinomlist$`Convergence Warnings`, c(0, 0))
  expect_equal(CVbinomlist$Dependent, c("diagnosis", "diagnosis"))
  expect_equal(CVbinomlist$Fixed, c("score", "age"))

  # Enter sub tibbles
  expect_is(CVbinomlist$Predictions[[1]], "tbl_df")
  expect_equal(
    colnames(CVbinomlist$Predictions[[1]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction", "Predicted Class"
    )
  )
  expect_equal(nrow(CVbinomlist$Predictions[[1]]), 30)

  expect_is(CVbinomlist$ROC[[1]]$.folds, "roc")
  expect_equal(
    names(CVbinomlist$ROC[[1]]$.folds),
    c(
      "percent", "sensitivities", "specificities", "thresholds",
      "direction", "cases", "controls", "fun.sesp", "auc", "call",
      "original.predictor", "original.response", "predictor", "response",
      "levels"
    )
  )
  expect_equal(CVbinomlist$ROC[[1]]$.folds$levels, c("1", "0"))
  expect_equal(as.numeric(CVbinomlist$ROC[[1]]$.folds$auc), 0.747685185185185, tolerance = 1e-5)
  expect_equal(CVbinomlist$ROC[[1]]$.folds$direction, ">")
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$sensitivities,
    c(1, 1, 1, 1, 0.916666666666667, 0.916666666666667, 0.916666666666667,
    0.916666666666667, 0.833333333333333, 0.833333333333333, 0.833333333333333,
    0.833333333333333, 0.75, 0.75, 0.75, 0.666666666666667, 0.666666666666667,
    0.666666666666667, 0.666666666666667, 0.583333333333333, 0.5,
    0.416666666666667, 0.416666666666667, 0.333333333333333, 0.25,
    0.25, 0.166666666666667, 0.0833333333333333, 0.0833333333333333,
    0),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$specificities,
    c(0, 0.0555555555555556, 0.111111111111111, 0.166666666666667,
    0.222222222222222, 0.277777777777778, 0.333333333333333, 0.388888888888889,
    0.388888888888889, 0.444444444444444, 0.5, 0.555555555555556,
    0.555555555555556, 0.611111111111111, 0.666666666666667, 0.666666666666667,
    0.722222222222222, 0.777777777777778, 0.833333333333333, 0.833333333333333,
    0.833333333333333, 0.833333333333333, 0.888888888888889, 0.888888888888889,
    0.888888888888889, 0.944444444444444, 0.944444444444444, 0.944444444444444,
    1, 1),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds$thresholds,
    c(Inf, 0.934020744586008, 0.904215196791356, 0.891679194000951,
    0.885092557373248, 0.870500677113118, 0.817370482282321, 0.775314148242544,
    0.770445420121476, 0.764568494056223, 0.751876043966269, 0.732032329405983,
    0.718228755022429, 0.705257363311605, 0.675452184439646, 0.638965035749196,
    0.617497207831997, 0.566341981618961, 0.516282646461478, 0.481802525310446,
    0.438640212416923, 0.40604895253334, 0.384099214345533, 0.313431488336455,
    0.198808108757015, 0.150590529552897, 0.124520148716463, 0.0804638631263696,
    0.0474149270573646, -Inf),
    tolerance = 1e-5
  )

  expect_equal(
    CVbinomlist$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0),  Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )

})

test_that("binomial mixed models work with cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(20)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  CVbinomlistrand <- cross_validate(dat,
    formulas = c("diagnosis~score + (1|session)", "diagnosis~age"),
    fold_cols = ".folds",
    family = "binomial",
    verbose = FALSE,
    positive = 1
  )

  expect_equal(CVbinomlistrand$AUC, c(0.8472222, 0.4166667), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Lower CI`, c(0.6952664, 0.1964321), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Upper CI`, c(0.9991780, 0.6369012), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$Kappa, c(0.58333333, -0.08695652), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$Sensitivity, c(0.75, 0.25), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$Specificity, c(0.8333333, 0.6666667), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Pos Pred Value`, c(0.750, 0.3333333), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Neg Pred Value`, c(0.8333333, 0.5714286), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$F1, c(0.750, 0.2857143), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$Prevalence, c(0.4, 0.4), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Detection Rate`, c(0.3, 0.1), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Detection Prevalence`, c(0.4, 0.3), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$`Balanced Accuracy`, c(0.7916667, 0.4583333), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$MCC, c(0.58333333, -0.08908708), tolerance = 1e-3)
  expect_equal(CVbinomlistrand$Folds, c(4, 4))
  expect_equal(CVbinomlistrand$`Fold Columns`, c(1, 1))
  expect_equal(CVbinomlistrand$`Convergence Warnings`, c(0, 0))
  expect_equal(CVbinomlistrand$Dependent, c("diagnosis", "diagnosis"))
  expect_equal(CVbinomlistrand$Fixed, c("score", "age"))
  expect_equal(CVbinomlistrand$Random, c("(1|session)", NA))


  # Enter sub tibbles
  expect_is(CVbinomlistrand$Predictions[[1]], "tbl_df")
  expect_equal(
    colnames(CVbinomlistrand$Predictions[[1]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction", "Predicted Class"
    )
  )
  expect_equal(nrow(CVbinomlistrand$Predictions[[1]]), 30)

  expect_is(CVbinomlistrand$ROC[[1]]$.folds, "roc")
  expect_equal(
    names(CVbinomlistrand$ROC[[1]]$.folds),
    c(
      "percent", "sensitivities", "specificities", "thresholds",
      "direction", "cases", "controls", "fun.sesp", "auc", "call",
      "original.predictor", "original.response", "predictor", "response",
      "levels"
    )
  )
  expect_equal(CVbinomlistrand$ROC[[1]]$.folds$levels, c("1", "0"))
  expect_equal(as.numeric(CVbinomlistrand$ROC[[1]]$.folds$auc), 0.847222222222222, tolerance = 1e-5)
  expect_equal(CVbinomlistrand$ROC[[1]]$.folds$direction, ">")
  expect_equal(
    CVbinomlistrand$ROC[[1]]$.folds$sensitivities,
    c(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.916666666666667, 0.916666666666667,
    0.916666666666667, 0.916666666666667, 0.833333333333333, 0.75,
    0.666666666666667, 0.583333333333333, 0.5, 0.416666666666667,
    0.333333333333333, 0.25, 0.25, 0),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlistrand$ROC[[1]]$.folds$specificities,
    c(0, 0.0555555555555556, 0.111111111111111, 0.166666666666667,
    0.222222222222222, 0.277777777777778, 0.333333333333333, 0.388888888888889,
    0.444444444444444, 0.5, 0.555555555555556, 0.611111111111111,
    0.666666666666667, 0.666666666666667, 0.722222222222222, 0.777777777777778,
    0.833333333333333, 0.833333333333333, 0.833333333333333, 0.833333333333333,
    0.833333333333333, 0.833333333333333, 0.833333333333333, 0.833333333333333,
    0.833333333333333, 0.888888888888889, 1),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlistrand$ROC[[1]]$.folds$thresholds,
    c(Inf, 0.993513013802942, 0.990122822131509, 0.982593357075956,
    0.978073203960057, 0.974834920088241, 0.967489298252778, 0.951022352464516,
    0.926191328135861, 0.906411982969134, 0.880611797051915, 0.854512408532099,
    0.844429058082737, 0.825448120881781, 0.807815775362585, 0.77267721501017,
    0.715748901518313, 0.599746714157418, 0.408057876585855, 0.252253108215509,
    0.192102155750595, 0.181964804133168, 0.10447675558254, 0.0339994582131968,
    0.0179527181952166, 0.00100117667657215, -Inf),
    tolerance = 1e-5
  )

  expect_is(CVbinomlistrand$Predictions[[2]], "tbl_df")
  expect_equal(
    colnames(CVbinomlistrand$Predictions[[2]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction", "Predicted Class"
    )
  )
  expect_equal(nrow(CVbinomlistrand$Predictions[[2]]), 30)
  expect_is(CVbinomlistrand$ROC[[2]]$.folds, "roc")
  expect_equal(
    names(CVbinomlistrand$ROC[[2]]$.folds),
    c(
      "percent", "sensitivities", "specificities", "thresholds",
      "direction", "cases", "controls", "fun.sesp", "auc", "call",
      "original.predictor", "original.response", "predictor", "response",
      "levels"
    )
  )
  expect_equal(CVbinomlistrand$ROC[[2]]$.folds$levels, c("1", "0"))
  expect_equal(as.numeric(CVbinomlistrand$ROC[[2]]$.folds$auc), 0.416666666666667)
  expect_equal(CVbinomlistrand$ROC[[2]]$.folds$direction, ">")

  expect_equal(
    CVbinomlistrand$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )

})

test_that("gaussian model with cross_validate()", {

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  # Cross-validate the data
  CVed <- cross_validate(dat, "score~diagnosis",
    fold_cols = ".folds",
    family = "gaussian",
    REML = FALSE,
    verbose = FALSE,
    metrics = list("r2m" = TRUE, "r2c" = TRUE)
  )


  ## Testing 'CVed'                                                         ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing class
  expect_equal(
    class(CVed),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    CVed[["Fixed"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    CVed[["RMSE"]],
    17.16817,
    tolerance = 1e-4)
  expect_equal(
    CVed[["MAE"]],
    14.26914,
    tolerance = 1e-4)
  expect_equal(
    CVed[["NRMSE(IQR)"]],
    0.9074,
    tolerance = 1e-4)
  expect_equal(
    CVed[["RRSE"]],
    0.9805,
    tolerance = 1e-4)
  expect_equal(
    CVed[["RAE"]],
    0.98599,
    tolerance = 1e-4)
  expect_equal(
    CVed[["RMSLE"]],
    0.47943,
    tolerance = 1e-4)
  expect_equal(
    CVed[["r2m"]],
    0.26408,
    tolerance = 1e-4)
  expect_equal(
    CVed[["r2c"]],
    0.26408,
    tolerance = 1e-4)
  expect_equal(
    CVed[["AIC"]],
    194.6904,
    tolerance = 1e-4)
  expect_equal(
    CVed[["AICc"]],
    195.99628,
    tolerance = 1e-4)
  expect_equal(
    CVed[["BIC"]],
    198.02426,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Folds"]],
    4,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Fold Columns"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Convergence Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Singular Fit Messages"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CVed[["Dependent"]],
    "score",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(CVed),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "r2m", "r2c", "AIC", "AICc", "BIC", "Predictions", "Results",
      "Coefficients", "Folds", "Fold Columns", "Convergence Warnings",
      "Singular Fit Messages", "Other Warnings", "Warnings and Messages",
      "Process", "Dependent"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(CVed),
    c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "numeric", "numeric",
      "list", "list", "list", "integer", "integer", "integer", "integer",
      "integer", "list", "list", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(CVed),
    c("character", "double", "double", "double", "double", "double",
      "double", "double", "double", "double", "double", "double",
      "list", "list", "list", "integer", "integer", "integer", "integer",
      "integer", "list", "list", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(CVed),
    c(1L, 23L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(CVed)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'CVed'                                                ####

  expect_equal(CVed$Folds, 4)
  expect_equal(CVed$`Fold Columns`, 1)
  expect_equal(CVed$`Convergence Warnings`, 0)
  expect_equal(CVed$Dependent, "score")
  expect_equal(CVed$Fixed, "diagnosis")
  expect_equal(
    CVed$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )

  expect_true(
    as.character(CVed$Process[[1]]) %in%
    paste0("---\nProcess Information\n---\nTarget column: target\nPredi",
           "ction column: prediction\nFamily / type: Gaussian\nTarget su",
           "mmary: mean: 38.767, median: 35, range: [10, 81], SD: 19.294",
           ", IQR: 28\nPrediction summary: mean: 39.247, median: 33.167,",
           " range: [27.467, 53.333], SD: 9.863, IQR: 17.611\nLocale (LC",
           "_ALL): \n  ",
           c("en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8",
             "C/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8",
             Sys.getlocale()),
           "\n---"))

})

test_that("gaussian mixed models with cross_validate()", {

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  # Cross-validate the data
  CVed <- cross_validate(dat, c("score~diagnosis + (1|session)", "score~age + (1|session)"),
    fold_cols = ".folds",
    family = "gaussian",
    REML = FALSE,
    verbose = FALSE,
    metrics = list("r2m" = TRUE, "r2c" = TRUE)
  )

  expect_equal(
    colnames(CVed),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "r2m", "r2c", "AIC", "AICc", "BIC", "Predictions", "Results",
      "Coefficients", "Folds", "Fold Columns", "Convergence Warnings",
      "Singular Fit Messages", "Other Warnings", "Warnings and Messages",
      "Process", "Dependent", "Random"),
    fixed = TRUE)

  expect_equal(CVed$RMSE, c(9.65949, 15.20226), tolerance = 1e-3)
  expect_equal(CVed$MAE, c(7.145933, 13.577082), tolerance = 1e-3)
  expect_equal(CVed$r2m, c(0.28219291, 0.01319592), tolerance = 1e-3)
  expect_equal(CVed$r2c, c(0.8043140, 0.5016056), tolerance = 1e-3)
  expect_equal(CVed$AIC, c(175.9497, 194.6358), tolerance = 1e-3)
  expect_equal(CVed$AICc, c(178.2523, 196.9384), tolerance = 1e-3)
  expect_equal(CVed$BIC, c(180.3948, 199.0809), tolerance = 1e-3)
  expect_equal(CVed$Folds, c(4, 4))
  expect_equal(CVed$`Fold Columns`, c(1, 1))
  expect_equal(CVed$`Convergence Warnings`, c(0, 0))
  expect_equal(CVed$Dependent, c("score", "score"))
  expect_equal(CVed$Fixed, c("diagnosis", "age"))
  expect_equal(CVed$Random, c("(1|session)", "(1|session)"))
  expect_equal(
    CVed$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )
})

test_that("binomial models work with control specified in cross_validate()", {
  testthat::skip("mac and ubuntu give different warnings")
  # TODO fix such that travis get same results

  skip("testing different optimizers is too difficult given platform differences")

  # Load data and fold it
  xpectr::set_test_seed(7)
  dat <- groupdata2::fold(participant.scores,
    k = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  tryCatch(
    {
      cross_validate(
        dat,
        formulas = c("diagnosis~score + (1|session)"),
        fold_cols = ".folds",
        family = "binomial",
        REML = FALSE,
        control = lme4::glmerControl(optimizer = "bobyqa"),
        verbose = FALSE,
        positive = 1
      )
    },
    warning = function(w) {
      expect_true(grepl("unable to evaluate scaled gradient", as.character(w), ignore.case = TRUE))
    }
  )


  cv_Nelder_Mead <- cross_validate(
    dat %>% dplyr::bind_rows(dat, dat, dat, dat),
    formulas = c("diagnosis~score + age + (1|session)"),
    fold_cols = ".folds",
    family = "binomial",
    REML = FALSE,
    control = lme4::glmerControl(optimizer = "Nelder_Mead"),
    verbose = FALSE,
    positive = 1
  )

  cv_bobyqa <- cross_validate(
    dat %>% dplyr::bind_rows(dat, dat, dat, dat),
    formulas = c("diagnosis~score + age + (1|session)"),
    fold_cols = ".folds",
    family = "binomial",
    REML = FALSE,
    control = lme4::glmerControl(optimizer = "bobyqa"),
    verbose = FALSE,
    positive = 1
  )

  # Gather the results from the two different optimizers
  cv <- cv_Nelder_Mead %>%
    dplyr::bind_rows(cv_bobyqa)

  # Appears to be different on linux
  expect_true(cv$`Balanced Accuracy`[[1]] != cv$`Balanced Accuracy`[[2]]) # , c(0.736111111111111, 0.777777777777778))
  expect_true(cv$AUC[[1]] != cv$AUC[[2]]) # c(0.824074074074074, 0.875))
  expect_true(cv$F1[[1]] != cv$F1[[2]]) # c(0.666666666666667, 0.727272727272727))
  expect_equal(cv$Fixed, c("score+age", "score+age"))
  expect_equal(cv$Random, c("(1|session)", "(1|session)"))
  expect_equal(cv$Dependent, c("diagnosis", "diagnosis"))
})

test_that("binomial models gives warnings with control specified in cross_validate()", {
  testthat::skip("mac and ubuntu give different warnings")
  # Tested on both platforms on travis as well
  # Local test should run on mac as is

  # skip_test_if_old_R_version()

  # Load data and fold it
  xpectr::set_test_seed(7)
  dat <- groupdata2::fold(participant.scores,
    k = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  # Singular fit message

  cv_process <- tryCatch({
    purrr::map(.x = 1, .f = purrr::quietly(function(.x) {
      xpectr::set_test_seed(2)
      cross_validate(dat,
        formulas = c("diagnosis ~ score + age + (1|session) + (1|age)"),
        fold_cols = ".folds",
        family = "binomial",
        REML = FALSE,
        control = lme4::glmerControl(
          optimizer = "bobyqa",
          optCtrl = list(maxfun = 100)
        ),
        verbose = FALSE
      )
    }))
  })

  expect_equal(cv_process[[1]]$messages,
    "\n--------------------------------------------------\ncross_validate(): Boundary (Singular) Fit Message:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n1\nboundary (singular) fit: see ?isSingular\n",
    fixed = TRUE
  )

  ### NOTE: The warnings are different between mac and linux
  # So we cannot check the below :/
  expect_equal(
    xpectr::strip(cv_process[[1]]$warnings, remove_numbers = TRUE),
    xpectr::strip(c(
      "\n-------------------------------------\ncross_validate(): Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n1\nmaxfun < 10 * length(par)^2 is not recommended.",
      "\n-------------------------------------\ncross_validate(): Convergence Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n1\nconvergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded",
      "\n-------------------------------------\ncross_validate(): Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n2\nmaxfun < 10 * length(par)^2 is not recommended.",
      "\n-------------------------------------\ncross_validate(): Convergence Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n2\nconvergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded",
      "\n-------------------------------------\ncross_validate(): Convergence Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n2\nModel failed to converge with max|grad| = 0.0405867 (tol = 0.001, component 1)",
      "\n-------------------------------------\ncross_validate(): Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n3\nmaxfun < 10 * length(par)^2 is not recommended.",
      "\n-------------------------------------\ncross_validate(): Convergence Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n3\nconvergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded",
      "\n-------------------------------------\ncross_validate(): Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n3\nunable to evaluate scaled gradient",
      "\n-------------------------------------\ncross_validate(): Convergence Warning:\nIn model:\ndiagnosis ~ score + age + (1|session) + (1|age)\nFor fold column:\n.folds\nIn fold:\n3\nModel failed to converge: degenerate  Hessian with 1 negative eigenvalues"
    ), remove_numbers = TRUE),
    fixed = TRUE
  )


  # xpectr::set_test_seed(2)
  # cv_messages <- suppressMessages(cross_validate(dat,
  #                               formulas = c("diagnosis ~ score + age + (1|session) + (1|age)"),
  #                               fold_cols = '.folds',
  #                               family='binomial',
  #                               REML = FALSE,
  #                               control = lme4::glmerControl(optimizer="bobyqa",
  #                                                            optCtrl=list(maxfun=100))))
  # expect_equal(cv_messages$`Singular Fit Messages`, 2)
})

test_that("gaussian models work with control specified in cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(2)
  dat <- groupdata2::fold(participant.scores,
    k = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  CVgausslistrand <- cross_validate(dat,
    formulas = c("score~diagnosis + (1|session)"),
    fold_cols = ".folds",
    family = "gaussian",
    REML = FALSE,
    control = lme4::lmerControl(
      optimizer = "Nelder_Mead",
      optCtrl = list(maxfun = 1000000)
    ),
    verbose = FALSE
  )

  expect_equal(CVgausslistrand$RMSE, c(10.44299), tolerance = 1e-3)
  expect_equal(CVgausslistrand$RMSLE, c(0.2959932), tolerance = 1e-3)
  expect_equal(CVgausslistrand$`Convergence Warnings`, c(0))


  ## Testing 'cross_validate( dat, formulas = c("score~dia...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing side effects
  # Assigning side effects
  side_effects_15526 <- xpectr::capture_side_effects(cross_validate(
      dat,
      formulas = c("score~diagnosis + (1|session)"),
      fold_cols = ".folds",
      family = "gaussian",
      REML = FALSE,
      control = lme4::lmerControl(
        optimizer = "bobyqa",
        optCtrl = list(maxfun = 10)
      ),
      verbose = FALSE
    ), reset_seed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_15526[['warnings']], remove_numbers = TRUE),
    xpectr::strip(c("\n---\ncross_validate(): convergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded\nNote: Convergence Warning\nFor:\nFormula: score~diagnosis + (1|session)\nFold column: .folds\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"bobyqa\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(maxfun = 10))), model_verbose : FALSE, family : gaussian, is_special_fn : TRUE",
      "\n---\ncross_validate(): Model failed to converge with max|grad| = 0.429297 (tol = 0.002, component 1)\nNote: Convergence Warning\nFor:\nFormula: score~diagnosis + (1|session)\nFold column: .folds\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"bobyqa\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(maxfun = 10))), model_verbose : FALSE, family : gaussian, is_special_fn : TRUE",
      "\n---\ncross_validate(): convergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded\nNote: Convergence Warning\nFor:\nFormula: score~diagnosis + (1|session)\nFold column: .folds\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"bobyqa\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(maxfun = 10))), model_verbose : FALSE, family : gaussian, is_special_fn : TRUE",
      "\n---\ncross_validate(): convergence code 1 from bobyqa: bobyqa -- maximum number of function evaluations exceeded\nNote: Convergence Warning\nFor:\nFormula: score~diagnosis + (1|session)\nFold column: .folds\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"bobyqa\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(maxfun = 10))), model_verbose : FALSE, family : gaussian, is_special_fn : TRUE"),
      remove_numbers = TRUE),
    fixed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_15526[['messages']]),
    xpectr::strip(character(0)),
    fixed = TRUE)
  # Assigning output
  output_15526 <- xpectr::suppress_mw(cross_validate(
      dat,
      formulas = c("score~diagnosis + (1|session)"),
      fold_cols = ".folds",
      family = "gaussian",
      REML = FALSE,
      control = lme4::lmerControl(
        optimizer = "bobyqa",
        optCtrl = list(maxfun = 10)
      ),
      verbose = FALSE
    ))
  # Testing class
  expect_equal(
    class(output_15526),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    output_15526[["Fixed"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    output_15526[["RMSE"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["MAE"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["NRMSE(IQR)"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["RRSE"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["RAE"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["RMSLE"]],
    NaN,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["AIC"]],
    NA_real_,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["AICc"]],
    NA_real_,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["BIC"]],
    NA_real_,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Predictions"]],
    NA)
  expect_equal(
    output_15526[["Folds"]],
    3,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Fold Columns"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Convergence Warnings"]],
    3,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Singular Fit Messages"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_15526[["Dependent"]],
    "score",
    fixed = TRUE)
  expect_equal(
    output_15526[["Random"]],
    "(1|session)",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(output_15526),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "AIC", "AICc", "BIC", "Predictions", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent",
      "Random"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(output_15526),
    c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "logical", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "list", "character", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(output_15526),
    c("character", "double", "double", "double", "double", "double",
      "double", "double", "double", "double", "logical", "list", "list",
      "integer", "integer", "integer", "integer", "integer", "list",
      "list", "character", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(output_15526),
    c(1L, 22L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(output_15526)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'cross_validate( dat, formulas = c("score~dia...'     ####



  # TODO When counting singular (boundary fit) messages, uncomment and change expected warning/message
  # # Warning because of too few iterations
  # expect_warning(cross_validate(dat,
  #                               formulas = c("age~diagnosis*score + (score|session) + (1|score)"),
  #                               fold_cols = '.folds',
  #                               family='gaussian',
  #                               REML = FALSE,
  #                               control = lme4::lmerControl(optimizer="Nelder_Mead",
  #                                                            optCtrl=list(maxfun=100)),
  #                               verbose=FALSE), "cross_validate(): Convergence Warning:", fixed = TRUE)
})

test_that("model using dot in formula ( y ~ . ) works with cross_validate()", {

  # We wish to test if using the dot "y~." method in the model formula
  # correctly leaves out .folds column.

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  ) %>%
    dplyr::select(-dplyr::all_of(c("participant", "session")))

  # Expect no warnings
  # https://stackoverflow.com/questions/22003306/is-there-something-in-testthat-like-expect-no-warnings
  expect_warning(cross_validate(dat,
    formulas = c("diagnosis~."),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE, verbose = FALSE
  ),
  regexp = NA
  )

  # Expect no warnings
  # https://stackoverflow.com/questions/22003306/is-there-something-in-testthat-like-expect-no-warnings
  expect_warning(cross_validate(dat,
    formulas = c("score~."),
    fold_cols = ".folds", family = "gaussian",
    REML = FALSE, verbose = FALSE
  ),
  regexp = NA
  )
})

test_that("model using inline functions in formulas works with cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
                          k = 4,
                          cat_col = "diagnosis",
                          id_col = "participant"
  )

  # Expect no warnings
  # https://stackoverflow.com/questions/22003306/is-there-something-in-testthat-like-expect-no-warnings
  expect_warning(
    cv1 <- cross_validate(
      dat,
      formulas = c("diagnosis~log(score)",
                   "diagnosis~log(score) + (1|session)"),
      fold_cols = ".folds",
      family = "binomial",
      REML = FALSE,
      verbose = FALSE
    ),
    regexp = NA
  )

  # Testing values
  expect_equal(cv1$Fixed, c("log(score)", "log(score)"), fixed = TRUE)
  expect_equal(cv1$Random, c(NA, "(1|session)"), fixed = TRUE)
  expect_equal(cv1$F1, c(0.78947, 0.85714), tolerance = 1e-4)


  # Expect no warnings
  expect_warning(cv2 <- cross_validate(
    dat,
    formulas = c("score~log(age)", "score~log(age)+(1|session)"),
    fold_cols = ".folds", family = "gaussian",
    REML = FALSE, verbose = FALSE
  ),
  regexp = NA
  )

  # Testing values
  expect_equal(cv2$Fixed, c("log(age)", "log(age)"), fixed = TRUE)
  expect_equal(cv2$Random, c(NA, "(1|session)"), fixed = TRUE)
  expect_equal(cv2$RMSE, c(20.67139, 15.19049), tolerance = 1e-4)

})

test_that("binomial models work with repeated cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(2)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant",
    num_fold_cols = 2
  )

  CVbinomlist <- cross_validate(dat,
    formulas = c("diagnosis~score", "diagnosis~age"),
    fold_cols = c(".folds_1", ".folds_2"), family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  # Some platforms give slightly different AUCs for the second model
  # Possibly due to compilation differences affecting model probabilities?
  # So for those we check each model differently with multiple acceptable results
  expect_equal(CVbinomlist$AUC[[1]], 0.750, tolerance = 1e-3)
  expect_true(round(CVbinomlist$AUC[[2]], 3) %in% c(0.229, 0.233))

  expect_equal(CVbinomlist$`Lower CI`[[1]], 0.5638021, tolerance = 1e-3)
  expect_true(round(CVbinomlist$`Lower CI`[[2]], 3) %in% c(0.104, 0.106))

  expect_equal(CVbinomlist$`Upper CI`[[1]], 0.9361979, tolerance = 1e-3)
  expect_true(round(CVbinomlist$`Upper CI`[[2]], 3) %in% c(0.365, 0.369))

  expect_equal(CVbinomlist$Kappa, c(0.4606625, -0.3043478), tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, c(0.5833333, 0.1250000), tolerance = 1e-3)
  expect_equal(CVbinomlist$Specificity, c(0.8611111, 0.5833333), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Pos Pred Value`, c(0.7388889, 0.1666667), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Neg Pred Value`, c(0.7559524, 0.5000000), tolerance = 1e-3)
  expect_equal(CVbinomlist$F1, c(0.6515152, NA), tolerance = 1e-3)
  expect_equal(CVbinomlist$Prevalence, c(0.4, 0.4), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Rate`, c(0.2333333, 0.0500000), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Detection Prevalence`, c(0.3166667, 0.3), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Balanced Accuracy`, c(0.7222222, 0.3541667), tolerance = 1e-3)
  expect_equal(CVbinomlist$MCC, c(0.4689197, -0.3118048), tolerance = 1e-3)
  expect_equal(CVbinomlist$Folds, c(8, 8))
  expect_equal(CVbinomlist$`Fold Columns`, c(2, 2))
  expect_equal(CVbinomlist$`Convergence Warnings`, c(0, 0))
  expect_equal(CVbinomlist$Dependent, c("diagnosis", "diagnosis"))
  expect_equal(CVbinomlist$Fixed, c("score", "age"))

  # Enter sub tibbles
  expect_is(CVbinomlist$Predictions[[1]], "tbl_df")
  expect_is(CVbinomlist$Results[[1]], "tbl_df")
  expect_is(CVbinomlist$ROC[[1]]$.folds_1, "roc")
  expect_is(CVbinomlist$`Confusion Matrix`[[1]], "tbl_df")
  expect_equal(
    colnames(CVbinomlist$Predictions[[1]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction", "Predicted Class"
    )
  )
  expect_equal(
    colnames(CVbinomlist$Results[[1]]),
    c(
      "Fold Column", "Balanced Accuracy", "F1",
      "Sensitivity", "Specificity", "Pos Pred Value",
      "Neg Pred Value", "AUC", "Lower CI", "Upper CI",
      "Kappa", "MCC", "Detection Rate", "Detection Prevalence",
      "Prevalence"
    )
  )
  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]), c("Fold Column", "Prediction", "Target", "Pos_0", "Pos_1", "N"))
  expect_equal(nrow(CVbinomlist$Predictions[[1]]), 60)

  expect_equal(CVbinomlist$ROC[[1]]$.folds_1$direction, ">")
  expect_equal(as.numeric(CVbinomlist$ROC[[1]]$.folds_1$auc), 0.74537037037037)
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds_1$thresholds,
    c(Inf, 0.910863952937387, 0.904347165731837, 0.882622758109746,
    0.862043303565538, 0.850432570801589, 0.81565409306014, 0.776959885099219,
    0.748708837071827, 0.725216199854617, 0.715948808241982, 0.702694250196696,
    0.691111330225435, 0.683724065998779, 0.641975043550198, 0.595977361608476,
    0.582348401542413, 0.556320081310892, 0.517829000923097, 0.483586410050964,
    0.437738151479785, 0.380785717586265, 0.302221473822472, 0.227423806423394,
    0.165052601019878, 0.112813610322215, 0.091762887125031, 0.0674387702549503,
    -Inf),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds_1$sensitivities,
    c(1, 1, 0.916666666666667, 0.916666666666667, 0.916666666666667,
    0.916666666666667, 0.916666666666667, 0.916666666666667, 0.916666666666667,
    0.833333333333333, 0.833333333333333, 0.833333333333333, 0.75,
    0.75, 0.666666666666667, 0.666666666666667, 0.583333333333333,
    0.583333333333333, 0.583333333333333, 0.583333333333333, 0.5,
    0.416666666666667, 0.333333333333333, 0.25, 0.25, 0.166666666666667,
    0.0833333333333333, 0.0833333333333333, 0),
    tolerance = 1e-5
  )
  expect_equal(
    CVbinomlist$ROC[[1]]$.folds_1$specificities,
    c(0, 0.0555555555555556, 0.111111111111111, 0.166666666666667,
    0.222222222222222, 0.277777777777778, 0.333333333333333, 0.388888888888889,
    0.444444444444444, 0.444444444444444, 0.5, 0.555555555555556,
    0.611111111111111, 0.666666666666667, 0.666666666666667, 0.722222222222222,
    0.722222222222222, 0.777777777777778, 0.833333333333333, 0.888888888888889,
    0.888888888888889, 0.888888888888889, 0.888888888888889, 0.888888888888889,
    0.944444444444444, 0.944444444444444, 0.944444444444444, 1, 1
    ),
    tolerance = 1e-5
  )

  expect_equal(
    colnames(CVbinomlist$Coefficients[[1]]),
    c("Fold Column", "Fold", "term", "estimate", "std.error", "conf.level",
      "conf.low", "conf.high", "statistic", "df.error", "p.value")
  )
  expect_equal(
    colnames(CVbinomlist$Coefficients[[2]]),
    c("Fold Column", "Fold", "term", "estimate", "std.error", "conf.level",
      "conf.low", "conf.high", "statistic", "df.error", "p.value")
  )
  expect_equal(
    CVbinomlist$Coefficients[[1]]$p.value,
    c(
      0.01294663, 0.01931905, 0.04833047, 0.05703608, 0.01900735,
      0.03169528, 0.04338194, 0.05071058, 0.01639887, 0.02528367,
      0.04118387, 0.04851447, 0.01412963, 0.02178988, 0.04010255, 0.04752187
    )
  )
  expect_equal(
    CVbinomlist$Coefficients[[2]]$p.value,
    c(
      0.62478703, 0.84891972, 0.18479764, 0.14117487, 0.09443436,
      0.12527121, 0.13628239, 0.10142803, 0.66873537, 0.89953532, 0.21621108,
      0.18124974, 0.67252682, 0.45631539, 0.87784992, 0.77268412
    )
  )
  expect_equal(
    CVbinomlist$Coefficients[[1]]$estimate,
    c(
      5.26664582, -0.12300571, 2.56877552, -0.05531371, 3.03166220,
      -0.06342121, 2.80320088, -0.06211859, 3.61808397, -0.07362534,
      2.70911710, -0.06030082, 4.53407887, -0.10779184, 2.62880059, -0.05770449
    )
  )
  expect_equal(
    CVbinomlist$Coefficients[[2]]$estimate,
    c(
      0.818615019, -0.010725093, -2.665418817, 0.106774730, 4.483382847,
      -0.141769999, -3.054394678, 0.116528272, 0.714444780, -0.007070299,
      -3.594942567, 0.123832276, -0.701387195, 0.044667112, -0.340316825,
      0.024469156
    )
  )
  expect_equal(
    CVbinomlist$Coefficients[[1]]$std.error,
    c(
      2.11917509, 0.05258190, 1.30100740, 0.02906606, 1.29260651, 0.02952251,
      1.38771363, 0.03179176, 1.50758829, 0.03291193, 1.32690970, 0.03056558,
      1.84766496, 0.04698838, 1.28066063, 0.02911987
    )
  )
  expect_equal(
    CVbinomlist$Coefficients[[2]]$std.error,
    c(
      1.67379503, 0.05630060, 2.00992729, 0.07256528, 2.68071734, 0.09247766,
      2.05023360, 0.07114190, 1.66971492, 0.05600309, 2.90696307, 0.09262526,
      1.65937069, 0.05996167, 2.21422931, 0.08470771
    )
  )
  expect_equal(CVbinomlist$Coefficients[[1]]$statistic,
    c(
      2.485234, -2.339317, 1.974451, -1.903034, 2.345387, -2.148233,
      2.020014, -1.953921, 2.399915, -2.237041, 2.041674, -1.972834,
      2.453951, -2.294011, 2.052691, -1.981619
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist$Coefficients[[2]]$statistic,
    c(
      0.4890772, -0.1904970, -1.3261270, 1.4714300, 1.6724564, -1.5330188,
      -1.4897789, 1.6379696, 0.4278843, -0.1262484, -1.2366661, 1.3369170,
      -0.4226826, 0.7449278, -0.1536954, 0.2888657
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist$Coefficients[[1]]$Fold, c(1, 1, 2, 2, 3, 3, 4, 4, 1, 1, 2, 2, 3, 3, 4, 4))
  expect_equal(CVbinomlist$Coefficients[[2]]$Fold, c(1, 1, 2, 2, 3, 3, 4, 4, 1, 1, 2, 2, 3, 3, 4, 4))
  expect_equal(CVbinomlist$Coefficients[[1]]$`Fold Column`, rep(c(".folds_1", ".folds_2"), each = 8))
  expect_equal(CVbinomlist$Coefficients[[2]]$`Fold Column`, rep(c(".folds_1", ".folds_2"), each = 8))

  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$`Fold Column`, rep(c(".folds_1", ".folds_2"), each = 4))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$Prediction, as.character(c(0, 1, 0, 1, 0, 1, 0, 1)))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$Target, as.character(c(0, 0, 1, 1, 0, 0, 1, 1)))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$Pos_0, rep(c("TP", "FN", "FP", "TN"), 2))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$Pos_1, rep(c("TN", "FP", "FN", "TP"), 2))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(7, 5, 2, 16, 7, 5, 3, 15))

  expect_equal(CVbinomlist$Results[[1]]$`Fold Column`, c(".folds_1", ".folds_2"))
  expect_equal(CVbinomlist$Results[[1]]$`Balanced Accuracy`, c(0.7361111, 0.7083333), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$F1, c(0.6666667, 0.6363636), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$Sensitivity, c(0.5833333, 0.5833333), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$Specificity, c(0.8888889, 0.8333333), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Pos Pred Value`, c(0.7777778, 0.7), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Neg Pred Value`, c(0.7619048, 0.75), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$AUC, c(0.7453704, 0.7546296), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Lower CI`, c(0.5545666, 0.5730375), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Upper CI`, c(0.9361741, 0.9362218), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$Kappa, c(0.4927536, 0.4285714), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Detection Rate`, c(0.2333333, 0.2333333), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$`Detection Prevalence`, c(0.3000000, 0.33333), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$Prevalence, c(0.4, 0.4), tolerance = 1e-3)
  expect_equal(CVbinomlist$Results[[1]]$MCC, c(0.5048268, 0.4330127), tolerance = 1e-3)

  expect_equal(
    CVbinomlist$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )
})

test_that("binomial models work with `positive` as.character in cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(2)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant",
    num_fold_cols = 2
  ) %>%
    dplyr::mutate(diagnosis = factor(ifelse(diagnosis == 0, "E", "B")))

  # Positive == 1
  CVbinomlist <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = c(".folds_1"),
    family = "binomial",
    REML = FALSE,
    verbose = FALSE,
    positive = 1
  )

  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]),
               c("Fold Column", "Prediction", "Target", "Pos_B", "Pos_E", "N"))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(16, 2, 5, 7))
  expect_equal(CVbinomlist$F1, 0.8205, tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, 0.8888, tolerance = 1e-3)

  # Positive == 2
  CVbinomlist <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = c(".folds_1"),
    family = "binomial",
    REML = FALSE,
    verbose = FALSE,
    positive = 2
  )

  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]), c("Fold Column", "Prediction", "Target", "Pos_B", "Pos_E", "N"))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(16, 2, 5, 7))
  expect_equal(CVbinomlist$F1, 0.6666, tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, 0.5833, tolerance = 1e-3)

  # Positive == E (#2 alphabetically)
  CVbinomlist <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = c(".folds_1"),
    family = "binomial",
    REML = FALSE,
    verbose = FALSE,
    positive = "E"
  )

  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]), c("Fold Column", "Prediction", "Target", "Pos_B", "Pos_E", "N"))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(16, 2, 5, 7))
  expect_equal(CVbinomlist$F1, 0.6666, tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, 0.5833, tolerance = 1e-3)

  # Positive == B (#1 alphabetically)
  CVbinomlist <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = c(".folds_1"),
    family = "binomial",
    REML = FALSE,
    verbose = FALSE,
    positive = "B"
  )

  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]),
               c("Fold Column", "Prediction", "Target", "Pos_B", "Pos_E", "N"))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(16, 2, 5, 7))
  expect_equal(CVbinomlist$F1, c(0.8205), tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, 0.8888, tolerance = 1e-3)

  # Positive == C (not in dependent)
  expect_error(
    cross_validate(
      dat,
      formulas = c("diagnosis~score"),
      fold_cols = c(".folds_1"),
      family = "binomial",
      REML = FALSE,
      verbose = FALSE,
      positive = "C"
    ),
    "When 'positive' is a character, it must correspond to a factor level in the dependent variable.\n'positive' is C and levels are B and E."
  )

  # Interchanging the level names

  xpectr::set_test_seed(2)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant",
    num_fold_cols = 2
  ) %>%
    dplyr::mutate(diagnosis = factor(ifelse(diagnosis == 0, "B", "E")))

  CVbinomlist <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = c(".folds_1"),
    family = "binomial",
    REML = FALSE,
    verbose = FALSE,
    positive = 1
  )

  expect_equal(colnames(CVbinomlist$`Confusion Matrix`[[1]]),
               c("Fold Column", "Prediction", "Target", "Pos_B", "Pos_E", "N"))
  expect_equal(CVbinomlist$`Confusion Matrix`[[1]]$N, c(7, 5, 2, 16))
  expect_equal(CVbinomlist$F1, 0.6666, tolerance = 1e-3)
  expect_equal(CVbinomlist$Sensitivity, 0.5833, tolerance = 1e-3)

  expect_equal(
    CVbinomlist$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )

})

test_that("gaussian models work with repeated cross_validate()", {

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant",
    num_fold_cols = 2
  )

  CVgausslist <- cross_validate(dat,
    formulas = c("score~diagnosis", "score~age"),
    fold_cols = c(".folds_1", ".folds_2"), family = "gaussian",
    metrics = list("MALE" = TRUE),
    REML = FALSE, verbose = FALSE
  )


  expect_equal(CVgausslist$RMSE, c(16.69535, 20.07280), tolerance = 1e-3)
  expect_equal(CVgausslist$MAE, c(13.92099, 16.60674), tolerance = 1e-3)
  expect_equal(CVgausslist$`NRMSE(IQR)`, c(0.842571059775381, 1.03282819160309), tolerance = 1e-3)
  expect_equal(CVgausslist$RRSE, c(0.917347061955258, 1.09615517380249), tolerance = 1e-3)
  expect_equal(CVgausslist$RAE, c(0.925413461506034, 1.10589002785508), tolerance = 1e-3)
  expect_equal(CVgausslist$RMSLE, c(0.4737409, 0.5595062), tolerance = 1e-3)
  expect_equal(CVgausslist$MALE, c(0.3837563, 0.4488940), tolerance = 1e-3)
  expect_equal(CVgausslist$AIC, c(194.6793, 201.9189), tolerance = 1e-3)
  expect_equal(CVgausslist$AICc, c(195.9852, 203.2248), tolerance = 1e-3)
  expect_equal(CVgausslist$BIC, c(198.0132, 205.2527), tolerance = 1e-3)
  expect_equal(CVgausslist$Folds, c(8, 8))
  expect_equal(CVgausslist$`Fold Columns`, c(2, 2))
  expect_equal(CVgausslist$`Convergence Warnings`, c(0, 0))
  expect_equal(CVgausslist$Dependent, c("score", "score"))
  expect_equal(CVgausslist$Fixed, c("diagnosis", "age"))

  # Enter sub tibbles
  expect_is(CVgausslist$Results[[1]], "tbl_df")
  expect_is(CVgausslist$Coefficients[[1]], "tbl_df")
  expect_equal(
    colnames(CVgausslist$Results[[1]]),
    c("Fold Column", "Fold", "RMSE", "MAE", "NRMSE(IQR)", "RRSE",
    "RAE", "RMSLE", "MALE", "AIC", "AICc", "BIC")
  )
  expect_equal(
    CVgausslist$Results[[1]]$`Fold Column`,
    rep(c(".folds_1", ".folds_2"), each = 4)
  )
  expect_equal(
    colnames(CVgausslist$Predictions[[1]]),
    c(
      "Fold Column", "Fold", "Observation",
      "Target", "Prediction"
    )
  )
  expect_equal(unique(CVgausslist$Predictions[[1]]$`Fold Column`), c(".folds_1", ".folds_2"))
  expect_equal(
    colnames(CVgausslist$Coefficients[[1]]),
    c("Fold Column", "Fold", "term", "estimate", "std.error", "conf.level",
      "conf.low", "conf.high", "statistic", "df.error", "p.value")
  )
  expect_equal(
    CVgausslist$Coefficients[[1]]$p.value,
    c(
      8.230130e-09, 1.290208e-02, 1.346678e-08, 1.073278e-02, 2.434650e-09,
      1.980414e-03, 9.363412e-08, 4.950473e-02, 6.908574e-09, 1.352584e-02,
      9.209672e-08, 3.018268e-02, 1.700920e-08, 1.994213e-02, 2.078447e-09, 1.105365e-03
    )
  )
  expect_equal(
    CVgausslist$Coefficients[[2]]$p.value,
    c(
      0.009808606, 0.637825565, 0.258490506, 0.636485814, 0.010557346,
      0.646231138, 0.085798324, 0.557360724, 0.151732090, 0.638915931,
      0.093280236, 0.851553515, 0.017961529, 0.953518671, 0.021408386, 0.658577591
    )
  )
  expect_equal(CVgausslist$Coefficients[[1]]$estimate,
    c(
      49.55556, -18.88889, 53.33333, -21.16667, 51.00000, -23.53333,
      49.77778, -16.61111, 49.55556, -18.55556, 49.77778, -18.52778,
      51.00000, -18.80000, 53.33333, -25.58333
    ),
    tolerance = 1e-5
  )
  expect_equal(CVgausslist$Coefficients[[2]]$estimate,
    c(
      45.14324082, -0.25604297, 29.32953312, 0.42100977, 43.24298347,
      -0.25509419, 30.55040872, 0.33242507, 28.88530466, 0.34729296,
      35.47615039, 0.12093154, 40.14314003, -0.03262612, 46.87397730,
      -0.27329111
    ),
    tolerance = 1e-5
  )
  expect_equal(CVgausslist$Coefficients[[1]]$std.error,
    c(
      5.518360, 6.980235, 5.656682, 7.483087, 5.301864, 6.706387,
      5.984982, 7.917387, 5.463479, 6.910816, 5.978424, 7.908712,
      5.922223, 7.491086, 5.036852, 6.663130
    ),
    tolerance = 1e-6
  )
  expect_equal(CVgausslist$Coefficients[[2]]$std.error,
    c(
      15.9672206, 0.5363914, 25.1784502, 0.8765252, 15.4721646,
      0.5481382, 16.8586576, 0.5565485, 19.4513547, 0.7299321,
      20.0765470, 0.6374769, 15.6973841, 0.5533890, 18.6950224,
      0.6087937
    ),
    tolerance = 1e-6
  )
  expect_equal(CVgausslist$Coefficients[[1]]$statistic,
    c(
      8.980123, -2.706053, 9.428378, -2.828601, 9.619258,
      -3.509093, 8.317114, -2.098055, 9.070329, -2.685002,
      8.326238, -2.342705, 8.611631, -2.509650, 10.588623,
      -3.839537
    ),
    tolerance = 1e-6
  )
  expect_equal(CVgausslist$Coefficients[[2]]$statistic,
    c(
      2.82724476, -0.47734352, 1.16486650, 0.48031677,
      2.79488905, -0.46538302, 1.81214955, 0.59729753,
      1.48500220, 0.47578805, 1.76704442, 0.18970340,
      2.55731400, -0.05895694, 2.50729719, -0.44890594
    ),
    tolerance = 1e-6
  )

  expect_equal(
    CVgausslist$`Warnings and Messages`[[1]],
    structure(list(
      `Fold Column` = character(0), Fold = integer(0), Function = character(0),
      Type = character(0), Message = character(0)
    ),
    row.names = integer(0L), class = c("tbl_df", "tbl", "data.frame")
    )
  )
})

test_that("preprocessing works with binomial models in cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  CVbinomlist_no_preprocessing <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_standardize <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "standardize",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_center <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "center",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_scale <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "scale",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_range <- cross_validate(
    dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "range",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  # NOTE for range version:
  # recipe::step_range : "When a new data point is outside of the ranges seen in the training set,
  # the new values are truncated at min or max."
  # Hence, we might get different predictions for that specific cv

  expect_error(
    xpectr::strip_msg(cross_validate(dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "standardization",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )),
  xpectr::strip(paste0(
    "1 assertions failed:\n * Variable 'preprocessing': Must be ",
    "element of set\n * {'standardize','scale','center','range'},",
    " but is 'standardization'."
  )),
  fixed = TRUE
  )

  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_center)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_scale)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_range)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    c(
      "Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity", "Pos Pred Value",
      "Neg Pred Value", "AUC", "Lower CI", "Upper CI", "Kappa", "MCC",
      "Detection Rate", "Detection Prevalence", "Prevalence", "Predictions",
      "ROC", "Confusion Matrix", "Results", "Coefficients", "Preprocess",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent"
    )
  )
  expect_equal(
    colnames(CVbinomlist_no_preprocessing),
    c(
      "Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity", "Pos Pred Value",
      "Neg Pred Value", "AUC", "Lower CI", "Upper CI", "Kappa", "MCC",
      "Detection Rate", "Detection Prevalence", "Prevalence", "Predictions",
      "ROC", "Confusion Matrix", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent"
    )
  )

  # No preprocessing
  expect_equal(CVbinomlist_no_preprocessing$`Balanced Accuracy`,
               c(0.7361111),
               tolerance = 1e-3)
  expect_equal(CVbinomlist_no_preprocessing$Coefficients[[1]]$estimate,
    c(
      3.19756037998759, -0.0678513281319974, 3.34308245247553, -0.0729927574874602,
      4.14788399490261, -0.0972715556624782, 2.49701032610746, -0.0536164806074169
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    c(
      0.773796146713643, 0.369523238509566, 0.0912557913458876, 0.892058194778159,
      0.736201417552938, 0.552827586629779, 0.830792778437875, 0.604289894202438,
      0.175457433384499, 0.931703421562706, 0.830792778437875, 0.514597941502579,
      0.92690981065346, 0.687473913088783, 0.586709578310841, 0.718679853099963,
      0.267467733090702, 0.0934653326479855, 0.8597682733868, 0.248845336158729,
      0.132050116263308, 0.650317136061844, 0.454175525786427, 0.156424602826963,
      0.844587191118717, 0.708583813132483, 0.587187598636642, 0.851495581323134,
      0.760714060837398, 0.708583813132483
    ),
    tolerance = 1e-6
  )

  # Standardize
  expect_equal(
    colnames(CVbinomlist_standardize$Preprocess[[1]]),
    c("Fold Column", "Fold", "Measure", "score")
  )
  expect_equal(CVbinomlist_standardize$Preprocess[[1]]$score,
    c(
      37.75, 18.6925932785759, 41.2380952380952, 19.7177705684612,
      36.2916666666667, 19.4276342104357, 40.2857142857143, 19.4220051929322
    ),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_standardize$Preprocess[[1]]$Measure,
    rep(c("Mean", "SD"), 4),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_standardize$`Balanced Accuracy`, c(0.7361111), tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$Coefficients[[1]]$estimate,
    c(
      0.636172743004689, -1.26831728018262, 0.333000167516465, -1.43925444529707,
      0.617737120651834, -1.88975620249066, 0.337032107351522, -1.041339564784
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_standardize$Predictions[[1]]$Prediction,
    c(
      0.773796146713643, 0.369523238509566, 0.0912557913458876, 0.892058194778159,
      0.736201417552938, 0.552827586629779, 0.830792778437875, 0.604289894202438,
      0.175457433384499, 0.931703421562706, 0.830792778437875, 0.514597941502579,
      0.92690981065346, 0.687473913088783, 0.586709578310841, 0.718679853099963,
      0.267467733090702, 0.0934653326479855, 0.8597682733868, 0.248845336158729,
      0.132050116263308, 0.650317136061844, 0.454175525786427, 0.156424602826963,
      0.844587191118717, 0.708583813132483, 0.587187598636642, 0.851495581323134,
      0.760714060837398, 0.708583813132483
    ),
    tolerance = 1e-6
  )

  # Scale
  expect_equal(
    colnames(CVbinomlist_scale$Preprocess[[1]]),
    c("Fold Column", "Fold", "Measure", "score")
  )
  expect_equal(CVbinomlist_scale$Preprocess[[1]]$score,
    c(
      18.6925932785759, 19.7177705684612,
      19.4276342104357, 19.4220051929322
    ),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_scale$Preprocess[[1]]$Measure,
    rep(c("SD"), 4),
    tolerance = 1e-3
  )

  expect_equal(CVbinomlist_scale$`Balanced Accuracy`, c(0.7361111), tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$Coefficients[[1]]$estimate,
    c(
      3.19756037998759, -1.26831728018262, 3.34308245247554, -1.43925444529707,
      4.14788399490261, -1.88975620249067, 2.49701032610746, -1.041339564784
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_scale$Predictions[[1]]$Prediction,
    c(
      0.773796146713643, 0.369523238509566, 0.0912557913458876, 0.892058194778159,
      0.736201417552938, 0.552827586629779, 0.830792778437875, 0.604289894202438,
      0.175457433384499, 0.931703421562706, 0.830792778437875, 0.514597941502579,
      0.92690981065346, 0.687473913088783, 0.586709578310841, 0.718679853099963,
      0.267467733090702, 0.0934653326479855, 0.8597682733868, 0.248845336158729,
      0.132050116263308, 0.650317136061844, 0.454175525786427, 0.156424602826963,
      0.844587191118717, 0.708583813132483, 0.587187598636642, 0.851495581323134,
      0.760714060837398, 0.708583813132483
    ),
    tolerance = 1e-6
  )

  # Center
  expect_equal(
    colnames(CVbinomlist_center$Preprocess[[1]]),
    c("Fold Column", "Fold", "Measure", "score")
  )
  expect_equal(CVbinomlist_center$Preprocess[[1]]$score,
    c(
      37.75, 41.2380952380952,
      36.2916666666667, 40.2857142857143
    ),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_center$Preprocess[[1]]$Measure,
    rep(c("Mean"), 4),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_center$`Balanced Accuracy`, c(0.7361111), tolerance = 1e-3)
  expect_equal(CVbinomlist_center$Coefficients[[1]]$estimate,
    c(
      0.636172743004689, -0.0678513281319975, 0.333000167516464,
      -0.0729927574874603, 0.617737120651835, -0.0972715556624782,
      0.337032107351522, -0.0536164806074169
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_center$Predictions[[1]]$Prediction,
    c(
      0.773796146713643, 0.369523238509566, 0.0912557913458876, 0.892058194778159,
      0.736201417552938, 0.552827586629779, 0.830792778437875, 0.604289894202438,
      0.175457433384499, 0.931703421562706, 0.830792778437875, 0.514597941502579,
      0.92690981065346, 0.687473913088783, 0.586709578310841, 0.718679853099963,
      0.267467733090702, 0.0934653326479855, 0.8597682733868, 0.248845336158729,
      0.132050116263308, 0.650317136061844, 0.454175525786427, 0.156424602826963,
      0.844587191118717, 0.708583813132483, 0.587187598636642, 0.851495581323134,
      0.760714060837398, 0.708583813132483
    ),
    tolerance = 1e-6
  )

  # Range
  expect_equal(
    colnames(CVbinomlist_range$Preprocess[[1]]),
    c("Fold Column", "Fold", "Measure", "score")
  )
  expect_equal(CVbinomlist_range$Preprocess[[1]]$score,
    c(10, 78, 14, 81, 10, 81, 10, 81),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_range$Preprocess[[1]]$Measure,
    rep(c("Min", "Max"), 4),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_range$`Balanced Accuracy`, c(0.736111111111111), tolerance = 1e-3)
  expect_equal(CVbinomlist_range$Coefficients[[1]]$estimate,
    c(
      2.51904709866762, -4.61389031297583, 2.3211838476511, -4.89051475165984,
      3.17516843827782, -6.90628045203595, 1.96084552003329, -3.8067701231266
    ),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_range$Predictions[[1]]$Prediction,
    c(
      0.773796146713643, 0.369523238509566, 0.10959904472987, 0.892058194778159,
      0.736201417552938, 0.552827586629779, 0.830792778437875, 0.604289894202438,
      0.175457433384499, 0.910616345870464, 0.830792778437875, 0.514597941502578,
      0.910616345870464, 0.687473913088783, 0.586709578310841, 0.718679853099963,
      0.267467733090702, 0.0934653326479854, 0.8597682733868, 0.248845336158729,
      0.132050116263308, 0.650317136061844, 0.454175525786427, 0.156424602826963,
      0.844587191118717, 0.708583813132483, 0.587187598636643, 0.851495581323134,
      0.760714060837398, 0.708583813132483
    ),
    tolerance = 1e-6
  )

  expect_equal(
    CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_standardize$Predictions[[1]]$Prediction
  )
  expect_equal(
    CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_scale$Predictions[[1]]$Prediction
  )
  expect_equal(
    CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_center$Predictions[[1]]$Prediction
  )
  # Note: Ranged version is not identical to the non-preprocessed version
  # as recipes::step_range truncates values in the test set that lie below or above
  # the min/max in the training set.


  # get_split <- function(fold = 2){
  #   train_data <- dat %>%
  #     dplyr::filter(.data$.folds != fold)
  #   test_data <- dat %>%
  #     dplyr::filter(.data$.folds == fold)
  #   preprocessed_split <- preprocess_functions("range")(
  #     train_data = train_data,
  #     test_data = test_data,
  #     formula = as.formula("diagnosis ~ age"),
  #     NULL)
  #   list("no_preprocessing" = list("train" = train_data, "test" = test_data),
  #        "ranged" = preprocessed_split)
  # }
  #
  # train_test <- get_split(2)
  #
  # glm_no_preprocessing <- glm(diagnosis ~ age, data = train_test$no_preprocessing$train)
  # glm_ranged <- glm(diagnosis ~ age, data = train_test$ranged$train)
  # predictions_no_preprocessing <- predict(glm_no_preprocessing, newdata = train_test$no_preprocessing$test)
  # predictions_ranged <- predict(glm_ranged, newdata = train_test$ranged$test)


  expect_equal(CVbinomlist_no_preprocessing$AUC, c(0.7615741), tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$AUC, c(0.7615741), tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$AUC, c(0.7615741), tolerance = 1e-3)
  expect_equal(CVbinomlist_center$AUC, c(0.7615741), tolerance = 1e-3)
  expect_equal(CVbinomlist_range$AUC, c(0.7615741), tolerance = 1e-3)

  expect_equal(CVbinomlist_no_preprocessing$MCC, c(0.5048268), tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$MCC, c(0.5048268), tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$MCC, c(0.5048268), tolerance = 1e-3)
  expect_equal(CVbinomlist_center$MCC, c(0.5048268), tolerance = 1e-3)
  expect_equal(CVbinomlist_range$MCC, c(0.504826790279024), tolerance = 1e-3) # Why is this different?

})

test_that("preprocessing works with binomial mixed models in cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  ) %>%
    dplyr::mutate(session = as.factor(session)) %>%
    dplyr::slice(rep(1:30, each = 3))

  CVbinomlist_no_preprocessing <- cross_validate(
    dat,
    formulas = c("diagnosis~score+(1|session)"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_standardize <- cross_validate(
    dat,
    formulas = c("diagnosis~score+(1|session)"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "standardize",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_center <- cross_validate(
    dat,
    formulas = c("diagnosis~score+(1|session)"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "center",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_scale <- cross_validate(
    dat,
    formulas = c("diagnosis~score+(1|session)"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "scale",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  CVbinomlist_range <- cross_validate(
    dat,
    formulas = c("diagnosis~score+(1|session)"),
    fold_cols = ".folds", family = "binomial",
    preprocessing = "range",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )

  # NOTE for range version:
  # recipe::step_range : "When a new data point is outside of the ranges seen in the training set,
  # the new values are truncated at min or max."
  # Hence, we might get different predictions for that specific cv

  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_center)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_scale)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    colnames(CVbinomlist_range)
  )
  expect_equal(
    colnames(CVbinomlist_standardize),
    c(
      "Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity", "Pos Pred Value",
      "Neg Pred Value", "AUC", "Lower CI", "Upper CI", "Kappa", "MCC",
      "Detection Rate", "Detection Prevalence", "Prevalence", "Predictions",
      "ROC", "Confusion Matrix", "Results", "Coefficients", "Preprocess",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages",  "Process",
      "Dependent", "Random"
    )
  )
  expect_equal(
    colnames(CVbinomlist_no_preprocessing),
    c(
      "Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity", "Pos Pred Value",
      "Neg Pred Value", "AUC", "Lower CI", "Upper CI", "Kappa", "MCC",
      "Detection Rate", "Detection Prevalence", "Prevalence", "Predictions",
      "ROC", "Confusion Matrix", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process",
      "Dependent", "Random"
    )
  )

  expect_equal(CVbinomlist_standardize$Preprocess[[1]]$score,
    c(
      39.3809523809524, 18.3632070530147, 38.3333333333333, 19.6682157217532,
      38.5555555555556, 19.3494836035298
    ),
    tolerance = 1e-5
  )
  expect_equal(CVbinomlist_scale$Preprocess[[1]]$score,
    c(18.3632070530147, 19.6682157217532, 19.3494836035298),
    tolerance = 1e-5
  )
  expect_equal(CVbinomlist_center$Preprocess[[1]]$score,
    c(39.3809523809524, 38.3333333333333, 38.5555555555556),
    tolerance = 1e-5
  )
  expect_equal(CVbinomlist_range$Preprocess[[1]]$score,
    c(11, 78, 10, 81, 10, 81),
    tolerance = 1e-5
  )

  # No preprocessing
  expect_equal(CVbinomlist_no_preprocessing$`Balanced Accuracy`, c(0.8611111), tolerance = 1e-3)
  expect_equal(CVbinomlist_no_preprocessing$Coefficients[[1]]$estimate,
               c(8.3579464874966, -0.193892587463713, 2.81255078275366, 55.2463242203065,
               -1.33929033061194, 53.1381999808447,  7.64561163963771, -0.168532399577982,
               2.68897250187443),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    c(
      0.324456114093013, 0.324456114093013, 0.324456114093013, 0.109287077948817,
      0.109287077948817, 0.109287077948817, 0.0175510977497869, 0.0175510977497869,
      0.0175510977497869, 0.950291428617545, 0.950291428617545, 0.950291428617545,
      0.980403748861395, 0.980403748861395, 0.980403748861395, 0.950508742035668,
      0.950508742035668, 0.950508742035668, 0.856588042939281, 0.856588042939281,
      0.856588042939281, 0.913842836648199, 0.913842836648199, 0.913842836648199,
      0.958873004064922, 0.958873004064922, 0.958873004064922, 0.681645372889445,
      0.681645372889445, 0.681645372889445, 0.838249513761936, 0.838249513761936,
      0.838249513761936, 1.44502905719389e-10, 1.44502905719389e-10,
      1.44502905719389e-10, 0.681645372889445, 0.681645372889445, 0.681645372889445,
      3.72801881555979e-08, 3.72801881555979e-08, 3.72801881555979e-08,
      1.1697916866856e-07, 1.1697916866856e-07, 1.1697916866856e-07,
      0.999999987178186, 0.999999987178186, 0.999999987178186, 0.999761693013033,
      0.999761693013033, 0.999761693013033, 0.999994784274243, 0.999994784274243,
      0.999994784274243, 0.166242005117161, 0.166242005117161, 0.166242005117161,
      0.373839375611282, 0.373839375611282, 0.373839375611282, 0.0788323931366848,
      0.0788323931366848, 0.0788323931366848, 0.218328494540638, 0.218328494540638,
      0.218328494540638, 0.264759089772758, 0.264759089772758, 0.264759089772758,
      0.392711382027383, 0.392711382027383, 0.392711382027383, 0.85297773335794,
      0.85297773335794, 0.85297773335794, 0.945569734698759, 0.945569734698759,
      0.945569734698759, 0.981033143319363, 0.981033143319363, 0.981033143319363,
      0.872881689544893, 0.872881689544893, 0.872881689544893, 0.975814737517958,
      0.975814737517958, 0.975814737517958, 0.996428661773137, 0.996428661773137,
      0.996428661773137
    ),
    tolerance = 1e-6
  )


  # Standardize
  expect_equal(CVbinomlist_standardize$`Balanced Accuracy`, c(0.8611111), tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$Coefficients[[1]]$estimate,
               c(0.722270851420555, -3.56049245923584, 2.81254774249312,
               3.90685791812926, -26.3414459230668, 53.1381599664327, 1.14775661484688,
               -3.2610246203655, 2.68898875394687),
    tolerance = 1e-4
  )
  expect_equal(CVbinomlist_standardize$Predictions[[1]]$Prediction,
    c(
      0.324456114093013, 0.324456114093013, 0.324456114093013, 0.109287077948817,
      0.109287077948817, 0.109287077948817, 0.0175510977497869, 0.0175510977497869,
      0.0175510977497869, 0.950291428617545, 0.950291428617545, 0.950291428617545,
      0.980403748861395, 0.980403748861395, 0.980403748861395, 0.950508742035668,
      0.950508742035668, 0.950508742035668, 0.856588042939281, 0.856588042939281,
      0.856588042939281, 0.913842836648199, 0.913842836648199, 0.913842836648199,
      0.958873004064922, 0.958873004064922, 0.958873004064922, 0.681645372889445,
      0.681645372889445, 0.681645372889445, 0.838249513761936, 0.838249513761936,
      0.838249513761936, 1.44502905719389e-10, 1.44502905719389e-10,
      1.44502905719389e-10, 0.681645372889445, 0.681645372889445, 0.681645372889445,
      3.72801881555979e-08, 3.72801881555979e-08, 3.72801881555979e-08,
      1.1697916866856e-07, 1.1697916866856e-07, 1.1697916866856e-07,
      0.999999987178186, 0.999999987178186, 0.999999987178186, 0.999761693013033,
      0.999761693013033, 0.999761693013033, 0.999994784274243, 0.999994784274243,
      0.999994784274243, 0.166242005117161, 0.166242005117161, 0.166242005117161,
      0.373839375611282, 0.373839375611282, 0.373839375611282, 0.0788323931366848,
      0.0788323931366848, 0.0788323931366848, 0.218328494540638, 0.218328494540638,
      0.218328494540638, 0.264759089772758, 0.264759089772758, 0.264759089772758,
      0.392711382027383, 0.392711382027383, 0.392711382027383, 0.85297773335794,
      0.85297773335794, 0.85297773335794, 0.945569734698759, 0.945569734698759,
      0.945569734698759, 0.981033143319363, 0.981033143319363, 0.981033143319363,
      0.872881689544893, 0.872881689544893, 0.872881689544893, 0.975814737517958,
      0.975814737517958, 0.975814737517958, 0.996428661773137, 0.996428661773137,
      0.996428661773137
    ),
    tolerance = 1e-6
  )

  # Scale
  expect_equal(CVbinomlist_scale$`Balanced Accuracy`, c(0.8611111), tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$Coefficients[[1]]$estimate,
               c(8.35794641518746, -3.56048970107377, 2.81255070401071, 55.2462927019219,
               -26.3414369433445, 53.1381485501194, 7.64561182809043, -3.26101498104044,
               2.6889727017439),
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_scale$Predictions[[1]]$Prediction,
    c(
      0.324456114093013, 0.324456114093013, 0.324456114093013, 0.109287077948817,
      0.109287077948817, 0.109287077948817, 0.0175510977497869, 0.0175510977497869,
      0.0175510977497869, 0.950291428617545, 0.950291428617545, 0.950291428617545,
      0.980403748861395, 0.980403748861395, 0.980403748861395, 0.950508742035668,
      0.950508742035668, 0.950508742035668, 0.856588042939281, 0.856588042939281,
      0.856588042939281, 0.913842836648199, 0.913842836648199, 0.913842836648199,
      0.958873004064922, 0.958873004064922, 0.958873004064922, 0.681645372889445,
      0.681645372889445, 0.681645372889445, 0.838249513761936, 0.838249513761936,
      0.838249513761936, 1.44502905719389e-10, 1.44502905719389e-10,
      1.44502905719389e-10, 0.681645372889445, 0.681645372889445, 0.681645372889445,
      3.72801881555979e-08, 3.72801881555979e-08, 3.72801881555979e-08,
      1.1697916866856e-07, 1.1697916866856e-07, 1.1697916866856e-07,
      0.999999987178186, 0.999999987178186, 0.999999987178186, 0.999761693013033,
      0.999761693013033, 0.999761693013033, 0.999994784274243, 0.999994784274243,
      0.999994784274243, 0.166242005117161, 0.166242005117161, 0.166242005117161,
      0.373839375611282, 0.373839375611282, 0.373839375611282, 0.0788323931366848,
      0.0788323931366848, 0.0788323931366848, 0.218328494540638, 0.218328494540638,
      0.218328494540638, 0.264759089772758, 0.264759089772758, 0.264759089772758,
      0.392711382027383, 0.392711382027383, 0.392711382027383, 0.85297773335794,
      0.85297773335794, 0.85297773335794, 0.945569734698759, 0.945569734698759,
      0.945569734698759, 0.981033143319363, 0.981033143319363, 0.981033143319363,
      0.872881689544893, 0.872881689544893, 0.872881689544893, 0.975814737517958,
      0.975814737517958, 0.975814737517958, 0.996428661773137, 0.996428661773137,
      0.996428661773137
    ),
    tolerance = 1e-6
  )

  # Center
  expect_equal(CVbinomlist_center$`Balanced Accuracy`, c(0.8611111), tolerance = 1e-3)
  expect_equal(CVbinomlist_center$Coefficients[[1]]$estimate,
               c(0.722270851499559, -0.193892736123874, 2.81254774327143,
               3.90685679177423, -1.33929005889398, 53.1381319242869, 1.14775662092732,
               -0.168532902834402, 2.6889888039097),
    tolerance = 1e-4
  )
  expect_equal(CVbinomlist_center$Predictions[[1]]$Prediction,
    c(
      0.324456114093013, 0.324456114093013, 0.324456114093013, 0.109287077948817,
      0.109287077948817, 0.109287077948817, 0.0175510977497869, 0.0175510977497869,
      0.0175510977497869, 0.950291428617545, 0.950291428617545, 0.950291428617545,
      0.980403748861395, 0.980403748861395, 0.980403748861395, 0.950508742035668,
      0.950508742035668, 0.950508742035668, 0.856588042939281, 0.856588042939281,
      0.856588042939281, 0.913842836648199, 0.913842836648199, 0.913842836648199,
      0.958873004064922, 0.958873004064922, 0.958873004064922, 0.681645372889445,
      0.681645372889445, 0.681645372889445, 0.838249513761936, 0.838249513761936,
      0.838249513761936, 1.44502905719389e-10, 1.44502905719389e-10,
      1.44502905719389e-10, 0.681645372889445, 0.681645372889445, 0.681645372889445,
      3.72801881555979e-08, 3.72801881555979e-08, 3.72801881555979e-08,
      1.1697916866856e-07, 1.1697916866856e-07, 1.1697916866856e-07,
      0.999999987178186, 0.999999987178186, 0.999999987178186, 0.999761693013033,
      0.999761693013033, 0.999761693013033, 0.999994784274243, 0.999994784274243,
      0.999994784274243, 0.166242005117161, 0.166242005117161, 0.166242005117161,
      0.373839375611282, 0.373839375611282, 0.373839375611282, 0.0788323931366848,
      0.0788323931366848, 0.0788323931366848, 0.218328494540638, 0.218328494540638,
      0.218328494540638, 0.264759089772758, 0.264759089772758, 0.264759089772758,
      0.392711382027383, 0.392711382027383, 0.392711382027383, 0.85297773335794,
      0.85297773335794, 0.85297773335794, 0.945569734698759, 0.945569734698759,
      0.945569734698759, 0.981033143319363, 0.981033143319363, 0.981033143319363,
      0.872881689544893, 0.872881689544893, 0.872881689544893, 0.975814737517958,
      0.975814737517958, 0.975814737517958, 0.996428661773137, 0.996428661773137,
      0.996428661773137
    ),
    tolerance = 1e-3
  )

  # Range
  expect_equal(CVbinomlist_range$`Balanced Accuracy`, c(0.8611111), tolerance = 1e-3)
  expect_equal(CVbinomlist_range$Coefficients[[1]]$estimate,
               c(6.22513139971877, -12.9908044682518, 2.81255450568456, 41.8533992397597,
               -95.0895775189958, 53.1381034729376, 5.9602950441032, -11.9658077360469,
               2.68897631778515),
    tolerance = 1e-3
  )
  expect_equal(CVbinomlist_range$Predictions[[1]]$Prediction,
    c(
      0.324456042975257, 0.324456042975257, 0.324456042975257, 0.109287076272275,
      0.109287076272275, 0.109287076272275, 0.030970688236543, 0.030970688236543,
      0.030970688236543, 0.94029055481341, 0.94029055481341, 0.94029055481341,
      0.980403758381411, 0.980403758381411, 0.980403758381411, 0.950508787930829,
      0.950508787930829, 0.950508787930829, 0.856588029494522, 0.856588029494522,
      0.856588029494522, 0.913842865244269, 0.913842865244269, 0.913842865244269,
      0.958873043191456, 0.958873043191456, 0.958873043191456, 0.681645466159717,
      0.681645466159717, 0.681645466159717, 0.838248709938151, 0.838248709938151,
      0.838248709938151, 1.44504001024224e-10, 1.44504001024224e-10,
      1.44504001024224e-10, 0.681645466159717, 0.681645466159717, 0.681645466159717,
      3.72802314439356e-08, 3.72802314439356e-08, 3.72802314439356e-08,
      1.16979759155026e-07, 1.16979759155026e-07, 1.16979759155026e-07,
      0.999999987178107, 0.999999987178107, 0.999999987178107, 0.99976169099732,
      0.99976169099732, 0.99976169099732, 0.999994784245105, 0.999994784245105,
      0.999994784245105, 0.166241799664311, 0.166241799664311, 0.166241799664311,
      0.373839317386791, 0.373839317386791, 0.373839317386791, 0.078832328751711,
      0.078832328751711, 0.078832328751711, 0.218328276983336, 0.218328276983336,
      0.218328276983336, 0.264758980767326, 0.264758980767326, 0.264758980767326,
      0.392711467485616, 0.392711467485616, 0.392711467485616, 0.852977807679943,
      0.852977807679943, 0.852977807679943, 0.945569828688762, 0.945569828688762,
      0.945569828688762, 0.981033200177857, 0.981033200177857, 0.981033200177857,
      0.872881766816275, 0.872881766816275, 0.872881766816275, 0.975814792859071,
      0.975814792859071, 0.975814792859071, 0.996428676339139, 0.996428676339139,
      0.996428676339139
    ),
    tolerance = 1e-3
  )


  expect_equal(CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_standardize$Predictions[[1]]$Prediction,
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_scale$Predictions[[1]]$Prediction,
    tolerance = 1e-6
  )
  expect_equal(CVbinomlist_no_preprocessing$Predictions[[1]]$Prediction,
    CVbinomlist_center$Predictions[[1]]$Prediction,
    tolerance = 1e-6
  )
  # Note: Ranged version is not identical to the non-preprocessed version
  # as recipes::step_range truncates values in the test set that lie below or above
  # the min/max in the training set.

  expect_equal(CVbinomlist_no_preprocessing$AUC, 0.8912037, tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$AUC, 0.8912037, tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$AUC, 0.8912037, tolerance = 1e-3)
  expect_equal(CVbinomlist_center$AUC, 0.8912037, tolerance = 1e-3)
  expect_equal(CVbinomlist_range$AUC, 0.8912037, tolerance = 1e-3)

  expect_equal(CVbinomlist_no_preprocessing$MCC, 0.7222222, tolerance = 1e-3)
  expect_equal(CVbinomlist_standardize$MCC, 0.7222222, tolerance = 1e-3)
  expect_equal(CVbinomlist_scale$MCC, 0.7222222, tolerance = 1e-3)
  expect_equal(CVbinomlist_center$MCC, 0.7222222, tolerance = 1e-3)
  expect_equal(CVbinomlist_range$MCC, 0.7222222, tolerance = 1e-3) # Why is this different?
})

test_that("that singular fit messages are caught, counted and messaged about in cross_validate()", {

  testthat::skip_on_cran()

  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  ## Testing 'cross_validate( dat, formulas = c("diagnosis...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing side effects
  # Assigning side effects
  side_effects_12059 <- xpectr::capture_side_effects(cross_validate(
      dat,
      formulas = c("diagnosis~score+(1|participant)+(1|session)"),
      family = "binomial",
      REML = FALSE,
      verbose = FALSE,
      positive = 2
    ), reset_seed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_12059[['warnings']], remove_numbers = TRUE),
    xpectr::strip(c("\n---\ncross_validate(): Model failed to converge with max|grad| = 0.0281933 (tol = 0.002, component 1)\nNote: Convergence Warning\nFor:\nFormula: diagnosis~score+(1|participant)+(1|session)\nFold column: .folds\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : FALSE, family : binomial, is_special_fn : TRUE",
      "cross_validate(): predictions contained NA."), remove_numbers = TRUE),
    fixed = TRUE)
  # TODO Add confidence intervals for random effects messages here
  expect_equal(
    xpectr::strip(side_effects_12059[['messages']]),
    xpectr::strip(c("---\ncross_validate(): boundary (singular) fit: see help('isSingular')\nNote: Boundary (Singular) Fit Message\nFor:\nFormula: diagnosis~score+(1|participant)+(1|session)\nFold column: .folds\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : FALSE, family : binomial, is_special_fn : TRUE\n\n",
        "---\ncross_validate(): Confidence intervals for random effect parameters are currently not supported for\n  multiple grouping variables.\nFor:\nFold column: .folds\nFold: 1\n\n",
        "---\ncross_validate(): boundary (singular) fit: see help('isSingular')\nNote: Boundary (Singular) Fit Message\nFor:\nFormula: diagnosis~score+(1|participant)+(1|session)\nFold column: .folds\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : FALSE, family : binomial, is_special_fn : TRUE\n\n",
        "---\ncross_validate(): Confidence intervals for random effect parameters are currently not supported for\n  multiple grouping variables.\nFor:\nFold column: .folds\nFold: 2\n\n",
        "---\ncross_validate(): boundary (singular) fit: see help('isSingular')\nNote: Boundary (Singular) Fit Message\nFor:\nFormula: diagnosis~score+(1|participant)+(1|session)\nFold column: .folds\nFold: 4\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : FALSE, family : binomial, is_special_fn : TRUE\n\n",
        "---\ncross_validate(): Confidence intervals for random effect parameters are currently not supported for\n  multiple grouping variables.\nFor:\nFold column: .folds\nFold: 4\n\n"
    )),
    fixed = TRUE)
  # Assigning output
  output_12059 <- xpectr::suppress_mw(cross_validate(
      dat,
      formulas = c("diagnosis~score+(1|participant)+(1|session)"),
      family = "binomial",
      REML = FALSE,
      verbose = FALSE,
      positive = 2
    ))
  # Testing class
  expect_equal(
    class(output_12059),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    output_12059[["Fixed"]],
    "score",
    fixed = TRUE)
  expect_equal(
    output_12059[["Balanced Accuracy"]],
    NA)
  expect_equal(
    output_12059[["F1"]],
    NA)
  expect_equal(
    output_12059[["Sensitivity"]],
    NA)
  expect_equal(
    output_12059[["Specificity"]],
    NA)
  expect_equal(
    output_12059[["Pos Pred Value"]],
    NA)
  expect_equal(
    output_12059[["Neg Pred Value"]],
    NA)
  expect_equal(
    output_12059[["AUC"]],
    NA)
  expect_equal(
    output_12059[["Lower CI"]],
    NA)
  expect_equal(
    output_12059[["Upper CI"]],
    NA)
  expect_equal(
    output_12059[["Kappa"]],
    NA)
  expect_equal(
    output_12059[["MCC"]],
    NA)
  expect_equal(
    output_12059[["Detection Rate"]],
    NA)
  expect_equal(
    output_12059[["Detection Prevalence"]],
    NA)
  expect_equal(
    output_12059[["Prevalence"]],
    NA)
  expect_equal(
    output_12059[["Predictions"]],
    NA)
  expect_equal(
    output_12059[["ROC"]],
    NA)
  expect_equal(
    output_12059[["Folds"]],
    4,
    tolerance = 1e-4)
  expect_equal(
    output_12059[["Fold Columns"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    output_12059[["Convergence Warnings"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    output_12059[["Singular Fit Messages"]],
    3,
    tolerance = 1e-4)
  expect_equal(
    output_12059[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_12059[["Dependent"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    output_12059[["Random"]],
    "(1|participant)+(1|session)",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(output_12059),
    c("Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity",
      "Pos Pred Value", "Neg Pred Value", "AUC", "Lower CI", "Upper CI",
      "Kappa", "MCC", "Detection Rate", "Detection Prevalence", "Prevalence",
      "Predictions", "ROC", "Results", "Coefficients", "Folds", "Fold Columns",
      "Convergence Warnings", "Singular Fit Messages", "Other Warnings",
      "Warnings and Messages", "Process", "Dependent", "Random"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(output_12059),
    c("character", "logical", "logical", "logical", "logical", "logical",
      "logical", "logical", "logical", "logical", "logical", "logical",
      "logical", "logical", "logical", "logical", "logical", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "logical", "character", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(output_12059),
    c("character", "logical", "logical", "logical", "logical", "logical",
      "logical", "logical", "logical", "logical", "logical", "logical",
      "logical", "logical", "logical", "logical", "logical", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "logical", "character", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(output_12059),
    c(1L, 28L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(output_12059)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'cross_validate( dat, formulas = c("diagnosis...'     ####


  xpectr::suppress_mw(
      CVbinom <- cross_validate(
        dat,
        formulas = c("diagnosis~score+(1|participant)+(1|session)"),
        family = "binomial",
        REML = FALSE,
        verbose = FALSE,
        positive = 2
      ))

  expect_equal(CVbinom$`Singular Fit Messages`, 3)

  # Can't expect the same warnings on mac and ubuntu
  # so we just check that the singular fit message is there
  expect_true(
    "boundary (singular) fit: see help('isSingular')\n" %in%
      CVbinom$`Warnings and Messages`[[1]]$Message
  )
})

test_that("the expected errors are thrown by cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    num_fold_cols = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  )
  dat[[".folds_3"]] <- as.character(dat[[".folds_3"]])

  expect_error(cross_validate(dat,
    formulas = c("diagnosis~score", "diagnosis~age"),
    fold_cols = paste0(".folds_", 1:3), family = "binomial",
    REML = FALSE, verbose = FALSE,
    positive = 1
  ),
  "At least one of the fold columns is not a factor.",
  fixed = TRUE
  )
  expect_error(
    xpectr::strip_msg(cross_validate(dat,
    formulas = c("diagnosis~score", "diagnosis~age"),
    fold_cols = paste0(".folds_", 1), family = "fdsfs",
    REML = FALSE, verbose = FALSE,
    positive = 1
  )),
  xpectr::strip(paste0(
    "Assertion on 'family' failed: Must be element",
    " of set\n * {'gaussian','binomial'}, but is 'f",
    "dsfs'."
  )),
  fixed = TRUE
  )

  ## Testing 'cross_validate( dat, formulas = "diagnosis~s...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing side effects
  # Assigning side effects
  side_effects_14820 <- xpectr::capture_side_effects(cross_validate(
      dat,
      formulas = "diagnosis~score",
      fold_cols = ".folds_1", family = "binomial",
      link = "probit", REML = FALSE, verbose = FALSE,
      positive = 1
    ), reset_seed = TRUE)
  expect_match(
    xpectr::strip(side_effects_14820[['error']], remove_spaces = TRUE),
    xpectr::strip("`link` argument of `cross_validate()`", remove_spaces = TRUE),
    fixed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_14820[['error_class']]),
    xpectr::strip(c("lifecycle_error_deprecated", "defunctError", "rlang_error",
      "error", "condition")),
    fixed = TRUE)
  ## Finished testing 'cross_validate( dat, formulas = "diagnosis~s...'     ####

  # **** NOTE!! ****: The two below will only throw warnings once per session
  # So test_this() won't necessarily see them, while the overall testing should!
  # expect_warning(cross_validate(dat,
  #   models = "diagnosis~score",
  #   fold_cols = ".folds_1", family = "binomial",
  #   REML = FALSE, verbose = FALSE,
  #   positive = 1
  # ),
  # ".*`models` argument of `cross_validate\\(\\)` is deprecated.*",
  # fixed = FALSE, perl=TRUE
  # )
  # expect_warning(cross_validate(dat,
  #   formulas = "diagnosis~score",
  #   fold_cols = ".folds_1", family = "binomial",
  #   REML = FALSE, model_verbose = FALSE,
  #   positive = 1
  # ),
  # ".*`model_verbose` argument of `cross_validate\\(\\)` is deprecated.*",
  # fixed = FALSE, perl=TRUE
  # )

  # With rm_nc, the non-converged model is removed
  # and we only have an empty data frame left
  # Note, this may fail on ubuntu?
  xpectr::set_test_seed(2)
  expect_identical(
    xpectr::suppress_mw(
      cross_validate(
        dplyr::sample_frac(dat, 0.2),
        formulas = c("diagnosis~score*age+(1|session)"),
        family = "gaussian",
        fold_cols = ".folds_1",
        REML = FALSE,
        verbose = FALSE,
        control = lme4::lmerControl(
          optimizer = "bobyqa",
          optCtrl = list(maxfun = 10)
        ),
        rm_nc = TRUE
      )
    ),
    structure(list(
      Fixed = character(0), RMSE = numeric(0), MAE = numeric(0),
      `NRMSE(IQR)` = numeric(0), RRSE = numeric(0), RAE = numeric(0),
      RMSLE = numeric(0), AIC = numeric(0), AICc = numeric(0), BIC = numeric(0),
      Predictions = logical(0), Results = list(), Coefficients = list(),
      Folds = integer(0), `Fold Columns` = integer(0), `Convergence Warnings` = integer(0),
      `Singular Fit Messages` = integer(0), `Other Warnings` = integer(0),
      `Warnings and Messages` = list(), Process = list(),
      Dependent = character(0),
      Random = character(0)
    ), class = c("tbl_df", "tbl", "data.frame"), row.names = c(NA, 0L))
  )
})

test_that("verbose reports the correct model functions in cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    num_fold_cols = 3,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  if (!is_tibble_v2() && is_newer_lme4()){

    # Test the list of verbose messages
    # glm()

    ## Testing 'cross_validate(dat, formulas = c("diagnosi...'              ####
    ## Initially generated by xpectr
    xpectr::set_test_seed(42)
    # Testing side effects
    # Assigning side effects
    side_effects_14820 <- xpectr::capture_side_effects(cross_validate(dat,
                       formulas = c("diagnosis~score"),
                       fold_cols = paste0(".folds_", 1), family = "binomial",
                       REML = FALSE, verbose = TRUE,
                       positive = 1
        ), reset_seed = TRUE)
    expect_equal(
      xpectr::strip(side_effects_14820[['warnings']]),
      xpectr::strip(character(0)),
      fixed = TRUE)
    expect_equal(
      xpectr::strip(side_effects_14820[['messages']]),
      xpectr::strip(c("Will cross-validate 1 models. This requires fitting 4 model instances.\n",
        "\n---\ncross_validate(): cross_validate(): Used glm() to fit the model.'\nFor:\nFormula: diagnosis~score\nFold column: .folds_1\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used glm() to fit the model.'\nFor:\nFormula: diagnosis~score\nFold column: .folds_1\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used glm() to fit the model.'\nFor:\nFormula: diagnosis~score\nFold column: .folds_1\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used glm() to fit the model.'\nFor:\nFormula: diagnosis~score\nFold column: .folds_1\nFold: 4\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n")),
      fixed = TRUE)
    # Assigning output
    output_14820 <- xpectr::suppress_mw(cross_validate(dat,
                       formulas = c("diagnosis~score"),
                       fold_cols = paste0(".folds_", 1), family = "binomial",
                       REML = FALSE, verbose = TRUE,
                       positive = 1
        ))
    # Testing class
    expect_equal(
      class(output_14820),
      c("tbl_df", "tbl", "data.frame"),
      fixed = TRUE)
    # Testing column values
    expect_equal(
      output_14820[["Fixed"]],
      "score",
      fixed = TRUE)
    expect_equal(
      output_14820[["Balanced Accuracy"]],
      0.73611,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["F1"]],
      0.66667,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Sensitivity"]],
      0.58333,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Specificity"]],
      0.88889,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Pos Pred Value"]],
      0.77778,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Neg Pred Value"]],
      0.7619,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["AUC"]],
      0.76157,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Lower CI"]],
      0.58512,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Upper CI"]],
      0.93803,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Kappa"]],
      0.49275,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["MCC"]],
      0.50483,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Detection Rate"]],
      0.23333,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Detection Prevalence"]],
      0.3,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Prevalence"]],
      0.4,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Folds"]],
      4,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Fold Columns"]],
      1,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Convergence Warnings"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Singular Fit Messages"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Other Warnings"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_14820[["Dependent"]],
      "diagnosis",
      fixed = TRUE)
    # Testing column names
    expect_equal(
      names(output_14820),
      c("Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity",
        "Pos Pred Value", "Neg Pred Value", "AUC", "Lower CI", "Upper CI",
        "Kappa", "MCC", "Detection Rate", "Detection Prevalence", "Prevalence",
        "Predictions", "ROC", "Confusion Matrix", "Results", "Coefficients",
        "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
        "Other Warnings", "Warnings and Messages", "Process", "Dependent"),
      fixed = TRUE)
    # Testing column classes
    expect_equal(
      xpectr::element_classes(output_14820),
      c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
        "numeric", "numeric", "numeric", "numeric", "numeric", "numeric",
        "numeric", "numeric", "numeric", "list", "list", "list", "list",
        "list", "integer", "integer", "integer", "integer", "integer",
        "list", "list", "character"),
      fixed = TRUE)
    # Testing column types
    expect_equal(
      xpectr::element_types(output_14820),
      c("character", "double", "double", "double", "double", "double",
        "double", "double", "double", "double", "double", "double",
        "double", "double", "double", "list", "list", "list", "list",
        "list", "integer", "integer", "integer", "integer", "integer",
        "list", "list", "character"),
      fixed = TRUE)
    # Testing dimensions
    expect_equal(
      dim(output_14820),
      c(1L, 28L))
    # Testing group keys
    expect_equal(
      colnames(dplyr::group_keys(output_14820)),
      character(0),
      fixed = TRUE)
    ## Finished testing 'cross_validate(dat, formulas = c("diagnosi...'     ####


    # glmer

    ## Testing 'cross_validate(dat, formulas = c("diagnosi...'              ####
    ## Initially generated by xpectr
    xpectr::set_test_seed(42)
    # Testing side effects
    # Assigning side effects
    side_effects_19148 <- xpectr::capture_side_effects(cross_validate(dat,
                       formulas = c("diagnosis~score+(1|session)"),
                       fold_cols = paste0(".folds_", 1), family = "binomial",
                       REML = FALSE, verbose = TRUE,
                       positive = 1
        ), reset_seed = TRUE)
    expect_equal(
      xpectr::strip(side_effects_19148[['warnings']]),
      xpectr::strip(character(0)),
      fixed = TRUE)
    expect_equal(
      xpectr::strip(side_effects_19148[['messages']]),
      xpectr::strip(c("Will cross-validate 1 models. This requires fitting 4 model instances.\n",
        "\n---\ncross_validate(): cross_validate(): Used lme4::glmer() to fit the model.'\nFor:\nFormula: diagnosis~score+(1|session)\nFold column: .folds_1\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used lme4::glmer() to fit the model.'\nFor:\nFormula: diagnosis~score+(1|session)\nFold column: .folds_1\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used lme4::glmer() to fit the model.'\nFor:\nFormula: diagnosis~score+(1|session)\nFold column: .folds_1\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n",
        "\n---\ncross_validate(): cross_validate(): Used lme4::glmer() to fit the model.'\nFor:\nFormula: diagnosis~score+(1|session)\nFold column: .folds_1\nFold: 4\nHyperparameters: REML : FALSE, control : list(list(optimizer = c(\"bobyqa\", \"Nelder_Mead\"), restart_edge = FALSE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\", check.response.not.const = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, \n    relTol = NULL), check.conv.singular = list(action = \"message\", tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list(), tolPwrss = 1e-07, compDev = TRUE, nAGQ0initStep = TRUE)), model_verbose : TRUE, family : binomial, is_special_fn : TRUE\n")),
      fixed = TRUE)
    # Assigning output
    output_19148 <- xpectr::suppress_mw(cross_validate(dat,
                       formulas = c("diagnosis~score+(1|session)"),
                       fold_cols = paste0(".folds_", 1), family = "binomial",
                       REML = FALSE, verbose = TRUE,
                       positive = 1
        ))
    # Testing class
    expect_equal(
      class(output_19148),
      c("tbl_df", "tbl", "data.frame"),
      fixed = TRUE)
    # Testing column values
    expect_equal(
      output_19148[["Fixed"]],
      "score",
      fixed = TRUE)
    expect_equal(
      output_19148[["Balanced Accuracy"]],
      0.83333,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["F1"]],
      0.8,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Sensitivity"]],
      0.83333,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Specificity"]],
      0.83333,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Pos Pred Value"]],
      0.76923,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Neg Pred Value"]],
      0.88235,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["AUC"]],
      0.86111,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Lower CI"]],
      0.71033,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Upper CI"]],
      1,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Kappa"]],
      0.65753,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["MCC"]],
      0.65908,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Detection Rate"]],
      0.33333,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Detection Prevalence"]],
      0.43333,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Prevalence"]],
      0.4,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Folds"]],
      4,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Fold Columns"]],
      1,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Convergence Warnings"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Singular Fit Messages"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Other Warnings"]],
      0,
      tolerance = 1e-4)
    expect_equal(
      output_19148[["Dependent"]],
      "diagnosis",
      fixed = TRUE)
    expect_equal(
      output_19148[["Random"]],
      "(1|session)",
      fixed = TRUE)
    # Testing column names
    expect_equal(
      names(output_19148),
      c("Fixed", "Balanced Accuracy", "F1", "Sensitivity", "Specificity",
        "Pos Pred Value", "Neg Pred Value", "AUC", "Lower CI", "Upper CI",
        "Kappa", "MCC", "Detection Rate", "Detection Prevalence", "Prevalence",
        "Predictions", "ROC", "Confusion Matrix", "Results", "Coefficients",
        "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
        "Other Warnings", "Warnings and Messages", "Process", "Dependent", "Random"),
      fixed = TRUE)
    # Testing column classes
    expect_equal(
      xpectr::element_classes(output_19148),
      c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
        "numeric", "numeric", "numeric", "numeric", "numeric", "numeric",
        "numeric", "numeric", "numeric", "list", "list", "list", "list",
        "list", "integer", "integer", "integer", "integer", "integer",
        "list", "list", "character", "character"),
      fixed = TRUE)
    # Testing column types
    expect_equal(
      xpectr::element_types(output_19148),
      c("character", "double", "double", "double", "double", "double",
        "double", "double", "double", "double", "double", "double",
        "double", "double", "double", "list", "list", "list", "list",
        "list", "integer", "integer", "integer", "integer", "integer",
        "list", "list", "character", "character"),
      fixed = TRUE)
    # Testing dimensions
    expect_equal(
      dim(output_19148),
      c(1L, 29L))
    # Testing group keys
    expect_equal(
      colnames(dplyr::group_keys(output_19148)),
      character(0),
      fixed = TRUE)
    ## Finished testing 'cross_validate(dat, formulas = c("diagnosi...'     ####

  }

  # lm


  ## Testing 'cross_validate(dat, formulas = c("score~diag...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing side effects
  # Assigning side effects
  side_effects_19148 <- xpectr::capture_side_effects(cross_validate(dat,
                   formulas = c("score~diagnosis"),
                   fold_cols = paste0(".folds_", 1), family = "gaussian",
                   REML = FALSE, verbose = TRUE,
                   positive = 1
    ), reset_seed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_19148[['warnings']]),
    xpectr::strip(character(0)),
    fixed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_19148[['messages']]),
    xpectr::strip(c("Will cross-validate 1 models. This requires fitting 4 model instances.\n",
      "\n---\ncross_validate(): cross_validate(): Used lm() to fit the model.'\nFor:\nFormula: score~diagnosis\nFold column: .folds_1\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lm() to fit the model.'\nFor:\nFormula: score~diagnosis\nFold column: .folds_1\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lm() to fit the model.'\nFor:\nFormula: score~diagnosis\nFold column: .folds_1\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lm() to fit the model.'\nFor:\nFormula: score~diagnosis\nFold column: .folds_1\nFold: 4\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n")),
    fixed = TRUE)
  # Assigning output
  output_19148 <- xpectr::suppress_mw(cross_validate(dat,
                   formulas = c("score~diagnosis"),
                   fold_cols = paste0(".folds_", 1), family = "gaussian",
                   REML = FALSE, verbose = TRUE,
                   positive = 1
    ))
  # Testing class
  expect_equal(
    class(output_19148),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    output_19148[["Fixed"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    output_19148[["RMSE"]],
    17.16817,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["MAE"]],
    14.26914,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["NRMSE(IQR)"]],
    0.9074,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RRSE"]],
    0.9805,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RAE"]],
    0.98599,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RMSLE"]],
    0.47943,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["AIC"]],
    194.6904,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["AICc"]],
    195.99628,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["BIC"]],
    198.02426,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Folds"]],
    4,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Fold Columns"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Convergence Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Singular Fit Messages"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Dependent"]],
    "score",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(output_19148),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "AIC", "AICc", "BIC", "Predictions", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(output_19148),
    c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "list", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "list", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(output_19148),
    c("character", "double", "double", "double", "double", "double",
      "double", "double", "double", "double", "list", "list", "list",
      "integer", "integer", "integer", "integer", "integer", "list",
      "list", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(output_19148),
    c(1L, 21L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(output_19148)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'cross_validate(dat, formulas = c("score~diag...'     ####

  # lmer

  ## Testing 'cross_validate(dat, formulas = c("score~diag...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing side effects
  # Assigning side effects
  side_effects_19148 <- xpectr::capture_side_effects(cross_validate(dat,
                   formulas = c("score~diagnosis+(1|session)"),
                   fold_cols = paste0(".folds_", 1), family = "gaussian",
                   REML = FALSE, verbose = TRUE,
                   positive = 1
    ), reset_seed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_19148[['warnings']]),
    xpectr::strip(character(0)),
    fixed = TRUE)
  expect_equal(
    xpectr::strip(side_effects_19148[['messages']]),
    xpectr::strip(c("Will cross-validate 1 models. This requires fitting 4 model instances.\n",
      "\n---\ncross_validate(): cross_validate(): Used lme4::lmer() to fit the model.'\nFor:\nFormula: score~diagnosis+(1|session)\nFold column: .folds_1\nFold: 1\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lme4::lmer() to fit the model.'\nFor:\nFormula: score~diagnosis+(1|session)\nFold column: .folds_1\nFold: 2\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lme4::lmer() to fit the model.'\nFor:\nFormula: score~diagnosis+(1|session)\nFold column: .folds_1\nFold: 3\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n",
      "\n---\ncross_validate(): cross_validate(): Used lme4::lmer() to fit the model.'\nFor:\nFormula: score~diagnosis+(1|session)\nFold column: .folds_1\nFold: 4\nHyperparameters: REML : FALSE, control : list(list(optimizer = \"nloptwrap\", restart_edge = TRUE, boundary.tol = 1e-05, calc.derivs = TRUE, use.last.params = FALSE, checkControl = list(check.nobs.vs.rankZ = \"ignore\", check.nobs.vs.nlev = \"stop\", check.nlev.gtreq.5 = \"ignore\", check.nlev.gtr.1 = \"stop\", check.nobs.vs.nRE = \"stop\", check.rankX = \"message+drop.cols\", check.scaleX = \"warning\", check.formula.LHS = \"stop\"), checkConv = list(check.conv.grad = list(action = \"warning\", tol = 0.002, relTol = NULL), check.conv.singular = list(action = \"message\", \n    tol = 1e-04), check.conv.hess = list(action = \"warning\", tol = 1e-06)), optCtrl = list())), model_verbose : TRUE, family : gaussian, is_special_fn : TRUE\n")),
    fixed = TRUE)
  # Assigning output
  output_19148 <- xpectr::suppress_mw(cross_validate(dat,
                   formulas = c("score~diagnosis+(1|session)"),
                   fold_cols = paste0(".folds_", 1), family = "gaussian",
                   REML = FALSE, verbose = TRUE,
                   positive = 1
    ))
  # Testing class
  expect_equal(
    class(output_19148),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    output_19148[["Fixed"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    output_19148[["RMSE"]],
    9.65949,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["MAE"]],
    7.14593,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["NRMSE(IQR)"]],
    0.51285,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RRSE"]],
    0.56239,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RAE"]],
    0.50368,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["RMSLE"]],
    0.26177,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["AIC"]],
    175.94969,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["AICc"]],
    178.25232,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["BIC"]],
    180.39484,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Folds"]],
    4,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Fold Columns"]],
    1,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Convergence Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Singular Fit Messages"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    output_19148[["Dependent"]],
    "score",
    fixed = TRUE)
  expect_equal(
    output_19148[["Random"]],
    "(1|session)",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(output_19148),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "AIC", "AICc", "BIC", "Predictions", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent",
      "Random"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(output_19148),
    c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "list", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "list", "character", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(output_19148),
    c("character", "double", "double", "double", "double", "double",
      "double", "double", "double", "double", "list", "list", "list",
      "integer", "integer", "integer", "integer", "integer", "list",
      "list", "character", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(output_19148),
    c(1L, 22L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(output_19148)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'cross_validate(dat, formulas = c("score~diag...'     ####

})

test_that("binomial models with metrics list work with cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  CVbinomlist <- cross_validate(dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE,
    metrics = list(
      "AUC" = FALSE,
      "Accuracy" = TRUE,
      "Prevalence" = FALSE,
      "AICc" = TRUE
    ),
    verbose = FALSE,
    positive = 1
  )

  expect_equal(
    colnames(CVbinomlist),
    c(
      "Fixed", "Balanced Accuracy", "Accuracy", "F1", "Sensitivity", "Specificity",
      "Pos Pred Value", "Neg Pred Value", "Lower CI", "Upper CI", "Kappa",
      "MCC", "Detection Rate", "Detection Prevalence", "AICc", "Predictions",
      "ROC", "Confusion Matrix", "Results", "Coefficients", "Folds",
      "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process",
      "Dependent"
    )
  )

  expect_equal(CVbinomlist$Accuracy, c(0.766666666666667), tolerance = 1e-3)
  expect_equal(CVbinomlist$`Balanced Accuracy`, c(0.7361111), tolerance = 1e-3)

  expect_error(cross_validate(dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE,
    metrics = list(
      "AKG" = FALSE, # error here
      "Accuracy" = TRUE,
      "Prevalencer" = FALSE
    ),
    verbose = FALSE,
    positive = 1
  ),
  "'metrics_list' contained unknown metric names: AKG, Prevalencer.",
  fixed = TRUE
  )
  expect_error(
    xpectr::strip_msg(cross_validate(dat,
    formulas = c("diagnosis~score"),
    fold_cols = ".folds", family = "binomial",
    REML = FALSE,
    metrics = list(
      "AUC" = 1,
      "Accuracy" = TRUE,
      "Prevalence" = FALSE
    ),
    verbose = FALSE,
    positive = 1
  )),
  xpectr::strip(paste0(
    "1 assertions failed:\n * Variable 'metrics': May only conta",
    "in the following types: logical."
  )),
  fixed = TRUE
  )
})

test_that("gaussian models with metrics list work with cross_validate()", {

  testthat::skip_on_cran()

  # Load data and fold it
  xpectr::set_test_seed(1)
  dat <- groupdata2::fold(participant.scores,
    k = 4,
    cat_col = "diagnosis",
    id_col = "participant"
  )

  # Cross-validate the data
  CVed <- cross_validate(dat, "score~diagnosis",
    fold_cols = ".folds",
    family = "gaussian",
    REML = FALSE,
    metrics = list(
      "RMSE" = FALSE,
      "r2m" = TRUE
    ),
    verbose = FALSE
  )

  expect_equal(
    colnames(CVed),
    c("Fixed", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE", "r2m",
    "AIC", "AICc", "BIC", "Predictions", "Results", "Coefficients",
    "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
    "Other Warnings", "Warnings and Messages", "Process", "Dependent"
    )
  )
  expect_equal(
    colnames(CVed$Results[[1]]),
    c("Fold Column", "Fold", "MAE", "NRMSE(IQR)", "RRSE", "RAE",
    "RMSLE", "r2m", "AIC", "AICc", "BIC")
  )

  # Cross-validate the data
  expect_error(cross_validate(dat, "score~diagnosis",
    fold_cols = ".folds",
    family = "gaussian",
    REML = FALSE,
    metrics = list(
      "Accuracy" = TRUE, # Should error in gaussian
      "r2m" = TRUE
    ),
    verbose = FALSE
  ),
  "'metrics_list' contained unknown metric names: Accuracy.",
  fixed = TRUE
  )
})

test_that("varying number of folds in repeated cv with cross_validate()", {

  xpectr::set_test_seed(1)
  dat <- participant.scores
  dat$.folds_1 <- factor(sample(c(1,2), size = nrow(dat), replace = TRUE))
  dat$.folds_2 <- factor(sample(c(1,2,3), size = nrow(dat), replace = TRUE))
  dat$.folds_3 <- factor(sample(c(1,2,3,4), size = nrow(dat), replace = TRUE))

  CV <- cross_validate(
    dat,
    "score~diagnosis",
    fold_cols = paste0(".folds_", 1:3),
    family = "gaussian",
    REML = FALSE,
    verbose = FALSE
  )

  ## Testing 'CV'                                                           ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing class
  expect_equal(
    class(CV),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    CV[["Fixed"]],
    "diagnosis",
    fixed = TRUE)
  expect_equal(
    CV[["RMSE"]],
    17.49385,
    tolerance = 1e-4)
  expect_equal(
    CV[["MAE"]],
    14.70984,
    tolerance = 1e-4)
  expect_equal(
    CV[["NRMSE(IQR)"]],
    0.7814,
    tolerance = 1e-4)
  expect_equal(
    CV[["RRSE"]],
    0.98927,
    tolerance = 1e-4)
  expect_equal(
    CV[["RAE"]],
    0.99803,
    tolerance = 1e-4)
  expect_equal(
    CV[["RMSLE"]],
    0.50401,
    tolerance = 1e-4)
  expect_equal(
    CV[["AIC"]],
    165.81884,
    tolerance = 1e-4)
  expect_equal(
    CV[["AICc"]],
    167.51801,
    tolerance = 1e-4)
  expect_equal(
    CV[["BIC"]],
    168.61092,
    tolerance = 1e-4)
  expect_equal(
    CV[["Folds"]],
    9,
    tolerance = 1e-4)
  expect_equal(
    CV[["Fold Columns"]],
    3,
    tolerance = 1e-4)
  expect_equal(
    CV[["Convergence Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CV[["Singular Fit Messages"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CV[["Other Warnings"]],
    0,
    tolerance = 1e-4)
  expect_equal(
    CV[["Dependent"]],
    "score",
    fixed = TRUE)
  # Testing column names
  expect_equal(
    names(CV),
    c("Fixed", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE", "RMSLE",
      "AIC", "AICc", "BIC", "Predictions", "Results", "Coefficients",
      "Folds", "Fold Columns", "Convergence Warnings", "Singular Fit Messages",
      "Other Warnings", "Warnings and Messages", "Process", "Dependent"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(CV),
    c("character", "numeric", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "list", "list",
      "list", "integer", "integer", "integer", "integer", "integer",
      "list", "list", "character"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(CV),
    c("character", "double", "double", "double", "double", "double",
      "double", "double", "double", "double", "list", "list", "list",
      "integer", "integer", "integer", "integer", "integer", "list",
      "list", "character"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(CV),
    c(1L, 21L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(CV)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'CV'                                                  ####


  ## Testing 'CV$Results[[1]]'                                              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing class
  expect_equal(
    class(CV$Results[[1]]),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    CV$Results[[1]][["Fold Column"]],
    c(".folds_1", ".folds_1", ".folds_2", ".folds_2", ".folds_2", ".folds_3",
      ".folds_3", ".folds_3", ".folds_3"),
    fixed = TRUE)
  expect_equal(
    CV$Results[[1]][["Fold"]],
    c(1, 2, 1, 2, 3, 1, 2, 3, 4),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["RMSE"]],
    c(18.72588, 16.08458, 15.73932, 20.92178, 15.84949, 17.71002, 19.71718,
      14.04354, 18.82042),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["MAE"]],
    c(16.79356, 12.45238, 11.99286, 17.75604, 13.2899, 14.32647, 15.78632,
      13.84314, 16.68519),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["NRMSE(IQR)"]],
    c(0.59923, 0.90617, 0.74949, 0.83687, 0.64038, 1.18067, 0.75835,
      0.56174, 0.89621),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["RRSE"]],
    c(0.85218, 1.09378, 1.07973, 1.03806, 0.907, 1.01467, 1.2762, 0.68781,
      0.96761),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["RAE"]],
    c(0.91394, 1.02549, 0.92983, 1.05735, 0.87434, 1.04726, 1.17528,
      0.81966, 1.23991),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["RMSLE"]],
    c(0.57541, 0.43876, 0.39183, 0.52014, 0.60865, 0.34631, 0.5437,
      0.42518, 0.67712),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["AIC"]],
    c(118.60501, 142.88317, 200.20563, 140.93014, 176.37168, 215.25651,
      146.96483, 233.69377, 180.92466),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["AICc"]],
    c(121.00501, 144.88317, 201.46879, 142.77629, 177.87168, 216.39937,
      148.81098, 234.73725, 182.33642),
    tolerance = 1e-4)
  expect_equal(
    CV$Results[[1]][["BIC"]],
    c(120.52218, 145.20094, 203.61211, 143.42978, 179.35887, 218.91314,
      149.46447, 237.58128, 184.05823),
    tolerance = 1e-4)
  # Testing column names
  expect_equal(
    names(CV$Results[[1]]),
    c("Fold Column", "Fold", "RMSE", "MAE", "NRMSE(IQR)", "RRSE", "RAE",
      "RMSLE", "AIC", "AICc", "BIC"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(CV$Results[[1]]),
    c("character", "integer", "numeric", "numeric", "numeric", "numeric",
      "numeric", "numeric", "numeric", "numeric", "numeric"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(CV$Results[[1]]),
    c("character", "integer", "double", "double", "double", "double",
      "double", "double", "double", "double", "double"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(CV$Results[[1]]),
    c(9L, 11L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(CV$Results[[1]])),
    character(0),
    fixed = TRUE)
  ## Finished testing 'CV$Results[[1]]'                                     ####


  ## Testing 'CV$Coefficients'                                              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Testing class
  expect_equal(
    class(CV$Coefficients),
    "list",
    fixed = TRUE)
  # Testing type
  expect_type(
    CV$Coefficients,
    type = "list")
  # Testing values
  if (utils::packageVersion("parameters") > "0.18.1") {
    out1 <- as.data.frame(CV$Coefficients[[1]])
    out2 <- data.frame(
      `Fold Column` = c(
        ".folds_1", ".folds_1", ".folds_1", ".folds_1",
        ".folds_2", ".folds_2", ".folds_2", ".folds_2",
        ".folds_2", ".folds_2", ".folds_3", ".folds_3",
        ".folds_3", ".folds_3", ".folds_3", ".folds_3",
        ".folds_3", ".folds_3"
      ),
      Fold = c(1L, 1L, 2L, 2L, 1L, 1L, 2L, 2L, 3L, 3L, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L),
      term = c(
        "(Intercept)", "diagnosis", "(Intercept)", "diagnosis", "(Intercept)",
        "diagnosis", "(Intercept)", "diagnosis", "(Intercept)", "diagnosis",
        "(Intercept)", "diagnosis", "(Intercept)", "diagnosis", "(Intercept)",
        "diagnosis", "(Intercept)", "diagnosis"
      ),
      estimate = c(
        48.6666666666667, -14.2121212121212, 51.6666666666667, -26.952380952381,
        54.375, -24.175, 47.5714285714286, -22.1714285714286, 50.4444444444444,
        -14.3535353535354, 49.125, -19.0073529411765, 52.2222222222222,
        -29.5972222222222, 50.5, -18.9705882352941, 51.6666666666667,
        -16.0833333333333
      ),
      std.error = c(
        8.41855414379772, 9.49740995414852, 6.21405764952783, 9.39477209901097,
        6.10243657425927, 7.55651524308974, 5.15115732323402, 6.71628744224089,
        6.01599273599761, 8.1119629503789, 5.86007109036599, 7.1063800232963,
        5.42520220176549, 7.90852326457585, 5.39137655673118, 6.79449349609756,
        5.45953571418165, 7.22228688680003
      ),
      conf.level = c(
        0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95, 0.95,
        0.95, 0.95, 0.95, 0.95, 0.95, 0.95
      ),
      conf.low = c(
        30.324212890912, -34.9051998688293, 38.3388385420056, -47.1021630863613,
        41.6842884135979, -39.8896337174252, 36.5919966357145, -36.4868563880417,
        37.8053127114571, -31.3961371056051, 37.0025193413177, -33.7080200588983,
        40.6586774546993, -46.4538405405513, 39.3962521290018, -32.9641095369386,
        40.2397270909832, -31.1997535152435
      ),
      conf.high = c(
        67.0091204424213, 6.48095744458683, 64.9944947913278, -6.80259881840061,
        67.0657115864021, -8.46036628257483, 58.5508605071426, -7.85600075481539,
        63.0835761774318, 2.6890663985344, 61.2474806586823, -4.30668582345469,
        63.7857669897452, -12.7406039038931, 61.6037478709982, -4.97706693364964,
        63.0936062423501, -0.966913151423173
      ),
      statistic = c(
        5.78088182785179, -1.49642073794164, 8.31448138730939, -2.86887011928883,
        8.91037528015606, -3.19922599535646, 9.23509525846942, -3.30114349067095,
        8.38505740583801, -1.76942812009082, 8.38300410395395, -2.67468850228473,
        9.62585730080767, -3.74244612199539, 9.36681002868373, -2.79205333645399,
        9.46356418778573, -2.22690313821906
      ),
      df.error = c(
        12L, 12L, 14L, 14L, 21L, 21L, 15L, 15L, 18L, 18L, 23L, 23L, 15L, 15L, 25L, 25L, 19L, 19L
      ),
      p.value = c(
        8.73044693341471e-05, 0.160375077413601, 8.71744764220858e-07, 0.0123802797806211,
        1.40309391631771e-08, 0.00431125235564896, 1.40863068775986e-07, 0.00484752749572593,
        1.2467032478442e-07, 0.0937616722887625, 1.90501051336518e-08, 0.0135331351313378,
        8.2329274043691e-08, 0.00196103615278914, 1.18033672582856e-09, 0.0098923901197315,
        1.26960822180091e-08, 0.038244411370448
      ),
      check.names = FALSE,
      row.names = NULL,
      stringsAsFactors = FALSE
    )

    expect_equal(out1, out2, check.attributes = FALSE, tolerance = 1e-3, ignore_attr = TRUE)
  }

  # Testing names
  expect_equal(
    names(CV$Coefficients),
    NULL,
    fixed = TRUE)
  # Testing length
  expect_equal(
    length(CV$Coefficients),
    1L)
  # Testing sum of element lengths
  expect_equal(
    sum(xpectr::element_lengths(CV$Coefficients)),
    11L)
  ## Finished testing 'CV$Coefficients'                                     ####


  ## Testing 'CV$Predictions[[1]] %>% dplyr::count(`Fold C...'              ####
  ## Initially generated by xpectr
  xpectr::set_test_seed(42)
  # Assigning output
  output_19148 <- CV$Predictions[[1]] %>%
      dplyr::count(`Fold Column`, Fold)
  # Testing class
  expect_equal(
    class(output_19148),
    c("tbl_df", "tbl", "data.frame"),
    fixed = TRUE)
  # Testing column values
  expect_equal(
    output_19148[["Fold Column"]],
    c(".folds_1", ".folds_1", ".folds_2", ".folds_2", ".folds_2", ".folds_3",
      ".folds_3", ".folds_3", ".folds_3"),
    fixed = TRUE)
  expect_equal(
    output_19148[["Fold"]],
    c(1, 2, 1, 2, 3, 1, 2, 3, 4),
    tolerance = 1e-4)
  expect_equal(
    output_19148[["n"]],
    c(16, 14, 7, 13, 10, 5, 13, 3, 9),
    tolerance = 1e-4)
  # Testing column names
  expect_equal(
    names(output_19148),
    c("Fold Column", "Fold", "n"),
    fixed = TRUE)
  # Testing column classes
  expect_equal(
    xpectr::element_classes(output_19148),
    c("character", "integer", "integer"),
    fixed = TRUE)
  # Testing column types
  expect_equal(
    xpectr::element_types(output_19148),
    c("character", "integer", "integer"),
    fixed = TRUE)
  # Testing dimensions
  expect_equal(
    dim(output_19148),
    c(9L, 3L))
  # Testing group keys
  expect_equal(
    colnames(dplyr::group_keys(output_19148)),
    character(0),
    fixed = TRUE)
  ## Finished testing 'CV$Predictions[[1]] %>% dplyr::count(`Fold C...'     ####


})

