% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/kfold.R
\name{kfold.brmsfit}
\alias{kfold.brmsfit}
\alias{kfold}
\title{K-Fold Cross-Validation}
\usage{
\method{kfold}{brmsfit}(
  x,
  ...,
  K = 10,
  Ksub = NULL,
  folds = NULL,
  group = NULL,
  joint = FALSE,
  compare = TRUE,
  resp = NULL,
  model_names = NULL,
  save_fits = FALSE,
  recompile = NULL,
  future_args = list()
)
}
\arguments{
\item{x}{A \code{brmsfit} object.}

\item{...}{Further arguments passed to \code{\link{brm}} and
\code{\link{log_lik}}.}

\item{K}{The number of subsets of equal (if possible) size
into which the data will be partitioned for performing
\eqn{K}-fold cross-validation. The model is refit \code{K} times, each time
leaving out one of the \code{K} subsets. If \code{K} is equal to the total
number of observations in the data then \eqn{K}-fold cross-validation is
equivalent to exact leave-one-out cross-validation.}

\item{Ksub}{Optional number of subsets (of those subsets defined by \code{K})
to be evaluated. If \code{NULL} (the default), \eqn{K}-fold cross-validation
will be performed on all subsets. If \code{Ksub} is a single integer,
\code{Ksub} subsets (out of all \code{K}) subsets will be randomly chosen.
If \code{Ksub} consists of multiple integers or a one-dimensional array
(created via \code{as.array}) potentially of length one, the corresponding
subsets will be used. This argument is primarily useful, if evaluation of
all subsets is infeasible for some reason.}

\item{folds}{Determines how the subsets are being constructed.
Possible values are \code{NULL} (the default), \code{"stratified"},
\code{"grouped"}, or \code{"loo"}. May also be a vector of length
equal to the number of observations in the data. Alters the way
\code{group} is handled. More information is provided in the 'Details'
section.}

\item{group}{Optional name of a grouping variable or factor in the model.
What exactly is done with this variable depends on argument \code{folds}.
More information is provided in the 'Details' section.}

\item{joint}{Indicates which observations' log likelihoods shall be
considered jointly in the ELPD computation. If \code{"obs"} or \code{FALSE}
(the default), each observation is considered separately. This enables
comparability of \code{kfold} with \code{loo}. If \code{"fold"} or
\code{TRUE}, the joint log likelihoods per fold are used. If
\code{"group"}, the joint log likelihoods per group within folds are used
(only available if argument \code{group} is specified).}

\item{compare}{A flag indicating if the information criteria
of the models should be compared to each other
via \code{\link{loo_compare}}.}

\item{resp}{Optional names of response variables. If specified, predictions
are performed only for the specified response variables.}

\item{model_names}{If \code{NULL} (the default) will use model names
derived from deparsing the call. Otherwise will use the passed
values as model names.}

\item{save_fits}{If \code{TRUE}, a component \code{fits} is added to
the returned object to store the cross-validated \code{brmsfit}
objects and the indices of the omitted observations for each fold.
Defaults to \code{FALSE}.}

\item{recompile}{Logical, indicating whether the Stan model should be
recompiled. This may be necessary if you are running \code{reloo} on
another machine than the one used to fit the model.}

\item{future_args}{A list of further arguments passed to
\code{\link[future:future]{future}} for additional control over parallel
execution if activated.}
}
\value{
\code{kfold} returns an object that has a similar structure as the
  objects returned by the \code{loo} and \code{waic} methods and
  can be used with the same post-processing functions.
}
\description{
Perform exact K-fold cross-validation by refitting the model \eqn{K}
times each leaving out one-\eqn{K}th of the original data.
Folds can be run in parallel using the \pkg{future} package.
}
\details{
The \code{kfold} function performs exact \eqn{K}-fold
  cross-validation. First the data are partitioned into \eqn{K} folds
  (i.e. subsets) of equal (or as close to equal as possible) size by default.
  Then the model is refit \eqn{K} times, each time leaving out one of the
  \code{K} subsets. If \eqn{K} is equal to the total number of observations
  in the data then \eqn{K}-fold cross-validation is equivalent to exact
  leave-one-out cross-validation (to which \code{loo} is an efficient
  approximation). The \code{compare_ic} function is also compatible with
  the objects returned by \code{kfold}.

  The subsets can be constructed in multiple different ways:
  \itemize{
  \item If both \code{folds} and \code{group} are \code{NULL}, the subsets
  are randomly chosen so that they have equal (or as close to equal as
  possible) size.
  \item If \code{folds} is \code{NULL} but \code{group} is specified, the
  data is split up into subsets, each time omitting all observations of one
  of the factor levels, while ignoring argument \code{K}.
  \item If \code{folds = "stratified"} the subsets are stratified after
  \code{group} using \code{\link[loo:kfold-helpers]{loo::kfold_split_stratified}}.
  \item If \code{folds = "grouped"} the subsets are split by
  \code{group} using \code{\link[loo:kfold-helpers]{loo::kfold_split_grouped}}.
  \item If \code{folds = "loo"} exact leave-one-out cross-validation
  will be performed and \code{K} will be ignored. Further, if \code{group}
  is specified, all observations corresponding to the factor level of the
  currently predicted single value are omitted. Thus, in this case, the
  predicted values are only a subset of the omitted ones.
  \item If \code{folds} is a numeric vector, it must contain one element per
  observation in the data. Each element of the vector is an integer in
  \code{1:K} indicating to which of the \code{K} folds the corresponding
  observation belongs. There are some convenience functions available in
  the \pkg{loo} package that create integer vectors to use for this purpose
  (see the Examples section below and also the
  \link[loo:kfold-helpers]{kfold-helpers} page).
  }

  When running \code{kfold} on a \code{brmsfit} created with the
  \pkg{cmdstanr} backend in a different \R session, several recompilations
  will be triggered because by default, \pkg{cmdstanr} writes the model
  executable to a temporary directory. To avoid that, set option
  \code{"cmdstanr_write_stan_file_dir"} to a nontemporary path of your choice
  before creating the original \code{brmsfit} (see section 'Examples' below).

By default, this method uses \code{sample_new_levels = "gaussian"}
  to sample parameter values for new grouping-factor levels (see also
  \code{\link{prepare_predictions}}). This default will fail for models with
  non-Gaussian group-level effects. In this case, we recommend setting
  \code{sample_new_levels = "uncertainty"}.
}
\section{Parallelization with multiple CPU cores}{


\pkg{brms} can make use of multiple CPU cores in parallel to speed
up computations in various ways. For efficient use of the
available resources it is recommended to only use parallelism to
an extend such that the available physical CPUs are not
oversubscribed. For example, when you have 8 CPU cores locally
available, then you may consider to run 4 chains with 2 threads
per chain for best performance if you happen to just run a single
model. In case you run a simulation study which requires to run
many times a given model, then neither chain nor within-chain
parallelization is advisable as the computational resources are
already exhausted by the simulation study and any further
parallelization beyond the simulation study itself will in fact
slow down the overall runtime. Please be aware that for
historical reasons the nomenclature of the arguments is possibly
confusing. The \code{cores} argument refers to running different
chains in parallel and the within-chain parallelization will
allocate for each chain as many threads as requested. The
requested threads therefore increase the use of overall CPUs in a
multiplicative way.

For more advanced parallelization (including beyond single model
fits), \pkg{brms} also integrates with the \pkg{future}
package. Importantly, this enables seamless integration with the
\pkg{mirai} parallelization framework through the use of the
\pkg{future.mirai} adapter. With \pkg{mirai} local and remote
machines can be used in a fully transparent manner to the
user. This includes the possibility to use large number of remote
machines running in the context of a computer cluster, which are
managed with queuing systems. Please refer to the section on
distributed computing of
\code{\link[mirai:daemons]{mirai::daemons}}.
}

\examples{
\dontrun{
fit1 <- brm(count ~ zAge + zBase * Trt + (1|patient) + (1|obs),
           data = epilepsy, family = poisson())
# throws warning about some pareto k estimates being too high
(loo1 <- loo(fit1))
# perform 10-fold cross validation
(kfold1 <- kfold(fit1, chains = 1))

# use joint likelihoods per fold for ELPD evaluation
kfold(fit1, chains = 1, joint = "fold")

# use the future package for parallelization of models
# that is to fit models belonging to different folds in parallel
library(future)
plan(multisession, workers = 4)
kfold(fit1, chains = 1)
plan(sequential)

## to avoid recompilations when running kfold() on a 'cmdstanr'-backend fit
## in a fresh R session, set option 'cmdstanr_write_stan_file_dir' before
## creating the initial 'brmsfit'
## CAUTION: the following code creates some files in the current working
## directory: two 'model_<hash>.stan' files, one 'model_<hash>(.exe)'
## executable, and one 'fit_cmdstanr_<some_number>.rds' file
set.seed(7)
fname <- paste0("fit_cmdstanr_", sample.int(.Machine$integer.max, 1))
options(cmdstanr_write_stan_file_dir = getwd())
fit_cmdstanr <- brm(rate ~ conc + state, data = Puromycin,
                    backend = "cmdstanr", file = fname)

# now restart the R session and run the following (after attaching 'brms')
set.seed(7)
fname <- paste0("fit_cmdstanr_", sample.int(.Machine$integer.max, 1))
fit_cmdstanr <- brm(rate ~ conc + state,
                    data = Puromycin,
                    backend = "cmdstanr",
                    file = fname)
kfold_cmdstanr <- kfold(fit_cmdstanr, K = 2)
}

}
\seealso{
\code{\link{loo}}, \code{\link{reloo}}
}
