Nothing
#' Replicate weights methods
#'
#' @description A function calculating replicate weights.
#'
#' @param data A data frame with information on (at least) cluster and strata indicators, and sampling weights. It could be \code{NULL} if the sampling design were indicated in the \code{design} argument (see \code{design}).
#' @param method A string of a name of replicate weights methods. Choose one of these: \code{JKn}, \code{dCV}, \code{bootstrap}, \code{subbootstrap}, \code{BRR}, \code{split}, \code{extrapolation}.
#' @param cluster A string of a cluster identifier. It could be \code{NULL} if the sampling design were plugged in the \code{design} argument.
#' @param strata A string of a strata identifier. It could be \code{NULL} if the sampling design were plugged in the \code{design} argument.
#' @param weights A string of a sampling weights' identifier. It could be \code{NULL} if the sampling design were included in the \code{design} argument.
#' @param design An object of class \code{survey.design} generated by \code{\link[survey]{svydesign}}. It could be \code{NULL} if information about \code{cluster}, \code{strata}, \code{weights} and \code{data} were given.
#' @param k An integer. The number of folds to be defined. Default is \code{k=10}. Only applies to the \code{dCV} method.
#' @param R An integer. The number of times the sample is partitioned. Default is \code{R=1}. Only applies to \code{dCV}, \code{split} or \code{extrapolation} methods.
#' @param B An integer. The number of bootstrap re-samples. Default is \code{B=200}. Only applies to \code{bootstrap} and \code{subbootstrap} methods.
#' @param train.prob A numeric between 0 and 1 that sets the proportion of clusters (for the method \code{split}) or strata (for the method \code{extrapolation}) in training sets. Default is \code{train.prob=0.7}. Only applies to \code{split} and \code{extrapolation} methods.
#' @param method.split A string of one the following replicate weights methods to be implemented under the \code{split} method in the \code{method} argument: \code{dCV}, \code{bootstrap} or \code{subbootstrap}.
#' @param rw.test A logical value. If \code{TRUE}, the function returns in the output object the replicate weights to the corresponding test sets. If \code{FALSE}, only the replicate weights of the training sets are returned. Default is \code{rw.test = FALSE}.
#' @param dCV.sw.test A logical value. If \code{TRUE}, original sampling weights for the units in the test sets are returned instead of the replicate weights. Default is \code{dCV.sw.test = FALSE}. Only applies to \code{dCV} method.
#'
#' @return The function returns a new data frame with new columns, each of which indicates replicate weights for different subsets.
#'
#' @examples
#' data(nhanes2013_sbc)
#'
#' ## JKn ---------------------------------------------------------------------
#' data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "JKn",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' rw.test = TRUE)
#'
#' ## dCV ---------------------------------------------------------------------
#' data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "dCV",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' k = 10, R = 20,
#' rw.test = TRUE)
#'
#' ## subbootstrap ------------------------------------------------------------
#' # rw.test = TRUE now also returns out-of-bag ("sw_r_1_test_*") weights:
#' # units whose PSU was not drawn into a given replicate (resample multiplier
#' # == 0) are treated as that replicate's held-out test set.
#' data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "subbootstrap",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' B = 100,
#' rw.test = TRUE)
#'
#' ## BRR ---------------------------------------------------------------------
#' data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "BRR",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' rw.test = TRUE)
#'
#' ## split ---------------------------------------------------------------------
#' \donttest{
#' data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "split",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' R=20,
#' train.prob = 0.5,
#' method.split = "subbootstrap",
#' rw.test = TRUE)
#'}
#' ## extrapolation -------------------------------------------------------------
#'data_rw <- replicate_weights(data = nhanes2013_sbc,
#' method = "extrapolation",
#' cluster = "SDMVPSU",
#' strata = "SDMVSTRA",
#' weights = "WTSAF2YR",
#' R=20,
#' train.prob = 0.5,
#' rw.test = TRUE)
#'
#'
#' @details
#' This function from R-\code{wlasso} (\url{https://github.com/aiparragirre/wlasso/}, recently registered in CRAN: \code{svyVarSel})
#' is slightly modified for our package development. The following description is also obtained from the original source for convenience.
#'
#' Some of replicate weights methods, such as \code{JKn}, \code{bootstrap}, \code{subbootstrap} and \code{BRR},
#' are implemented in R-\code{survey}, \code{\link[survey]{as.svrepdesign}}, from which
#' the \code{replicate_weights()} function generates replicate weights for these options;
#' \code{dCV}, \code{split} and \code{extrapolation} have been expressly defined in this function.
#'
#' Selecting any of the above-mentioned methods returns a new data frame,
#' in which new columns are added to the original data set; each column by replicate
#' weights always creates different pairs of training sets and optionally the corresponding test sets by setting \code{rw.test=TRUE}.
#' The number of new columns and their corresponding names are usually determined by numeric arguments, i.e.,\code{k}, or \code{R}, etc.
#' and specifically by the selected replicate weights method. The new columns indicating training and test sets
#' follow the similar structure for any selected method: \code{rw_r_x_train_t}, where \code{x=1,...,R}, and \code{t=1,...,T}
#' indicate the \code{x}-th partition of the sample,the \code{t}-th training set, respectively.
#' Similarly, the new columns for test sets are structured by \code{rw_r_x_test_t} or \code{sw_r_x_test_t}.
#' For other methods, `fold` or `set` for each unit respectively labeled as \code{fold_t} or \code{set_t}
#' is included in each partition depending on the method in a data set.
#'
#' For \code{JKn}, \code{bootstrap}, and \code{subbootstrap}, the \code{sw_r_1_test_*} test
#' columns (generated when \code{rw.test = TRUE}) represent an out-of-bag (OOB) style test
#' set: for each replicate, a unit's original sampling weight is assigned to the test column
#' if and only if that unit's PSU was not drawn into that particular replicate (resample
#' multiplier of exactly 0), and 0 otherwise. For \code{bootstrap}/\code{subbootstrap}
#' specifically, this means the OOB test fraction per replicate is random and not fixed in
#' advance -- in expectation roughly \eqn{1/e \approx 37\%} of PSUs are held out per
#' replicate for ordinary bootstrap, somewhat more for subbootstrap -- rather than a
#' predetermined split proportion as in \code{dCV} or \code{split}.
#'
#'
#' @export
replicate_weights <- function(data,
method = c("JKn", "dCV", "bootstrap", "subbootstrap",
"BRR", "split", "extrapolation"),
cluster = NULL, strata = NULL, weights = NULL, design = NULL,
k = 10, R=1, B = 200,
train.prob = 0.7, method.split = c("dCV", "bootstrap", "subbootstrap"),
rw.test = FALSE, dCV.sw.test = FALSE){
# Stops and messages:
if(method == "split"){
if(is.null(train.prob)){stop("Selected replicate weights method: 'split'.\nPlease, set a value between 0 and 1 for the argument 'train.prob'.")}
if(train.prob < 0 | train.prob > 1){stop("Selected replicate weights method: 'split'.\nPlease, set a value between 0 and 1 for the argument 'train.prob'.")}
if(length(method.split)!=1){stop("Selected replicate weights method: 'split'.\nPlease, set a valid method for the argument 'method.split'. Choose between: 'dCV', 'bootstrap' or 'subbootstrap'.")}
}
if(method == "extrapolation"){
if(is.null(train.prob)){stop("Selected replicate weights method: 'extrapolation'.\nPlease, set a value between 0 and 1 for the argument 'train.prob'.")}
if(train.prob < 0 | train.prob > 1){stop("Selected replicate weights method: 'extrapolation'.\nPlease, set a value between 0 and 1 for the argument 'train.prob'.")}
}
if(method %in% c("JKn", "bootstrap", "subbootstrap", "BRR")){
if(R!=1){message("Selected method:", method,". For this method, R = 1. Thus, the argument R =",R, "has been ignored.")}
}
if(method %in% c("dCV", "split", "extrapolation")){
if(R != round(R)){stop("The argument 'R' must be an integer greater or equal to 1. R=",R," is not an integer.\nPlease, set a valid value for 'R' or skip the argument to select the default option R=1.")}
if(R < 1){stop("The argument 'R' must be an integer greater or equal to 1. R=",R," lower than 1.\nPlease, set a valid value for 'R' or skip the argument to select the default option R=1.")}
}
if(method != "dCV"){
if(!is.null(k) & k!=10){message("Selected method:", method,". The argument k =",k, "is not needed and, hence, has been ignored.")}
}
if(method == "dCV"){
if(k != round(k)){stop("The argument 'k' must be an integer. k=",k," is not an integer.\nPlease, set a valid value for 'k' or skip the argument to select the default option k=10.")}
if(k < 1){stop("The argument 'k' must be a positive integer. k=",k," is not a positive integer.\nPlease, set a valid value for 'k' or skip the argument to select the default option k=10.")}
}
if(!(method %in% c("bootstrap", "subbootstrap"))){
if(!is.null(B) & B!=200){message("Selected method:", method,". The argument B =",B, "is not needed and, hence, has been ignored.")}
}
if(method %in% c("bootstrap", "subbootstrap")){
if(B != round(B)){stop("The argument 'B' must be an integer. B=",B," is not an integer.\nPlease, set a valid value for 'B' or skip the argument to select the default option B=200.")}
if(B < 1){stop("The argument 'B' must be a positive integer. B=",B," is not a positive integer.\nPlease, set a valid value for 'B' or skip the argument to select the default option B=200.")}
}
# Step 0: Notation
if(!is.null(design)){
cluster <- as.character(design$call$id[2])
if(cluster == "1" || cluster == "0"){
cluster <- NULL
}
strata <- as.character(design$call$strata[2])
weights <- as.character(design$call$weights[2])
data <- design$variables
}
if(method == "dCV"){
newdata <- cv_folds(data, k, weights, strata, cluster, R, rw.test, dCV.sw.test)
if(rw.test==TRUE & dCV.sw.test ==TRUE){
for(r in 1:R){
for(kk in 1:k){
newdata[, paste0("sw_r_",r,"_test_", kk)] <- rep(0, nrow(newdata))
newdata[which(newdata[,paste0("folds_",r)]==kk), paste0("sw_r_",r,"_test_", kk)] <- newdata[which(newdata[,paste0("folds_",r)]==kk), weights]
}
}
}
} else {
if(method == "split"){
newdata <- rw.split(data, train.prob, method = method.split,
weights, strata, cluster, R, rw.test)
} else {
if(method == "extrapolation"){
newdata <- split_strata(data, train.prob, strata, weights, R, rw.test)
} else {
# Define cluster formula
if(is.null(cluster)) {
formula.cluster <- as.formula("~1")
} else {
formula.cluster <- as.formula(paste0("~", cluster))
}
# Define strata formula
if(!is.null(strata)) {
formula.strata <- as.formula(paste0("~", strata))
}
# Define weights formula
if(!is.null(weights)){
formula.weights <- as.formula(paste0("~", weights))
}
# Define the design
des <- survey::svydesign(ids = formula.cluster,
strata = formula.strata,
weights = formula.weights,
data = data, nest=TRUE)
# Generate replicate weights based on the selected method
if(method %in% c("JKn", "bootstrap", "subbootstrap", "BRR")){
if(method %in% c("bootstrap", "subbootstrap")){
rep.des <- survey::as.svrepdesign(design = des, type = method, replicates = B)
} else {
rep.des <- survey::as.svrepdesign(design = des, type = method)
}
mat.repw.ind <- apply(rep.des$repweights$weights, 2, function(x){x[rep.des$repweights$index]})
mat.repw <- apply(mat.repw.ind, 2, function(x){x*data[,weights]})
colnames(mat.repw) <- paste0("rw_r_1_train_", 1:ncol(mat.repw))
newdata <- cbind(data, mat.repw)
}
# Define replicate weights for the testing set in BRR
if(method == "BRR" & rw.test == TRUE){
mat.repw.test <- mat.repw.ind
colnames(mat.repw.test) <- paste0("rw_r_1_test_", 1:ncol(mat.repw.test))
mat.repw.test <- -1*(mat.repw.test - 2)
mat.repw.test <- apply(mat.repw.test, 2, function(x){x*data[,weights]})
newdata <- cbind(newdata, mat.repw.test)
}
# Define each unit as out-of-bag (OOB) test weight for JKn, bootstrap,
# and subbootstrap. This is the same zero/nonzero inversion of
# mat.repw.ind for all three methods: wherever a unit's PSU was NOT
# drawn into a given replicate (resample multiplier == 0), that unit's
# original sampling weight becomes its test weight for that replicate;
# everywhere else the test weight is 0.
#
# For bootstrap/subbootstrap specifically, this generalizes the
# existing JKn-only logic below: as.svrepdesign(type="bootstrap"/
# "subbootstrap") already produces the resample-multiplier matrix
# (mat.repw.ind) needed for this, it was simply never used to build a
# held-out test set previously. Without this block, functions such as
# wElnet()/wXGBoost()/wRandomforest() that expect paired "_train_"/
# "_test_" columns for every replicate-weight method would fail (or
# silently misbehave) whenever method = "bootstrap" or "subbootstrap"
# was selected directly (as opposed to nested under method = "split").
if(method %in% c("JKn", "bootstrap", "subbootstrap") & rw.test == TRUE){
mat.repw.test <- -mat.repw.ind
mat.repw.test[which(mat.repw.test==0)] <- 1
mat.repw.test[which(mat.repw.test < 0)] <- 0
mat.repw.test <- apply(mat.repw.test, 2, function(x){x*data[,weights]})
colnames(mat.repw.test) <- paste0("sw_r_1_test_", 1:ncol(mat.repw.test))
newdata <- cbind(newdata, mat.repw.test)
}
}
}
}
return(newdata)
}
# @export
cv_folds <- function(data, k, weights, strata=NULL, cluster=NULL, R=1, rw.test, dCV.sw.test){
# function for dCV
if(is.null(cluster) & !is.null(strata)){
data$cluster <- 1:nrow(data)
cluster <- "cluster"
} else {
if(!is.null(cluster) & is.null(strata)){
data$strata <- rep(1, nrow(data))
strata <- "strata"
} else {
if(is.null(cluster) & is.null(strata)){
data$strata <- rep(1, nrow(data))
data$cluster <- 1:nrow(data)
strata <- "strata"
cluster <- "cluster"
}
}
}
for(r in 1:R){
data[,paste0("folds_",r)] <- f.folds(data, k=k, strata=strata, cluster=cluster)
for(kk in 1:k){
data[,paste0("rw_r_",r,"_train_", kk)] <- repl.weights(data,
folds = paste0("folds_",r),
test.fold=kk,
weights,
strata,
cluster)
}
if(rw.test==TRUE & dCV.sw.test == FALSE){
for(kk in 1:k){
data[,paste0("rw_r_",r,"_test_", kk)] <- repl.weights.test(data,
folds = paste0("folds_",r),
test.fold=kk,
weights,
strata,
cluster)
}
}
}
return(data)
}
f.folds <- function(data, k=5, strata=NULL, cluster=NULL){
hclus<- NULL
data$hclus <- interaction(data[,strata], data[,cluster], drop=TRUE)
newids <- levels(data$hclus)
n <- length(newids)
v.folds.newids <- sample(cut(seq(1,n),breaks=k,labels=FALSE))
names(v.folds.newids) <- newids
v.folds <- v.folds.newids[match(data$hclus, names(v.folds.newids))]
h.fold <- table(data[,strata], v.folds)!=0
h.fold.sum <- apply(h.fold, 1, sum)
h.onefold <- names(which(h.fold.sum==1))
if(length(h.onefold)!=0){
for(hh in h.onefold){
kk <- which(h.fold[hh,]==1)
id.h <- which(data[,strata]==hh)
psu.h <- unique(data$hclus[id.h])
selected.psu <- sample(psu.h, 1)
newk <- sample(c(1:k)[-kk], 1)
id.selected.psu <- which(data$hclus==selected.psu)
v.folds[id.selected.psu] <- newk
}
}
return(v.folds)
}
repl.weights <- function(data, folds, test.fold, weights, strata=NULL, cluster=NULL){
v.repl.weights <- rep(0, nrow(data))
id.test <- which(data[,folds]==test.fold)
data[,strata] <- as.factor(data[,strata])
str.clus <- table(data[,strata], data[,cluster])!=0
str.clus.test <- table(data[id.test,strata], data[id.test,cluster])!=0
v.mh <- apply(str.clus.test, 1, sum)
v.nh <- apply(str.clus, 1, sum)
coef <- v.nh/(v.nh - v.mh)
v.repl.weights[-id.test] <- data[-id.test,weights]*coef[match(data[-id.test,strata], names(coef))]
return(v.repl.weights)
}
repl.weights.test <- function(data, folds, test.fold, weights, strata=NULL, cluster=NULL){
v.repl.weights <- rep(0, nrow(data))
id.test <- which(data[,folds]!=test.fold)
data[,strata] <- as.factor(data[,strata])
str.clus <- table(data[,strata], data[,cluster])!=0
str.clus.test <- table(data[id.test,strata], data[id.test,cluster])!=0
v.mh <- apply(str.clus.test, 1, sum)
v.nh <- apply(str.clus, 1, sum)
coef <- v.nh/(v.nh - v.mh)
v.repl.weights[-id.test] <- data[-id.test,weights]*coef[match(data[-id.test,strata], names(coef))]
return(v.repl.weights)
}
rw.split <- function(data, train.prob, method = c("dCV", "bootstrap", "subbootstrap"),
weights, strata = NULL, cluster = NULL, R = 1, rw.test){
# function for Split-sample
if(is.null(cluster) & !is.null(strata)){
data$cluster <- 1:nrow(data)
cluster <- "cluster"
} else {
if(!is.null(cluster) & is.null(strata)){
data$strata <- rep(1, nrow(data))
strata <- "strata"
} else {
if(is.null(cluster) & is.null(strata)){
data$strata <- rep(1, nrow(data))
data$cluster <- 1:nrow(data)
strata <- "strata"
cluster <- "cluster"
}
}
}
for(r in 1:R){
data <- split_sample(data, train.prob, r, strata, cluster)
tags <- as.vector(unique(data[,paste0("set_",r)]))
if(method == "dCV"){
if(rw.test == FALSE){
data[,paste0("rw_r_",r,"_", "train")] <- repl.weights.test(data,
folds = paste0("set_",r),
test.fold = "train",
weights, strata, cluster)
} else {
for(tag in tags){
data[,paste0("rw_r_",r,"_", tag)] <- repl.weights.test(data,
folds = paste0("set_",r),
test.fold = tag,
weights, strata, cluster)
}
}
} else {
if(method %in% c("bootstrap", "subbootstrap")){
if(rw.test == FALSE){
data <- replicate.sample(data, set = paste0("set_",r), "train",
strata, weights, r,
boot.type = method)
} else {
for(tag in tags){
data <- replicate.sample(data, set = paste0("set_",r), tag,
strata, weights, r,
boot.type = method)
}
}
}
}
}
return(data)
}
# @export
split_sample <- function(data, train.prob, r,
strata = NULL, cluster = NULL){
hclus<- NULL
data[,strata] <- as.factor(data[,strata])
set <- paste0("set_",r)
data$hclus <- interaction(data[,strata], data[,cluster], drop=TRUE)
newids <- levels(data$hclus)
n <- length(newids)
factor <- c(0, train.prob, 1)
set.newids <- sample(cut(seq(1,n)/n, factor, labels = c("train", "test")))
names(set.newids) <- newids
data[,set] <- as.factor(set.newids[match(data$hclus, names(set.newids))])
train.0 <- table(data[which(data[,set]=="train"), strata])==0
if(sum(train.0) != 0){
h.0 <- names(which(train.0 == TRUE))
for(hh in h.0){
id.hh <- which(data[,strata]==hh)
psu.h <- unique(data$hclus[id.hh])
selected.psu <- sample(psu.h, size=1)
id.selected.psu <- which(data$hclus==selected.psu)
data[id.selected.psu,set] <- "train"
}
}
test.0 <- table(data[which(data[,set]=="test"), strata])==0
if(sum(test.0) != 0){
h.0 <- names(which(test.0 == TRUE))
for(hh in h.0){
id.hh <- which(data[,strata]==hh)
psu.h <- unique(data$hclus[id.hh])
selected.psu <- sample(psu.h, size=1)
id.selected.psu <- which(data$hclus==selected.psu)
data[id.selected.psu,set] <- "test"
}
}
data <- subset(data, select = -hclus)
return(data)
}
replicate.sample <- function(data, set, tag, strata, weights, r=1,
boot.type = c("bootstrap", "subbootstrap")){
data[,paste0("bootrep_r_",r,"_",tag)] <- rep(0, nrow(data))
data[which(data[,set] == tag),paste0("bootrep_r_",r,"_",tag)] <- 1
nh0 <- table(data[,strata])
if(boot.type == "bootstrap"){
new.nh <- nh0
} else {
if(boot.type == "subbootstrap"){
new.nh <- nh0 - 1
}
}
nh0.tag <- table(data[which(data[,set] == tag), strata])
for(hh in 1:length(unique(data[,strata]))){
if(nh0.tag[hh] < new.nh[hh]){
n.add <- new.nh[hh] - nh0.tag[hh]
id.opt <- which(data[,set] == tag & data[,strata] == hh)
if(length(id.opt)>1){
selected.id <- sample(id.opt, size = n.add, replace = TRUE)
} else {
selected.id <- rep(id.opt, n.add)
}
n.adds <- table(selected.id)
data[as.numeric(names(table(selected.id))),paste0("bootrep_r_",r,"_",tag)] <- data[as.numeric(names(table(selected.id))),paste0("bootrep_r_",r,"_",tag)] + n.adds
}
}
coef.h <- nh0/new.nh
coef <- coef.h[match(data[,strata], names(coef.h))]
data[,paste0("rw_r_",r,"_", tag)] <- data[, weights]*data[,paste0("bootrep_r_",r,"_",tag)]*coef
# Delete bootstrap repetition columns
col.bootrep <- grep("bootrep_", colnames(data))
data <- data[,-col.bootrep]
return(data)
}
# @export
split_strata <- function(data, train.prob, strata = NULL, weights, R = 1, rw.test){
# function for Extrapolation
if(is.null(strata)){stop("Extrapolation method cannot be applied if strata are not defined")}
h <- unique(data[,strata])
for(r in 1:R){
number.h <- floor(length(h)*train.prob)
train.h <- sample(1:length(h), number.h)
h.split <- vector(length = length(h))
names(h.split) <- h
h.split[train.h] <- "train"
h.split[-train.h] <- "test"
data[, paste0("set_", r)] <- as.vector(h.split[match(data[,strata], names(h.split))])
data[, paste0("rw_r_",r,"_train")] <- rep(0, nrow(data))
id.train <- which(data[,paste0("set_",r)] == "train")
data[id.train, paste0("rw_r_",r,"_train")] <- data[id.train, weights]
if(rw.test == TRUE){
data[, paste0("rw_r_",r,"_test")] <- rep(0, nrow(data))
id.test <- which(data[,paste0("set_",r)] == "test")
data[id.test, paste0("rw_r_",r,"_test")] <- data[id.test, weights]
}
}
return(data)
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.