Nothing
test_that("parscores returns parscores", {
expect_equal(sum(parscores(dmeas, modelameas)), 100)
expect_equal(sum(parscores(dmeas, modelbmeas)), 0)
# same test with algorithm features
expect_equal(sum(parscores(dmeas.algo, modelameas.algo)), 100)
expect_equal(sum(parscores(dmeas.algo, modelbmeas.algo)), 0)
})
test_that("parscores raises error if no successes are given", {
d = dmeas
d$success = NULL
expect_error(parscores(d, modelameas), "Need successes to calculate PAR scores!")
# same test with algorithm features
d.algo = dmeas.algo
d.algo$success = NULL
expect_error(parscores(d.algo, modelameas.algo), "Need successes to calculate PAR scores!")
})
test_that("parscores works without test split", {
fold = data.frame(id=1:10, a=rep.int(1, 10), b=rep.int(0, 10),
d=rep.int(F, 10), e=rep.int(T, 10))
d = list(data=fold, performance=c("a", "b"), success=c("d", "e"), ids=c("id"))
expect_equal(sum(parscores(d, modelameas)), 100)
expect_equal(sum(parscores(d, modelbmeas)), 0)
# same test with algorithm features
fold.algo = data.frame(id=rep.int(1:10, rep.int(2, 10)), p=rep.int(c(1, 0), 10),
s=rep.int(c(F, T), 10), f=rep.int(1, 20), a=rep.int(c("a", "b"), 10))
d.algo = list(data=fold.algo, performance=c("p"), success=c("s"), ids=c("id"),
algorithmFeatures=c("f"), algos=c("a"), algorithmNames=c("a", "b"))
expect_equal(sum(parscores(d.algo, modelameas.algo)), 100)
expect_equal(sum(parscores(d.algo, modelbmeas.algo)), 0)
})
test_that("parscores allows to specify factor", {
expect_equal(sum(parscores(dmeas, modelameas, 100)), 1000)
expect_equal(sum(parscores(dmeas, modelbmeas, 100)), 0)
# same test with algorithm features
expect_equal(sum(parscores(dmeas.algo, modelameas.algo, 100)), 1000)
expect_equal(sum(parscores(dmeas.algo, modelbmeas.algo, 100)), 0)
})
test_that("parscores allows to specify timeout", {
expect_equal(sum(parscores(dmeas, modelameas, timeout=5)), 500)
expect_equal(sum(parscores(dmeas, modelbmeas, timeout=5)), 0)
# same test with algorithm features
expect_equal(sum(parscores(dmeas.algo, modelameas.algo, timeout=5)), 500)
expect_equal(sum(parscores(dmeas.algo, modelbmeas.algo, timeout=5)), 0)
})
test_that("parscores takes feature costs into account", {
fold = data.frame(a=rep.int(1, 5), b=rep.int(0, 5),
d=rep.int(T, 5), e=rep.int(T, 5), c=rep.int(1, 5), c_cost=rep.int(2, 5))
d = list(data=rbind(cbind(id=1:5, fold), cbind(id=6:10, fold)),
test=list(1:5, 6:10), performance=c("a", "b"), ids=c("id"),
success=c("d", "e"), cost=c("c_cost"), features=c("c"))
expect_equal(sum(parscores(d, modelameas, timeout=1.5)), 150)
expect_equal(sum(parscores(d, modelameas, timeout=5)), 30)
expect_equal(sum(parscores(d, modelbmeas, timeout=2.5)), 20)
expect_equal(sum(parscores(d, modelameas, timeout=0.5, addCosts=F)), 50)
expect_equal(sum(parscores(d, modelameas, timeout=5, addCosts=F)), 10)
expect_equal(sum(parscores(d, modelbmeas, timeout=2.5, addCosts=F)), 0)
modela = modelameas
modelb = modelbmeas
attr(modela, "addCosts") = FALSE
attr(modelb, "addCosts") = FALSE
expect_equal(sum(parscores(d, modela, timeout=0.5)), 50)
expect_equal(sum(parscores(d, modela, timeout=5)), 10)
expect_equal(sum(parscores(d, modelb, timeout=2.5)), 0)
# same test with algorithm features
fold.algo = data.frame(p=rep.int(c(1, 0), 5),
s=rep.int(T, 10), c=rep.int(1, 10), c_cost=rep.int(2, 10),
f=rep.int(1, 10), a=rep.int(c("a", "b"), 5))
d.algo = list(data=rbind(cbind(id=rep.int(1:5, rep.int(2, 5)), fold.algo), cbind(id=rep.int(6:10, rep.int(2, 5)), fold.algo)),
test=list(1:5, 6:10), performance=c("p"), ids=c("id"),
success=c("s"), cost=c("c_cost"), features=c("c"), algos=c("a"),
algorithmFeatures=c("f"), algorithmNames=c("a", "b"))
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=1.5)), 150)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=5)), 30)
expect_equal(sum(parscores(d.algo, modelbmeas.algo, timeout=2.5)), 20)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=0.5, addCosts=F)), 50)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=5, addCosts=F)), 10)
expect_equal(sum(parscores(d.algo, modelbmeas.algo, timeout=2.5, addCosts=F)), 0)
modela.algo = modelameas.algo
modelb.algo = modelbmeas.algo
attr(modela.algo, "addCosts") = FALSE
attr(modelb.algo, "addCosts") = FALSE
expect_equal(sum(parscores(d.algo, modela.algo, timeout=0.5)), 50)
expect_equal(sum(parscores(d.algo, modela.algo, timeout=5)), 10)
expect_equal(sum(parscores(d.algo, modelb.algo, timeout=2.5)), 0)
})
test_that("parscores does not cost unused features", {
fold = data.frame(a=rep.int(1, 5), b=rep.int(0, 5),
d=rep.int(T, 5), e=rep.int(T, 5), c=rep.int(1, 5),
c_cost=rep.int(2, 5), f_cost=rep.int(1, 5))
d = list(data=rbind(cbind(id=1:5, fold), cbind(id=6:10, fold)),
test=list(1:5, 6:10), performance=c("a", "b"), ids=c("id"),
success=c("d", "e"), cost=c("c_cost", "f_cost"), features=c("c"))
expect_equal(sum(parscores(d, modelameas, timeout=1.5)), 150)
expect_equal(sum(parscores(d, modelameas, timeout=3.5)), 30)
# same test with algorithm features
fold.algo = data.frame(p=rep.int(c(1, 0), 5),
s=rep.int(T, 10), c=rep.int(1, 10), c_cost=rep.int(2, 10),
f_cost=rep.int(1, 10),
sf=rep.int(1, 10), a=rep.int(c("a", "b"), 5))
d.algo = list(data=rbind(cbind(id=rep.int(1:5, rep.int(2, 5)), fold.algo), cbind(id=rep.int(6:10, rep.int(2, 5)), fold.algo)),
test=list(1:5, 6:10), performance=c("p"), ids=c("id"),
success=c("s"), cost=c("c_cost", "f_cost"), features=c("c"), algos=c("a"),
algorithmFeatures=c("sf"), algorithmNames=c("a", "b"))
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=1.5)), 150)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=3.5)), 30)
})
test_that("parscores takes feature cost groups into account", {
fold = data.frame(a=rep.int(1, 5), e=rep.int(0, 5), f=rep.int(0, 5),
g=rep.int(0, 5), d=rep.int(T, 5), g1=rep.int(1, 5), g2=rep.int(1, 5))
groups = list(g1=c("e", "f"), g2=c("g"))
d = list(data=rbind(cbind(id=1:5, fold), cbind(id=6:10, fold)),
test=list(1:5, 6:10), performance=c("a"), ids=c("id"),
success=c("d"), costGroups=groups, cost=c("g1", "g2"), features=c("f", "g"))
e = d
e$features = c("f")
expect_equal(sum(parscores(d, modelameas, timeout=2.5)), 250)
expect_equal(sum(parscores(d, modelameas, timeout=1.5)), 150)
expect_equal(sum(parscores(d, modelameas, timeout=5)), 30)
expect_equal(sum(parscores(e, modelameas, timeout=1.5)), 150)
expect_equal(sum(parscores(e, modelameas, timeout=2.5)), 20)
# same test with algorithm features
fold.algo = data.frame(p=rep.int(1, 5), e=rep.int(0, 5), f=rep.int(0, 5),
g=rep.int(0, 5), d=rep.int(T, 5), g1=rep.int(1, 5), g2=rep.int(1, 5),
a=rep.int("a", 5), sf=rep.int(0, 5))
groups.algo = list(g1=c("e", "f"), g2=c("g"))
d.algo = list(data=rbind(cbind(id=1:5, fold.algo), cbind(id=6:10, fold.algo)),
test=list(1:5, 6:10), performance=c("p"), ids=c("id"), algos=c("a"),
success=c("d"), costGroups=groups.algo, cost=c("g1", "g2"), features=c("f", "g"),
algorithmFeatures=c("sf"), algorithmNames=c("a"))
e.algo = d.algo
e.algo$features = c("f")
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=2.5)), 250)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=1.5)), 150)
expect_equal(sum(parscores(d.algo, modelameas.algo, timeout=5)), 30)
expect_equal(sum(parscores(e.algo, modelameas.algo, timeout=1.5)), 150)
expect_equal(sum(parscores(e.algo, modelameas, timeout=2.5)), 20)
})
test_that("parscores works for test splits", {
model = function(data) {
data.frame(id=data$data$id, algorithm="a", score=1, iteration=1)
}
class(model) = "llama.model"
attr(model, "hasPredictions") = FALSE
attr(model, "addCosts") = TRUE
d = dmeas
d$test = list(1:5)
expect_equal(sum(parscores(d, model)), 50)
# same test with algorithm features
d.algo = dmeas.algo
d.algo$test = list(1:10)
expect_equal(sum(parscores(d.algo, model)), 50)
})
test_that("parscores works for test splits that repeat", {
i = 0
model = function(data) {
i <<- i + 1
data.frame(id=data$data$id, algorithm="a", score=1, iteration=i)
}
class(model) = "llama.model"
attr(model, "hasPredictions") = FALSE
attr(model, "addCosts") = TRUE
d = dmeas
d$test = list(c(1,2), c(1,2), c(2,3))
expect_equal(sum(parscores(d, model)), 60)
# same test with algorithm features
d.algo = dmeas.algo
d.algo$test = list(c(1,2,3,4), c(1,2,3,4), c(3,4,5,6))
expect_equal(sum(parscores(d.algo, model)), 60)
})
test_that("parscores works with NA predictions", {
nasmeas = data.frame(algorithm=rep.int(NA, 5), score=0, iteration=1)
asmeas = data.frame(algorithm=rep.int("b", 5), score=1, iteration=1)
model = list(predictions=rbind(cbind(nasmeas, id=1:5), cbind(asmeas, id=6:10)))
class(model) = "llama.model"
attr(model, "hasPredictions") = TRUE
pars = parscores(dmeas, model)
expect_equal(length(pars), 10)
expect_equal(sum(pars), 50)
# same test with algorithm features
pars = parscores(dmeas.algo, model)
expect_equal(length(pars), 10)
expect_equal(sum(pars), 50)
})
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.