Benchmarking POSSA.

This page serve simply to provide some benchmarks for the POSSA::sim() function’s internal mechanism. The rather speedy completion of calculations with empty functions highlights that POSSA’s overhead is minimal, the bulk of the time consumed in simulations is due to the user-given sampling and testing functions. Hence, attention should be paid to minimize the execution time of these user functions.

library('POSSA')

Some “empty” sampling and testing functions:

# minimal output
sampleEmpty = function(sample_size) {
  list(
    sample1 = rep(1, sample_size),
    sample2_h0 = rep(1, sample_size),
    sample2_h1 = rep(1, sample_size)
  )
}
testEmpty = function(sample1, sample2_h0, sample2_h1) {
  c(
    p_h0 = 0.5,
    p_h1 = 0.5
  )
}

# multiple (five) hypotheses
sampleEmptyMulti = function(sample_size) {
  list(
    sample1 = rep(1, sample_size),
    sample2_h0 = rep(1, sample_size),
    sample2_h1 = rep(1, sample_size),
    sample3_h0 = rep(1, sample_size),
    sample3_h1 = rep(1, sample_size),
    sample4_h0 = rep(1, sample_size),
    sample4_h1 = rep(1, sample_size),
    sample5_h0 = rep(1, sample_size),
    sample5_h1 = rep(1, sample_size)
  )
}
testEmptyMulti = function(sample1, sample2_h0, sample2_h1, sample3_h0, sample3_h1, sample4_h0, sample4_h1, sample5_h0, sample5_h1) {
  c(
    p_h0 = 0.5,
    p_h1 = 0.5,
    p_test_2_h0 = 0.5,
    p_test_2_h1 = 0.5,
    p_test_3_h0 = 0.5,
    p_test_3_h1 = 0.5,
    p_test_4_h0 = 0.5,
    p_test_4_h1 = 0.5,
    p_test_5_h0 = 0.5,
    p_test_5_h1 = 0.5
  )
}

# to check that they work correctly:
# do.call(testEmpty, sampleEmpty(100))
# do.call(testEmptyMulti, sampleEmptyMulti(100))

Now the benchmarking. (Caution: actually running all this takes a while.)

bm_results = microbenchmark::microbenchmark(
  fixed_120_obs =
    {
      sim(
        fun_obs = sampleEmpty,
        n_obs = 120,
        fun_test = testEmpty,
        n_iter = 10000,
        hush = TRUE
      )
    },
  sequential_120_obs =
    {
      sim(
        fun_obs = sampleEmpty,
        n_obs = c(40, 80, 120),
        fun_test = testEmpty,
        n_iter = 10000,
        hush = TRUE
      )
    },
  sequential_1200_obs =
    {
      sim(
        fun_obs = sampleEmpty,
        n_obs = c(400, 800, 1200),
        fun_test = testEmpty,
        n_iter = 10000,
        hush = TRUE
      )
    },
  sequential_120_obs_multi =
    {
      sim(
        fun_obs = sampleEmptyMulti,
        n_obs = c(40, 80, 120),
        fun_test = testEmptyMulti,
        n_iter = 10000,
        hush = TRUE
      )
    },
  sequential_1200_obs_multi =
    {
      sim(
        fun_obs = sampleEmptyMulti,
        n_obs = c(400, 800, 1200),
        fun_test = testEmptyMulti,
        n_iter = 10000,
        hush = TRUE
      )
    },
  times = 5
)
print(bm_results)

#> Unit: milliseconds
#>                       expr      min        lq       mean    median         uq        max neval
#>              fixed_120_obs  148.011  154.4439   172.5028  156.2547   175.3121   228.4922     5
#>         sequential_120_obs 1216.329 1499.6831  1512.8659 1524.3713  1571.4734  1752.4722     5
#>        sequential_1200_obs 3134.309 3218.3147  3496.8585 3685.3403  3711.9782  3734.3506     5
#>   sequential_120_obs_multi 3270.239 3387.0647  3794.1274 3739.9446  3881.3312  4692.0578     5
#>  sequential_1200_obs_multi 8980.836 9861.9136 10112.6613 9925.8795 10623.9350 11170.7427     5

Voila, even the extremely large-sample multiple-hypothesis sequential design takes only about 10 seconds (on my pretty much low-end PC).