{
  "levers": {
    "units": [
      [
        "v1",
        "s1"
      ],
      [
        "v1",
        "s42"
      ],
      [
        "v2",
        "s1"
      ],
      [
        "v2",
        "s42"
      ],
      [
        "v3",
        "s1"
      ],
      [
        "v3",
        "s42"
      ],
      [
        "v4",
        "s1"
      ],
      [
        "v4",
        "s42"
      ],
      [
        "v5",
        "s1"
      ],
      [
        "v5",
        "s42"
      ],
      [
        "v6",
        "s1"
      ],
      [
        "v6",
        "s42"
      ]
    ],
    "arms": [
      "base",
      "mthspl",
      "aux",
      "both"
    ],
    "cells": {
      "v1/s42/base": {
        "test/acc@1": 0.9274891774891776,
        "val/acc@1": 0.8851599370739381,
        "test/strength_acc": 0.9648268398268398,
        "test_mthspl/acc@1": 0.6643029167781643
      },
      "v1/s42/mthspl": {
        "test/acc@1": 0.9312770562770564,
        "val/acc@1": 0.8746722600943891,
        "test/strength_acc": 0.9675324675324676,
        "test_mthspl/acc@1": 0.8431897243778432
      },
      "v1/s1/base": {
        "test/acc@1": 0.9356060606060606,
        "val/acc@1": 0.8783429470372313,
        "test/strength_acc": 0.9686147186147186,
        "test_mthspl/acc@1": 0.6846400856301846
      },
      "v1/s1/mthspl": {
        "test/acc@1": 0.926948051948052,
        "val/acc@1": 0.8820136339800734,
        "test/strength_acc": 0.9686147186147186,
        "test_mthspl/acc@1": 0.8196414236018197
      },
      "v2/s42/base": {
        "test/acc@1": 0.906392694063927,
        "val/acc@1": 0.9142699115044248,
        "test/strength_acc": 0.9514840182648402,
        "test_mthspl/acc@1": 0.6395109395109395
      },
      "v2/s42/mthspl": {
        "test/acc@1": 0.9029680365296804,
        "val/acc@1": 0.9164823008849556,
        "test/strength_acc": 0.9526255707762558,
        "test_mthspl/acc@1": 0.836036036036036
      },
      "v2/s1/base": {
        "test/acc@1": 0.9058219178082192,
        "val/acc@1": 0.915929203539823,
        "test/strength_acc": 0.9549086757990868,
        "test_mthspl/acc@1": 0.7018018018018019
      },
      "v2/s1/mthspl": {
        "test/acc@1": 0.9023972602739726,
        "val/acc@1": 0.9236725663716814,
        "test/strength_acc": 0.952054794520548,
        "test_mthspl/acc@1": 0.8196911196911197
      },
      "v6/s42/base": {
        "test/acc@1": 0.9156184486373166,
        "val/acc@1": 0.8779904306220095,
        "test/strength_acc": 0.9554507337526206,
        "test_mthspl/acc@1": 0.7782540919719407
      },
      "v6/s1/base": {
        "test/acc@1": 0.9234800838574424,
        "val/acc@1": 0.8845693779904307,
        "test/strength_acc": 0.9627882599580712,
        "test_mthspl/acc@1": 0.8244349181605611
      },
      "v6/s1/mthspl": {
        "test/acc@1": 0.9287211740041929,
        "val/acc@1": 0.8977272727272727,
        "test/strength_acc": 0.9612159329140462,
        "test_mthspl/acc@1": 0.8801636788776306
      },
      "v2/s1/aux": {
        "test/acc@1": 0.9041095890410958,
        "val/acc@1": 0.913716814159292,
        "test/strength_acc": 0.9537671232876712,
        "test_mthspl/acc@1": 0.7054054054054054
      },
      "v2/s1/both": {
        "test/acc@1": 0.9041095890410958,
        "val/acc@1": 0.922566371681416,
        "test/strength_acc": 0.9509132420091324,
        "test_mthspl/acc@1": 0.8402831402831403
      },
      "v3/s42/aux": {
        "test/acc@1": 0.9074399531341536,
        "val/acc@1": 0.9139254385964912,
        "test/strength_acc": 0.9648506151142356,
        "test_mthspl/acc@1": 0.7595050215208035
      },
      "v3/s42/both": {
        "test/acc@1": 0.905096660808436,
        "val/acc@1": 0.918859649122807,
        "test/strength_acc": 0.9595782073813708,
        "test_mthspl/acc@1": 0.8610114777618364
      },
      "v3/s1/aux": {
        "test/acc@1": 0.9027533684827184,
        "val/acc@1": 0.9144736842105264,
        "test/strength_acc": 0.9595782073813708,
        "test_mthspl/acc@1": 0.7465925394548063
      },
      "v3/s1/both": {
        "test/acc@1": 0.906854130052724,
        "val/acc@1": 0.918859649122807,
        "test/strength_acc": 0.961335676625659,
        "test_mthspl/acc@1": 0.8726685796269728
      },
      "v4/s42/aux": {
        "test/acc@1": 0.8739991571849979,
        "val/acc@1": 0.9057788944723618,
        "test/strength_acc": 0.9624947324062368,
        "test_mthspl/acc@1": 0.6501857307544512
      },
      "v4/s42/both": {
        "test/acc@1": 0.8832701222081754,
        "val/acc@1": 0.9013819095477388,
        "test/strength_acc": 0.965023177412558,
        "test_mthspl/acc@1": 0.7782759062379916
      },
      "v4/s1/aux": {
        "test/acc@1": 0.8820058997050148,
        "val/acc@1": 0.9032663316582916,
        "test/strength_acc": 0.967551622418879,
        "test_mthspl/acc@1": 0.6824644549763034
      },
      "v4/s1/both": {
        "test/acc@1": 0.8841129372102824,
        "val/acc@1": 0.8969849246231156,
        "test/strength_acc": 0.9624947324062368,
        "test_mthspl/acc@1": 0.7789163571154093
      },
      "v5/s42/aux": {
        "test/acc@1": 0.9213547646383468,
        "val/acc@1": 0.8992287917737789,
        "test/strength_acc": 0.965556831228473,
        "test_mthspl/acc@1": 0.6893288943402839
      },
      "v5/s42/both": {
        "test/acc@1": 0.9064293915040184,
        "val/acc@1": 0.9017994858611824,
        "test/strength_acc": 0.9557979334098736,
        "test_mthspl/acc@1": 0.8072542491676888
      },
      "v5/s1/aux": {
        "test/acc@1": 0.9115958668197476,
        "val/acc@1": 0.8956298200514139,
        "test/strength_acc": 0.9626865671641792,
        "test_mthspl/acc@1": 0.6961626073243385
      },
      "v5/s1/both": {
        "test/acc@1": 0.9001148105625718,
        "val/acc@1": 0.897172236503856,
        "test/strength_acc": 0.9563719862227325,
        "test_mthspl/acc@1": 0.7997196425442439
      },
      "v6/s42/aux": {
        "test/acc@1": 0.9145702306079664,
        "val/acc@1": 0.8779904306220095,
        "test/strength_acc": 0.9544025157232704,
        "test_mthspl/acc@1": 0.7780592361652378
      },
      "v6/s42/both": {
        "test/acc@1": 0.9281970649895178,
        "val/acc@1": 0.8827751196172249,
        "test/strength_acc": 0.9622641509433962,
        "test_mthspl/acc@1": 0.8750974279033515
      },
      "v6/s1/aux": {
        "test/acc@1": 0.9224318658280922,
        "val/acc@1": 0.8833732057416268,
        "test/strength_acc": 0.9622641509433962,
        "test_mthspl/acc@1": 0.8250194855806703
      },
      "v6/s1/both": {
        "test/acc@1": 0.9276729559748428,
        "val/acc@1": 0.8905502392344498,
        "test/strength_acc": 0.9612159329140462,
        "test_mthspl/acc@1": 0.8758768511301637
      },
      "v3/s42/base": {
        "test/acc@1": 0.9080257762155828,
        "val/acc@1": 0.9144736842105264,
        "test/strength_acc": 0.964264792032806,
        "test_mthspl/acc@1": 0.7604017216642754
      },
      "v3/s42/mthspl": {
        "test/acc@1": 0.9056824838898652,
        "val/acc@1": 0.9205043859649122,
        "test/strength_acc": 0.9601640304628002,
        "test_mthspl/acc@1": 0.8588593974175036
      },
      "v3/s1/base": {
        "test/acc@1": 0.9045108377270064,
        "val/acc@1": 0.9144736842105264,
        "test/strength_acc": 0.961335676625659,
        "test_mthspl/acc@1": 0.7471305595408895
      },
      "v3/s1/mthspl": {
        "test/acc@1": 0.9091974223784416,
        "val/acc@1": 0.9237938596491228,
        "test/strength_acc": 0.9595782073813708,
        "test_mthspl/acc@1": 0.8760760401721664
      },
      "v4/s42/base": {
        "test/acc@1": 0.8731563421828908,
        "val/acc@1": 0.9076633165829144,
        "test/strength_acc": 0.9612305099030762,
        "test_mthspl/acc@1": 0.6664531830408608
      },
      "v4/s42/mthspl": {
        "test/acc@1": 0.8777918246944796,
        "val/acc@1": 0.9038944723618092,
        "test/strength_acc": 0.9646017699115044,
        "test_mthspl/acc@1": 0.7757141027283208
      },
      "v4/s1/base": {
        "test/acc@1": 0.8820058997050148,
        "val/acc@1": 0.9038944723618092,
        "test/strength_acc": 0.9603876949009692,
        "test_mthspl/acc@1": 0.646599205840912
      },
      "v4/s1/mthspl": {
        "test/acc@1": 0.8929624947324062,
        "val/acc@1": 0.8994974874371859,
        "test/strength_acc": 0.9641803624104508,
        "test_mthspl/acc@1": 0.7836556936083002
      },
      "v5/s42/base": {
        "test/acc@1": 0.920780711825488,
        "val/acc@1": 0.902827763496144,
        "test/strength_acc": 0.965556831228473,
        "test_mthspl/acc@1": 0.6909059050289119
      },
      "v5/s42/mthspl": {
        "test/acc@1": 0.907577497129736,
        "val/acc@1": 0.900771208226221,
        "test/strength_acc": 0.9552238805970148,
        "test_mthspl/acc@1": 0.8098826003154022
      },
      "v5/s1/base": {
        "test/acc@1": 0.9121699196326062,
        "val/acc@1": 0.8966580976863753,
        "test/strength_acc": 0.963260619977038,
        "test_mthspl/acc@1": 0.6977396180129666
      },
      "v5/s1/mthspl": {
        "test/acc@1": 0.904707233065442,
        "val/acc@1": 0.9023136246786632,
        "test/strength_acc": 0.9580941446613088,
        "test_mthspl/acc@1": 0.8123357280532679
      },
      "v6/s42/mthspl": {
        "test/acc@1": 0.929769392033543,
        "val/acc@1": 0.8845693779904307,
        "test/strength_acc": 0.9627882599580712,
        "test_mthspl/acc@1": 0.8801636788776306
      },
      "v1/s42/aux": {
        "test/acc@1": 0.928030303030303,
        "val/acc@1": 0.8825380178290508,
        "test/strength_acc": 0.9659090909090908,
        "test_mthspl/acc@1": 0.6648381054321648
      },
      "v1/s42/both": {
        "test/acc@1": 0.9264069264069263,
        "val/acc@1": 0.8694284216046145,
        "test/strength_acc": 0.9680735930735932,
        "test_mthspl/acc@1": 0.8398447952903398
      },
      "v1/s1/aux": {
        "test/acc@1": 0.935064935064935,
        "val/acc@1": 0.8783429470372313,
        "test/strength_acc": 0.9686147186147186,
        "test_mthspl/acc@1": 0.6854428686111854
      },
      "v1/s1/both": {
        "test/acc@1": 0.926948051948052,
        "val/acc@1": 0.8820136339800734,
        "test/strength_acc": 0.9675324675324676,
        "test_mthspl/acc@1": 0.8197752207653197
      },
      "v2/s42/aux": {
        "test/acc@1": 0.9052511415525114,
        "val/acc@1": 0.913716814159292,
        "test/strength_acc": 0.9526255707762558,
        "test_mthspl/acc@1": 0.6433719433719434
      },
      "v2/s42/both": {
        "test/acc@1": 0.8961187214611872,
        "val/acc@1": 0.9131637168141592,
        "test/strength_acc": 0.9463470319634704,
        "test_mthspl/acc@1": 0.8368082368082368
      }
    },
    "main": {
      "test/acc@1": {
        "data": {
          "mean": -0.0003472356548298953,
          "sd": 0.008180487109557643,
          "ci95": [
            -0.0055449042363332045,
            0.004850432926673413
          ],
          "n_pos": 6,
          "n_runs": 12
        },
        "head": {
          "mean": -0.0012966399770786379,
          "sd": 0.002005278746058973,
          "ci95": [
            -0.002570741897411678,
            -2.2538056745597702e-05
          ],
          "n_pos": 1,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.0015181475106823734,
          "sd": 0.0038818627474382973,
          "ci95": [
            -0.003984582060343218,
            0.0009482870389784712
          ],
          "n_pos": 4,
          "n_runs": 12
        }
      },
      "val/acc@1": {
        "data": {
          "mean": 0.0019680746438600996,
          "sd": 0.006371810691946444,
          "ci95": [
            -0.002080408019491688,
            0.006016557307211887
          ],
          "n_pos": 9,
          "n_runs": 12
        },
        "head": {
          "mean": -0.0020261970274191485,
          "sd": 0.0012355213606362397,
          "ci95": [
            -0.0028112151408823775,
            -0.0012411789139559197
          ],
          "n_pos": 0,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.0016737880540404666,
          "sd": 0.0028435055169537924,
          "ci95": [
            -0.0034804774465992984,
            0.00013290133851836497
          ],
          "n_pos": 2,
          "n_runs": 12
        }
      },
      "test/recall@5": {
        "data": {
          "mean": 0.0016848915153185873,
          "sd": 0.004588388916321771,
          "ci95": [
            -0.0012304513720529822,
            0.004600234402690157
          ],
          "n_pos": 8,
          "n_runs": 12
        },
        "head": {
          "mean": -0.0010626612893415988,
          "sd": 0.0021156849509279653,
          "ci95": [
            -0.0024069124385121396,
            0.00028158985982894183
          ],
          "n_pos": 5,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.001685525515102056,
          "sd": 0.004285531722783775,
          "ci95": [
            -0.004408440824695823,
            0.0010373897944917115
          ],
          "n_pos": 5,
          "n_runs": 12
        }
      },
      "test/strength_acc": {
        "data": {
          "mean": -0.0012828685507034383,
          "sd": 0.004528948311505397,
          "ci95": [
            -0.004160444424939731,
            0.0015947073235328547
          ],
          "n_pos": 3,
          "n_runs": 12
        },
        "head": {
          "mean": -0.00014723469843505588,
          "sd": 0.0013105289239225965,
          "ci95": [
            -0.0009799106652568115,
            0.0006854412683866998
          ],
          "n_pos": 5,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.0013265319107999367,
          "sd": 0.00344245172996043,
          "ci95": [
            -0.0035137761372778607,
            0.0008607123156779875
          ],
          "n_pos": 4,
          "n_runs": 12
        }
      },
      "test/dose_form_acc": {
        "data": {
          "mean": 0.000632149692021485,
          "sd": 0.0039009403164637574,
          "ci95": [
            -0.0018464062484327324,
            0.0031107056324757025
          ],
          "n_pos": 6,
          "n_runs": 12
        },
        "head": {
          "mean": -6.395343969016014e-05,
          "sd": 0.0012290377139917725,
          "ci95": [
            -0.0008448520128367562,
            0.000716945133456436
          ],
          "n_pos": 6,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.0001615497385739073,
          "sd": 0.0032262364998453916,
          "ci95": [
            -0.002211416435650381,
            0.0018883169585025665
          ],
          "n_pos": 4,
          "n_runs": 12
        }
      },
      "test/ingredient_acc": {
        "data": {
          "mean": -0.00015275917690489582,
          "sd": 0.0031908186517889472,
          "ci95": [
            -0.002180122295208859,
            0.0018746039413990675
          ],
          "n_pos": 6,
          "n_runs": 12
        },
        "head": {
          "mean": -0.0008416585346382038,
          "sd": 0.00114011005472614,
          "ci95": [
            -0.0015660547877638343,
            -0.00011726228151257337
          ],
          "n_pos": 3,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.001516416690350103,
          "sd": 0.004279272396831621,
          "ci95": [
            -0.004235354987156345,
            0.001202521606456139
          ],
          "n_pos": 5,
          "n_runs": 12
        }
      },
      "test_mthspl/acc@1": {
        "data": {
          "mean": 0.12301624451965555,
          "sd": 0.03627421530319475,
          "ci95": [
            0.09996855228873446,
            0.14606393675057666
          ],
          "n_pos": 12,
          "n_runs": 12
        },
        "head": {
          "mean": 0.0005968336180349615,
          "sd": 0.006797214684301545,
          "ci95": [
            -0.003721939669322191,
            0.004915606905392114
          ],
          "n_pos": 5,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.0028398904231276465,
          "sd": 0.01479588275322728,
          "ci95": [
            -0.012240809206094187,
            0.006561028359838893
          ],
          "n_pos": 3,
          "n_runs": 12
        }
      },
      "val_mthspl/acc@1": {
        "data": {
          "mean": 0.10454294328048021,
          "sd": 0.022750363909950423,
          "ci95": [
            0.09008795421578741,
            0.118997932345173
          ],
          "n_pos": 12,
          "n_runs": 12
        },
        "head": {
          "mean": -0.0028952801779660634,
          "sd": 0.00363382273331055,
          "ci95": [
            -0.005204116558823346,
            -0.0005864437971087808
          ],
          "n_pos": 3,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.00600201422022766,
          "sd": 0.01077511628169545,
          "ci95": [
            -0.01284824264273558,
            0.0008442142022802605
          ],
          "n_pos": 5,
          "n_runs": 12
        }
      },
      "test_mthspl/strength_acc": {
        "data": {
          "mean": 0.07642183213561264,
          "sd": 0.034239355350168615,
          "ci95": [
            0.054667036953969475,
            0.09817662731725581
          ],
          "n_pos": 12,
          "n_runs": 12
        },
        "head": {
          "mean": 0.0014108635894727283,
          "sd": 0.006693904284160298,
          "ci95": [
            -0.0028422689586759574,
            0.005663996137621414
          ],
          "n_pos": 7,
          "n_runs": 12
        },
        "interaction": {
          "mean": -0.004706260647639395,
          "sd": 0.01341744315285216,
          "ci95": [
            -0.013231354787596728,
            0.003818833492317938
          ],
          "n_pos": 2,
          "n_runs": 12
        }
      }
    }
  },
  "kfold": {
    "folds": [
      "fold0",
      "fold1",
      "fold2",
      "fold3",
      "fold4",
      "fold5",
      "fold6"
    ],
    "acc@1": [
      0.9323593073593074,
      0.8637440758293838,
      0.8847980997624703,
      0.8830687830687831,
      0.8982376350198976,
      0.8995433789954338,
      0.8947945205479452
    ],
    "n": [
      1848,
      1688,
      1684,
      1890,
      1759,
      1533,
      1825
    ],
    "oof": {
      "n": 12227,
      "correct": 10932,
      "acc@1": 0.8940868569559173,
      "wilson95": [
        0.8885079551700271,
        0.8994182002958405
      ],
      "recall@5": 0.9745644884272512,
      "coverage": 0.850751461174506,
      "n_va_strings": 14372
    },
    "lofo_p99": [
      0.9892125134843581,
      0.9976275207591934,
      0.9887179487179487,
      0.9867617107942973,
      0.9922394678492239,
      0.9862227324913893,
      0.9875776397515528
    ],
    "lofo_p95": [
      0.9671826625386997,
      0.9553767373811266,
      0.9218543046357616,
      0.9439601494396015,
      0.9397821909032671,
      0.9578544061302682,
      0.9586296056884293
    ]
  }
}