{ "schema_version": 1, "model": "independent_binomial_null_winner_selection", "source": { "title": "The reusable holdout: Preserving validity in adaptive data analysis", "authors": [ "Cynthia Dwork", "Vitaly Feldman", "Moritz Hardt", "Toniann Pitassi", "Omer Reingold", "Aaron Roth" ], "publication": "Science", "year": 2015, "doi": "10.1126/science.aaa9375", "url": "https://research.ibm.com/publications/the-reusable-holdout-preserving-validity-in-adaptive-data-analysis" }, "parameters": { "n_evaluation_items": 64, "null_accuracy": 0.5, "alpha": 0.05, "k_values": [ 1, 2, 5, 10, 20, 50, 100, 200 ], "selection_rule": "Select one candidate with the largest reused-evaluation count; ties may be broken arbitrarily.", "reused_test_rule": "Apply an exact inclusive upper-tail Binomial(n, p0) p-value to the selected reused count.", "bonferroni_rule": "Apply the same exact test at alpha/K before selecting any passing winner.", "fresh_test_rule": "Evaluate the already selected candidate once on a new independent Binomial(n, p0) count, with no re-selection, at alpha." }, "definitions": { "reused_selected_false_accept_rate": "P(max_i X_i reaches the nominal exact-binomial critical count) under K independent null candidates.", "bonferroni_familywise_false_accept_rate": "P(max_i X_i reaches the exact-binomial critical count for alpha/K) under K independent null candidates.", "fresh_selected_false_accept_rate": "P(Y_I reaches the nominal critical count), where I is selected from reused counts and all fresh Y_i are independent of that selection.", "expected_selected_reused_accuracy": "E[max_i X_i / n] on the reused evaluation.", "expected_fresh_selected_accuracy": "E[Y_I / n] on the untouched evaluation; under these assumptions it equals p0." }, "assumptions": [ "Every candidate is null: its true accuracy is p0.", "Candidate reused-evaluation counts X_i are mutually independent Binomial(n, p0) variables.", "Fresh counts Y_i are mutually independent Binomial(n, p0) variables and independent of every reused count X_i.", "The candidate index is selected only from the reused counts, never from fresh results.", "This is a finite model-selection special case, not a model of general sequential adaptivity.", "This calculation does not train an AI system, run a human experiment, or reimplement the private reusable-holdout mechanism.", "The guarantees shown rely on valid exact p-values and the stated independence; they are not asserted for dependent candidates or misspecified nulls." ], "rows": [ { "k": 1, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.029970594783499616, "bonferroni_cutoff": 0.05, "bonferroni_critical_count": 40, "bonferroni_critical_accuracy": 0.625, "bonferroni_single_candidate_attainable_rate": 0.029970594783499616, "bonferroni_familywise_false_accept_rate": 0.029970594783499616, "expected_selected_reused_count": 32.0, "expected_selected_reused_accuracy": 0.5, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 2, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.059042953015322495, "bonferroni_cutoff": 0.025, "bonferroni_critical_count": 41, "bonferroni_critical_accuracy": 0.640625, "bonferroni_single_candidate_attainable_rate": 0.0163828795494116, "bonferroni_familywise_false_accept_rate": 0.03249736035649267, "expected_selected_reused_count": 34.25235494944048, "expected_selected_reused_accuracy": 0.5351930460850075, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 5, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.1411358052744593, "bonferroni_cutoff": 0.01, "bonferroni_critical_count": 42, "bonferroni_critical_accuracy": 0.65625, "bonferroni_single_candidate_attainable_rate": 0.008429095022140565, "bonferroni_familywise_false_accept_rate": 0.04144094232598409, "expected_selected_reused_count": 36.64110698991029, "expected_selected_reused_accuracy": 0.5725172967173483, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 10, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.26235229501844853, "bonferroni_cutoff": 0.005, "bonferroni_critical_count": 43, "bonferroni_critical_accuracy": 0.671875, "bonferroni_single_candidate_attainable_rate": 0.004073451114349284, "bonferroni_familywise_false_accept_rate": 0.03999587932205039, "expected_selected_reused_count": 38.1352500090191, "expected_selected_reused_accuracy": 0.5958632813909235, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 20, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.45587586333545, "bonferroni_cutoff": 0.0025, "bonferroni_critical_count": 44, "bonferroni_critical_accuracy": 0.6875, "bonferroni_single_candidate_attainable_rate": 0.0018449821382700243, "bonferroni_familywise_false_accept_rate": 0.03625999420724568, "expected_selected_reused_count": 39.4370390585393, "expected_selected_reused_accuracy": 0.6162037352896765, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 50, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.7816038502027219, "bonferroni_cutoff": 0.001, "bonferroni_critical_count": 45, "bonferroni_critical_accuracy": 0.703125, "bonferroni_single_candidate_attainable_rate": 0.0007813946724140139, "bonferroni_familywise_false_accept_rate": 0.038331041976929124, "expected_selected_reused_count": 40.94025325288072, "expected_selected_reused_accuracy": 0.6396914570762613, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 100, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.9523031217537249, "bonferroni_cutoff": 0.0005, "bonferroni_critical_count": 46, "bonferroni_critical_accuracy": 0.71875, "bonferroni_single_candidate_attainable_rate": 0.0003086891320335649, "bonferroni_familywise_false_accept_rate": 0.030401953709062724, "expected_selected_reused_count": 41.952781657639626, "expected_selected_reused_accuracy": 0.6555122134006192, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 }, { "k": 200, "nominal_critical_count": 40, "nominal_critical_accuracy": 0.625, "nominal_single_candidate_attainable_rate": 0.029970594783499616, "reused_selected_false_accept_rate": 0.9977250078055601, "bonferroni_cutoff": 0.00025, "bonferroni_critical_count": 47, "bonferroni_critical_accuracy": 0.734375, "bonferroni_single_candidate_attainable_rate": 0.0001134411914416403, "bonferroni_familywise_false_accept_rate": 0.02243405382058313, "expected_selected_reused_count": 42.8818831898143, "expected_selected_reused_accuracy": 0.6700294248408485, "fresh_critical_count": 40, "fresh_critical_accuracy": 0.625, "fresh_selected_false_accept_rate": 0.029970594783499616, "expected_fresh_selected_accuracy": 0.5 } ] }