{ "complete": true, "n_completed": 320, "n_expected": 320, "precision_mismatch_count": 0, "primary_eligible_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "interpolation_rates": { "all_fits": { "width": [ { "value": 256, "n_fits": 160, "n_interpolators": 96, "share_interpolating": 0.6 }, { "value": 1024, "n_fits": 160, "n_interpolators": 136, "share_interpolating": 0.85 } ], "init_scale": [ { "value": 0.5, "n_fits": 160, "n_interpolators": 93, "share_interpolating": 0.58125 }, { "value": 2.0, "n_fits": 160, "n_interpolators": 139, "share_interpolating": 0.86875 } ], "lr": [ { "value": 0.003, "n_fits": 160, "n_interpolators": 112, "share_interpolating": 0.7 }, { "value": 0.01, "n_fits": 160, "n_interpolators": 120, "share_interpolating": 0.75 } ] }, "primary_eligible_datasets": { "width": [ { "value": 256, "n_fits": 120, "n_interpolators": 90, "share_interpolating": 0.75 }, { "value": 1024, "n_fits": 120, "n_interpolators": 116, "share_interpolating": 0.9666666666666667 } ], "init_scale": [ { "value": 0.5, "n_fits": 120, "n_interpolators": 89, "share_interpolating": 0.7416666666666667 }, { "value": 2.0, "n_fits": 120, "n_interpolators": 117, "share_interpolating": 0.975 } ], "lr": [ { "value": 0.003, "n_fits": 120, "n_interpolators": 100, "share_interpolating": 0.8333333333333334 }, { "value": 0.01, "n_fits": 120, "n_interpolators": 106, "share_interpolating": 0.8833333333333333 } ] } }, "datasets": [ { "dataset_seed": 1000, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 16, "precision_mismatch_count": 0, "n_ce_target": 13, "eligible_primary": true, "max_training_errors": 0, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.3387736646077061, 0.3745299707371509 ] }, { "dataset_seed": 1001, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 12, "precision_mismatch_count": 0, "n_ce_target": 12, "eligible_primary": true, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.3532622912269251, 0.39745968590654607 ] }, { "dataset_seed": 1002, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 14, "precision_mismatch_count": 0, "n_ce_target": 14, "eligible_primary": true, "max_training_errors": 3, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.35760397288434487, 0.3914864430193412 ] }, { "dataset_seed": 1003, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 6, "precision_mismatch_count": 0, "n_ce_target": 6, "eligible_primary": false, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.2688976006419121, 0.27500718422277737 ] }, { "dataset_seed": 1004, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 16, "precision_mismatch_count": 0, "n_ce_target": 16, "eligible_primary": true, "max_training_errors": 0, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.28663592934217375, 0.3001142540825504 ] }, { "dataset_seed": 1005, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 9, "precision_mismatch_count": 0, "n_ce_target": 7, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.2655884694271686, 0.27477345777498463 ] }, { "dataset_seed": 1006, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 11, "precision_mismatch_count": 0, "n_ce_target": 2, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.2993774186974182, 0.3358711371749633 ] }, { "dataset_seed": 1007, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 15, "precision_mismatch_count": 0, "n_ce_target": 15, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.29761980911912667, 0.3088573522272593 ] }, { "dataset_seed": 1008, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 3, "precision_mismatch_count": 0, "n_ce_target": 0, "eligible_primary": false, "max_training_errors": 5, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.35502704757952397, 0.3823274709378429 ] }, { "dataset_seed": 1009, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 15, "precision_mismatch_count": 0, "n_ce_target": 15, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": false, "gap_range_interpolators": [ 0.291922739522965, 0.3143188977268409 ] }, { "dataset_seed": 1010, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 7, "precision_mismatch_count": 0, "n_ce_target": 6, "eligible_primary": false, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.35029464233932617, 0.3707390834221462 ] }, { "dataset_seed": 1011, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 14, "precision_mismatch_count": 0, "n_ce_target": 14, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.29229042911904485, 0.3038045838060479 ] }, { "dataset_seed": 1012, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 14, "precision_mismatch_count": 0, "n_ce_target": 14, "eligible_primary": true, "max_training_errors": 4, "flips_equal_label_changes_all_interpolators": false, "gap_range_interpolators": [ 0.32963878883337994, 0.400704081458891 ] }, { "dataset_seed": 1013, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 14, "precision_mismatch_count": 0, "n_ce_target": 14, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.22598583535189715, 0.23259270700867968 ] }, { "dataset_seed": 1014, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 7, "precision_mismatch_count": 0, "n_ce_target": 2, "eligible_primary": false, "max_training_errors": 3, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.3095940493224343, 0.31401562517278236 ] }, { "dataset_seed": 1015, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 16, "precision_mismatch_count": 0, "n_ce_target": 16, "eligible_primary": true, "max_training_errors": 0, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.23263940389334564, 0.2558316219175501 ] }, { "dataset_seed": 1016, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 13, "precision_mismatch_count": 0, "n_ce_target": 13, "eligible_primary": true, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.3390883109424204, 0.3582594460218916 ] }, { "dataset_seed": 1017, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 3, "precision_mismatch_count": 0, "n_ce_target": 0, "eligible_primary": false, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.2908649544220148, 0.30210983147811515 ] }, { "dataset_seed": 1018, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 13, "precision_mismatch_count": 0, "n_ce_target": 13, "eligible_primary": true, "max_training_errors": 2, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.314790469140067, 0.35547299025148044 ] }, { "dataset_seed": 1019, "n_completed": 16, "n_expected": 16, "complete": true, "n_interpolators": 14, "precision_mismatch_count": 0, "n_ce_target": 14, "eligible_primary": true, "max_training_errors": 1, "flips_equal_label_changes_all_interpolators": true, "gap_range_interpolators": [ 0.28799352971164116, 0.3038633505749581 ] } ], "analyses": { "primary_interpolators": { "n_models": 232, "within_dataset": { "target": "generalization_gap", "min_models": 8, "estimand": "equal-weight mean over eligible datasets with defined Spearman correlation", "bootstrap": "percentile 95%; resample informative datasets with replacement", "per_dataset": [ { "dataset_seed": 1000, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.5294117647058824, "reason": null }, "Lmax": { "spearman": 0.37941176470588234, "reason": null }, "Lavg": { "spearman": 0.7205882352941175, "reason": null }, "Lavg_flip": { "spearman": 0.7058823529411764, "reason": null }, "band_mass_0.1": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_0.2": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_0.5": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_1.0": { "spearman": -0.7264705882352941, "reason": null }, "flat_rel": { "spearman": 0.7382352941176471, "reason": null }, "sam_sharp": { "spearman": 0.27941176470588236, "reason": null }, "hess_power": { "spearman": 0.4617647058823529, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.049999999999999996, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.2911764705882353, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.07058823529411765, "reason": null }, "normalized_Lmax": { "spearman": -0.2352941176470588, "reason": null }, "normalized_Lavg": { "spearman": 0.2588235294117647, "reason": null }, "normalized_margin_min": { "spearman": -0.5529411764705882, "reason": null }, "normalized_margin_q10": { "spearman": -0.7852941176470588, "reason": null } } }, { "dataset_seed": 1001, "n_models": 12, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.1398601398601399, "reason": null }, "Lmax": { "spearman": 0.6363636363636365, "reason": null }, "Lavg": { "spearman": 0.4895104895104896, "reason": null }, "Lavg_flip": { "spearman": 0.7622377622377624, "reason": null }, "band_mass_0.1": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_0.2": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_0.5": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_1.0": { "spearman": -0.8111888111888113, "reason": null }, "flat_rel": { "spearman": 0.06993006993006995, "reason": null }, "sam_sharp": { "spearman": 0.3076923076923077, "reason": null }, "hess_power": { "spearman": 0.07692307692307693, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6643356643356644, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.5244755244755246, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.7622377622377624, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.2937062937062937, "reason": null }, "normalized_Lmax": { "spearman": 0.08391608391608392, "reason": null }, "normalized_Lavg": { "spearman": 0.23076923076923078, "reason": null }, "normalized_margin_min": { "spearman": -0.06993006993006995, "reason": null }, "normalized_margin_q10": { "spearman": -0.3216783216783217, "reason": null } } }, { "dataset_seed": 1002, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.5340659340659341, "reason": null }, "Lmax": { "spearman": 0.054945054945054944, "reason": null }, "Lavg": { "spearman": -0.0945054945054945, "reason": null }, "Lavg_flip": { "spearman": 0.1032967032967033, "reason": null }, "band_mass_0.1": { "spearman": -0.2923076923076923, "reason": null }, "band_mass_0.2": { "spearman": -0.2967032967032967, "reason": null }, "band_mass_0.5": { "spearman": -0.2967032967032967, "reason": null }, "band_mass_1.0": { "spearman": -0.2967032967032967, "reason": null }, "flat_rel": { "spearman": -0.2175824175824176, "reason": null }, "sam_sharp": { "spearman": -0.02857142857142857, "reason": null }, "hess_power": { "spearman": 0.07692307692307693, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6307692307692307, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.5164835164835165, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.4065934065934066, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.002197802197802198, "reason": null }, "normalized_Lmax": { "spearman": 0.3054945054945055, "reason": null }, "normalized_Lavg": { "spearman": 0.621978021978022, "reason": null }, "normalized_margin_min": { "spearman": 0.02417582417582418, "reason": null }, "normalized_margin_q10": { "spearman": 0.7186813186813187, "reason": null } } }, { "dataset_seed": 1003, "n_models": 6, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1004, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.023529411764705882, "reason": null }, "Lmax": { "spearman": -0.13529411764705881, "reason": null }, "Lavg": { "spearman": -0.2411764705882353, "reason": null }, "Lavg_flip": { "spearman": -0.0029411764705882353, "reason": null }, "band_mass_0.1": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.2": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.5": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_1.0": { "spearman": 0.06470588235294117, "reason": null }, "flat_rel": { "spearman": -0.02647058823529412, "reason": null }, "sam_sharp": { "spearman": -0.3941176470588235, "reason": null }, "hess_power": { "spearman": -0.37941176470588234, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0823529411764706, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.047058823529411764, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.13823529411764704, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.09411764705882353, "reason": null }, "normalized_Lmax": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_Lavg": { "spearman": 0.08823529411764705, "reason": null }, "normalized_margin_min": { "spearman": 0.22352941176470587, "reason": null }, "normalized_margin_q10": { "spearman": 0.25, "reason": null } } }, { "dataset_seed": 1005, "n_models": 9, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.3666666666666667, "reason": null }, "Lmax": { "spearman": -0.7, "reason": null }, "Lavg": { "spearman": -0.7166666666666667, "reason": null }, "Lavg_flip": { "spearman": 0.39999999999999997, "reason": null }, "band_mass_0.1": { "spearman": -0.5499999999999999, "reason": null }, "band_mass_0.2": { "spearman": -0.5499999999999999, "reason": null }, "band_mass_0.5": { "spearman": -0.5499999999999999, "reason": null }, "band_mass_1.0": { "spearman": -0.5333333333333333, "reason": null }, "flat_rel": { "spearman": -0.7833333333333333, "reason": null }, "sam_sharp": { "spearman": -0.11666666666666665, "reason": null }, "hess_power": { "spearman": 0.19999999999999998, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.7833333333333333, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.6166666666666666, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.6166666666666666, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.6, "reason": null }, "normalized_Lmax": { "spearman": -0.4666666666666666, "reason": null }, "normalized_Lavg": { "spearman": 0.8166666666666667, "reason": null }, "normalized_margin_min": { "spearman": 0.9333333333333332, "reason": null }, "normalized_margin_q10": { "spearman": 0.7666666666666667, "reason": null } } }, { "dataset_seed": 1006, "n_models": 11, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.16363636363636366, "reason": null }, "Lmax": { "spearman": -0.22727272727272727, "reason": null }, "Lavg": { "spearman": -0.3181818181818182, "reason": null }, "Lavg_flip": { "spearman": -0.45454545454545453, "reason": null }, "band_mass_0.1": { "spearman": -0.14545454545454548, "reason": null }, "band_mass_0.2": { "spearman": -0.15454545454545454, "reason": null }, "band_mass_0.5": { "spearman": -0.1272727272727273, "reason": null }, "band_mass_1.0": { "spearman": -0.09090909090909091, "reason": null }, "flat_rel": { "spearman": -0.4000000000000001, "reason": null }, "sam_sharp": { "spearman": -0.4090909090909091, "reason": null }, "hess_power": { "spearman": -0.3181818181818182, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6818181818181819, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.6454545454545455, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.5727272727272729, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.045454545454545456, "reason": null }, "normalized_Lmax": { "spearman": 0.48181818181818187, "reason": null }, "normalized_Lavg": { "spearman": 0.6454545454545455, "reason": null }, "normalized_margin_min": { "spearman": 0.27272727272727276, "reason": null }, "normalized_margin_q10": { "spearman": 0.16363636363636366, "reason": null } } }, { "dataset_seed": 1007, "n_models": 15, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.3199286353466923, "reason": null }, "Lmax": { "spearman": -0.8285714285714284, "reason": null }, "Lavg": { "spearman": -0.682142857142857, "reason": null }, "Lavg_flip": { "spearman": -0.11428571428571425, "reason": null }, "band_mass_0.1": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_0.2": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_0.5": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_1.0": { "spearman": -0.6607142857142856, "reason": null }, "flat_rel": { "spearman": -0.7892857142857141, "reason": null }, "sam_sharp": { "spearman": -0.43214285714285705, "reason": null }, "hess_power": { "spearman": -0.5821428571428571, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6214285714285712, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.7321428571428571, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.6392857142857141, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.7285714285714285, "reason": null }, "normalized_Lmax": { "spearman": -0.7107142857142856, "reason": null }, "normalized_Lavg": { "spearman": 0.024999999999999998, "reason": null }, "normalized_margin_min": { "spearman": 0.6892857142857142, "reason": null }, "normalized_margin_q10": { "spearman": 0.14285714285714282, "reason": null } } }, { "dataset_seed": 1008, "n_models": 3, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1009, "n_models": 15, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.37115374447904503, "reason": null }, "nb_regions": { "spearman": -0.2486587163042157, "reason": null }, "Lmax": { "spearman": 0.5142857142857142, "reason": null }, "Lavg": { "spearman": 0.3, "reason": null }, "Lavg_flip": { "spearman": 0.37857142857142856, "reason": null }, "band_mass_0.1": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_0.2": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_0.5": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_1.0": { "spearman": -0.23571428571428568, "reason": null }, "flat_rel": { "spearman": -0.3607142857142856, "reason": null }, "sam_sharp": { "spearman": 0.12142857142857141, "reason": null }, "hess_power": { "spearman": 0.27499999999999997, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.09999999999999999, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.05357142857142857, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.010714285714285713, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.16428571428571426, "reason": null }, "normalized_Lmax": { "spearman": 0.3428571428571428, "reason": null }, "normalized_Lavg": { "spearman": 0.0, "reason": null }, "normalized_margin_min": { "spearman": -0.24642857142857136, "reason": null }, "normalized_margin_q10": { "spearman": 0.18928571428571428, "reason": null } } }, { "dataset_seed": 1010, "n_models": 7, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1011, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.01978021978021978, "reason": null }, "Lmax": { "spearman": 0.3010989010989011, "reason": null }, "Lavg": { "spearman": 0.4153846153846154, "reason": null }, "Lavg_flip": { "spearman": 0.7142857142857143, "reason": null }, "band_mass_0.1": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_0.2": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_0.5": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_1.0": { "spearman": -0.8021978021978022, "reason": null }, "flat_rel": { "spearman": -0.5076923076923077, "reason": null }, "sam_sharp": { "spearman": -0.3494505494505495, "reason": null }, "hess_power": { "spearman": 0.08131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.3538461538461538, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.4241758241758242, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.3494505494505495, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3142857142857143, "reason": null }, "normalized_Lmax": { "spearman": 0.06813186813186813, "reason": null }, "normalized_Lavg": { "spearman": 0.45494505494505494, "reason": null }, "normalized_margin_min": { "spearman": -0.49890109890109896, "reason": null }, "normalized_margin_q10": { "spearman": 0.5076923076923077, "reason": null } } }, { "dataset_seed": 1012, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.4472135954999579, "reason": null }, "nb_regions": { "spearman": 0.0022002213534027884, "reason": null }, "Lmax": { "spearman": -0.2527472527472528, "reason": null }, "Lavg": { "spearman": -0.410989010989011, "reason": null }, "Lavg_flip": { "spearman": -0.0857142857142857, "reason": null }, "band_mass_0.1": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_0.2": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_0.5": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_1.0": { "spearman": -0.07252747252747253, "reason": null }, "flat_rel": { "spearman": 0.1252747252747253, "reason": null }, "sam_sharp": { "spearman": 0.221978021978022, "reason": null }, "hess_power": { "spearman": 0.08131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.3098901098901099, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.367032967032967, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.26153846153846155, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3450549450549451, "reason": null }, "normalized_Lmax": { "spearman": 0.16923076923076924, "reason": null }, "normalized_Lavg": { "spearman": 0.3934065934065934, "reason": null }, "normalized_margin_min": { "spearman": 0.44175824175824174, "reason": null }, "normalized_margin_q10": { "spearman": 0.27912087912087913, "reason": null } } }, { "dataset_seed": 1013, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.02857142857142857, "reason": null }, "Lmax": { "spearman": -0.26593406593406593, "reason": null }, "Lavg": { "spearman": -0.3362637362637363, "reason": null }, "Lavg_flip": { "spearman": -0.08131868131868132, "reason": null }, "band_mass_0.1": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_0.2": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_0.5": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_1.0": { "spearman": -0.4945054945054946, "reason": null }, "flat_rel": { "spearman": -0.6527472527472528, "reason": null }, "sam_sharp": { "spearman": 0.0857142857142857, "reason": null }, "hess_power": { "spearman": -0.14285714285714285, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.1384615384615385, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.14725274725274726, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.032967032967032975, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.1384615384615385, "reason": null }, "normalized_Lmax": { "spearman": 0.221978021978022, "reason": null }, "normalized_Lavg": { "spearman": 0.04175824175824176, "reason": null }, "normalized_margin_min": { "spearman": 0.3934065934065934, "reason": null }, "normalized_margin_q10": { "spearman": 0.3010989010989011, "reason": null } } }, { "dataset_seed": 1014, "n_models": 7, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1015, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.0058823529411764705, "reason": null }, "Lmax": { "spearman": 0.36470588235294116, "reason": null }, "Lavg": { "spearman": 0.23823529411764705, "reason": null }, "Lavg_flip": { "spearman": 0.22058823529411767, "reason": null }, "band_mass_0.1": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.2": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.5": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_1.0": { "spearman": -0.18529411764705883, "reason": null }, "flat_rel": { "spearman": 0.09705882352941177, "reason": null }, "sam_sharp": { "spearman": 0.023529411764705882, "reason": null }, "hess_power": { "spearman": 0.6029411764705882, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.4176470588235294, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.32647058823529407, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.7029411764705883, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.38823529411764707, "reason": null }, "normalized_Lmax": { "spearman": 0.15294117647058825, "reason": null }, "normalized_Lavg": { "spearman": -0.4205882352941176, "reason": null }, "normalized_margin_min": { "spearman": -0.02941176470588235, "reason": null }, "normalized_margin_q10": { "spearman": -0.4794117647058824, "reason": null } } }, { "dataset_seed": 1016, "n_models": 13, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.13186813186813187, "reason": null }, "Lmax": { "spearman": 0.46703296703296704, "reason": null }, "Lavg": { "spearman": 0.7032967032967032, "reason": null }, "Lavg_flip": { "spearman": 0.5384615384615384, "reason": null }, "band_mass_0.1": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_0.2": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_0.5": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_1.0": { "spearman": -0.32967032967032966, "reason": null }, "flat_rel": { "spearman": 0.40659340659340654, "reason": null }, "sam_sharp": { "spearman": -0.027472527472527472, "reason": null }, "hess_power": { "spearman": 0.14835164835164835, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.07692307692307691, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.07692307692307691, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.23626373626373626, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.016483516483516484, "reason": null }, "normalized_Lmax": { "spearman": 0.1043956043956044, "reason": null }, "normalized_Lavg": { "spearman": 0.1758241758241758, "reason": null }, "normalized_margin_min": { "spearman": -0.510989010989011, "reason": null }, "normalized_margin_q10": { "spearman": -0.1868131868131868, "reason": null } } }, { "dataset_seed": 1017, "n_models": 3, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1018, "n_models": 13, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.41208791208791207, "reason": null }, "Lmax": { "spearman": 0.054945054945054944, "reason": null }, "Lavg": { "spearman": 0.17032967032967034, "reason": null }, "Lavg_flip": { "spearman": -0.027472527472527472, "reason": null }, "band_mass_0.1": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_0.2": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_0.5": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_1.0": { "spearman": 0.17032967032967034, "reason": null }, "flat_rel": { "spearman": 0.027472527472527472, "reason": null }, "sam_sharp": { "spearman": 0.2857142857142857, "reason": null }, "hess_power": { "spearman": 0.3131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.532967032967033, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.5549450549450549, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.6923076923076923, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.7747252747252747, "reason": null }, "normalized_Lmax": { "spearman": -0.41208791208791207, "reason": null }, "normalized_Lavg": { "spearman": -0.48901098901098894, "reason": null }, "normalized_margin_min": { "spearman": -0.25274725274725274, "reason": null }, "normalized_margin_q10": { "spearman": -0.6923076923076923, "reason": null } } }, { "dataset_seed": 1019, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.14725274725274726, "reason": null }, "Lmax": { "spearman": 0.6307692307692307, "reason": null }, "Lavg": { "spearman": 0.7714285714285714, "reason": null }, "Lavg_flip": { "spearman": 0.5384615384615385, "reason": null }, "band_mass_0.1": { "spearman": -0.5780219780219781, "reason": null }, "band_mass_0.2": { "spearman": -0.5560439560439561, "reason": null }, "band_mass_0.5": { "spearman": -0.5560439560439561, "reason": null }, "band_mass_1.0": { "spearman": -0.5560439560439561, "reason": null }, "flat_rel": { "spearman": 0.22637362637362637, "reason": null }, "sam_sharp": { "spearman": 0.3450549450549451, "reason": null }, "hess_power": { "spearman": 0.5208791208791209, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.22637362637362637, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.16043956043956045, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.3142857142857143, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.006593406593406593, "reason": null }, "normalized_Lmax": { "spearman": 0.3142857142857143, "reason": null }, "normalized_Lavg": { "spearman": -0.2, "reason": null }, "normalized_margin_min": { "spearman": -0.6351648351648352, "reason": null }, "normalized_margin_q10": { "spearman": -0.6131868131868132, "reason": null } } } ], "metrics": { "Leb_wrong": { "mean_spearman": 1.0, "ci95": [ 1.0, 1.0 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "label_changes": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 15, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 15, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "wrong_label_count": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 15, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 15, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nn_Leb_wrong": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 15, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 15, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nb_flips": { "mean_spearman": 0.4091836699895015, "ci95": [ 0.37115374447904503, 0.4472135954999579 ], "n_eligible_datasets": 15, "n_informative_datasets": 2, "informative_dataset_seeds": [ 1009, 1012 ], "n_constant_proxy_datasets": 13, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nb_regions": { "mean_spearman": -0.016217476461642878, "ci95": [ -0.1547147863815891, 0.12382941625446399 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lmax": { "mean_spearman": 0.06624924095512333, "ci95": [ -0.17063179467591225, 0.28213999236058057 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lavg": { "mean_spearman": 0.06725650166826637, "ci95": [ -0.18421849147584443, 0.31025830542007 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lavg_flip": { "mean_spearman": 0.2397004955828485, "ci95": [ 0.060491033476327616, 0.4192824332530213 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.1": { "mean_spearman": -0.37566159330865206, "ci95": [ -0.5234123278682101, -0.22191640222522582 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.2": { "mean_spearman": -0.3750954927425515, "ci95": [ -0.522023537736773, -0.22235998805116453 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.5": { "mean_spearman": -0.37327731092436967, "ci95": [ -0.5213643586805351, -0.21954010205480798 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_1.0": { "mean_spearman": -0.3706824874471933, "ci95": [ -0.5203296131972603, -0.21762995827701717 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "flat_rel": { "mean_spearman": -0.13645916175327938, "ci95": [ -0.34922965596495004, 0.08409174159174146 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "sam_sharp": { "mean_spearman": -0.0057992660933837295, "ci95": [ -0.14291339706045583, 0.12770254582019283 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "hess_power": { "mean_spearman": 0.09440089322442263, "ci95": [ -0.07011664560929266, 0.2553515087853323 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.1": { "mean_spearman": -0.18634202398908284, "ci95": [ -0.3950073341690989, 0.025611873420696943 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.2": { "mean_spearman": -0.1771822948293537, "ci95": [ -0.3668822851984616, 0.0206753295723883 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.5": { "mean_spearman": -0.13363603716544895, "ci95": [ -0.3522808188543483, 0.10310526728173781 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_1.0": { "mean_spearman": 0.020168263109439585, "ci95": [ -0.16945266987914043, 0.2065901108695225 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_Lmax": { "mean_spearman": 0.027626915568092054, "ci95": [ -0.14836078464019636, 0.18516956572838925 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_Lavg": { "mean_spearman": 0.17621747533512236, "ci95": [ -0.009193250377073895, 0.3601702766188061 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_margin_min": { "mean_spearman": 0.012113507407625061, "ci95": [ -0.21484068790686436, 0.25387423687423677 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_margin_q10": { "mean_spearman": 0.01602315984668922, "ci95": [ -0.23156439393939396, 0.26101764575293984 ], "n_eligible_datasets": 15, "n_informative_datasets": 15, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1005, 1006, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 } }, "excluded_empty_dataset_seeds": [] }, "pooled_descriptive": { "Leb_wrong": { "n_models": 232, "pearson": 1.0, "spearman": 0.9999999999999998 }, "label_changes": { "n_models": 232, "pearson": 0.5054509786229308, "spearman": 0.5084538012060361 }, "wrong_label_count": { "n_models": 232, "pearson": 0.7784940931125329, "spearman": 0.800926130189117 }, "nn_Leb_wrong": { "n_models": 232, "pearson": 0.970677362499789, "spearman": 0.9568171944062543 }, "nb_flips": { "n_models": 232, "pearson": 0.5069141356009138, "spearman": 0.5102114587557549 }, "nb_regions": { "n_models": 232, "pearson": -0.0001332522220885631, "spearman": -0.007737777614200639 }, "Lmax": { "n_models": 232, "pearson": -0.00932540705211393, "spearman": -0.015709538352723196 }, "Lavg": { "n_models": 232, "pearson": -0.33898185063638475, "spearman": -0.14210948146126479 }, "Lavg_flip": { "n_models": 232, "pearson": 0.027183046243260896, "spearman": 0.1409322511142845 }, "band_mass_0.1": { "n_models": 232, "pearson": 0.20194457951626094, "spearman": 0.15144275585299705 }, "band_mass_0.2": { "n_models": 232, "pearson": 0.2030489555527822, "spearman": 0.15400287148104222 }, "band_mass_0.5": { "n_models": 232, "pearson": 0.20428204266763006, "spearman": 0.15744807212914358 }, "band_mass_1.0": { "n_models": 232, "pearson": 0.20247080121813685, "spearman": 0.15771715335131048 }, "flat_rel": { "n_models": 232, "pearson": 0.22300284671553286, "spearman": 0.35629429028866644 }, "sam_sharp": { "n_models": 232, "pearson": 0.12845436690513695, "spearman": 0.09909780910224893 }, "hess_power": { "n_models": 232, "pearson": 0.17573849927468665, "spearman": 0.37113219768244177 }, "normalized_band_mass_0.1": { "n_models": 232, "pearson": -0.05065746609173772, "spearman": -0.13976847482841262 }, "normalized_band_mass_0.2": { "n_models": 232, "pearson": -0.025676772163720867, "spearman": -0.08291641760636874 }, "normalized_band_mass_0.5": { "n_models": 232, "pearson": -5.038054749678508e-05, "spearman": -0.05600925639404253 }, "normalized_band_mass_1.0": { "n_models": 232, "pearson": 0.17822203213315302, "spearman": 0.08053600979455648 }, "normalized_Lmax": { "n_models": 232, "pearson": 0.19100273181392213, "spearman": 0.39235694008522176 }, "normalized_Lavg": { "n_models": 232, "pearson": 0.4365910562742265, "spearman": 0.44855743634787576 }, "normalized_margin_min": { "n_models": 232, "pearson": 0.18135207333570055, "spearman": 0.3010211632381234 }, "normalized_margin_q10": { "n_models": 232, "pearson": 0.1449586464644308, "spearman": 0.2507788940377367 } } }, "secondary_ce_target": { "n_models": 206, "within_dataset": { "target": "generalization_gap", "min_models": 8, "estimand": "equal-weight mean over eligible datasets with defined Spearman correlation", "bootstrap": "percentile 95%; resample informative datasets with replacement", "per_dataset": [ { "dataset_seed": 1000, "n_models": 13, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.6318681318681317, "reason": null }, "Lmax": { "spearman": 0.5439560439560439, "reason": null }, "Lavg": { "spearman": 0.8846153846153846, "reason": null }, "Lavg_flip": { "spearman": 0.7857142857142857, "reason": null }, "band_mass_0.1": { "spearman": -0.7527472527472527, "reason": null }, "band_mass_0.2": { "spearman": -0.7527472527472527, "reason": null }, "band_mass_0.5": { "spearman": -0.7527472527472527, "reason": null }, "band_mass_1.0": { "spearman": -0.7527472527472527, "reason": null }, "flat_rel": { "spearman": 0.8626373626373626, "reason": null }, "sam_sharp": { "spearman": 0.45604395604395603, "reason": null }, "hess_power": { "spearman": 0.6263736263736264, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.21428571428571427, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.24725274725274726, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.35714285714285715, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.06043956043956044, "reason": null }, "normalized_Lmax": { "spearman": -0.4175824175824176, "reason": null }, "normalized_Lavg": { "spearman": 0.3131868131868132, "reason": null }, "normalized_margin_min": { "spearman": -0.8351648351648352, "reason": null }, "normalized_margin_q10": { "spearman": -0.7802197802197801, "reason": null } } }, { "dataset_seed": 1001, "n_models": 12, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.1398601398601399, "reason": null }, "Lmax": { "spearman": 0.6363636363636365, "reason": null }, "Lavg": { "spearman": 0.4895104895104896, "reason": null }, "Lavg_flip": { "spearman": 0.7622377622377624, "reason": null }, "band_mass_0.1": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_0.2": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_0.5": { "spearman": -0.8111888111888113, "reason": null }, "band_mass_1.0": { "spearman": -0.8111888111888113, "reason": null }, "flat_rel": { "spearman": 0.06993006993006995, "reason": null }, "sam_sharp": { "spearman": 0.3076923076923077, "reason": null }, "hess_power": { "spearman": 0.07692307692307693, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6643356643356644, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.5244755244755246, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.7622377622377624, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.2937062937062937, "reason": null }, "normalized_Lmax": { "spearman": 0.08391608391608392, "reason": null }, "normalized_Lavg": { "spearman": 0.23076923076923078, "reason": null }, "normalized_margin_min": { "spearman": -0.06993006993006995, "reason": null }, "normalized_margin_q10": { "spearman": -0.3216783216783217, "reason": null } } }, { "dataset_seed": 1002, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.5340659340659341, "reason": null }, "Lmax": { "spearman": 0.054945054945054944, "reason": null }, "Lavg": { "spearman": -0.0945054945054945, "reason": null }, "Lavg_flip": { "spearman": 0.1032967032967033, "reason": null }, "band_mass_0.1": { "spearman": -0.2923076923076923, "reason": null }, "band_mass_0.2": { "spearman": -0.2967032967032967, "reason": null }, "band_mass_0.5": { "spearman": -0.2967032967032967, "reason": null }, "band_mass_1.0": { "spearman": -0.2967032967032967, "reason": null }, "flat_rel": { "spearman": -0.2175824175824176, "reason": null }, "sam_sharp": { "spearman": -0.02857142857142857, "reason": null }, "hess_power": { "spearman": 0.07692307692307693, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6307692307692307, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.5164835164835165, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.4065934065934066, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.002197802197802198, "reason": null }, "normalized_Lmax": { "spearman": 0.3054945054945055, "reason": null }, "normalized_Lavg": { "spearman": 0.621978021978022, "reason": null }, "normalized_margin_min": { "spearman": 0.02417582417582418, "reason": null }, "normalized_margin_q10": { "spearman": 0.7186813186813187, "reason": null } } }, { "dataset_seed": 1003, "n_models": 6, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1004, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.023529411764705882, "reason": null }, "Lmax": { "spearman": -0.13529411764705881, "reason": null }, "Lavg": { "spearman": -0.2411764705882353, "reason": null }, "Lavg_flip": { "spearman": -0.0029411764705882353, "reason": null }, "band_mass_0.1": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.2": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.5": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_1.0": { "spearman": 0.06470588235294117, "reason": null }, "flat_rel": { "spearman": -0.02647058823529412, "reason": null }, "sam_sharp": { "spearman": -0.3941176470588235, "reason": null }, "hess_power": { "spearman": -0.37941176470588234, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0823529411764706, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.047058823529411764, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.13823529411764704, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.09411764705882353, "reason": null }, "normalized_Lmax": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_Lavg": { "spearman": 0.08823529411764705, "reason": null }, "normalized_margin_min": { "spearman": 0.22352941176470587, "reason": null }, "normalized_margin_q10": { "spearman": 0.25, "reason": null } } }, { "dataset_seed": 1005, "n_models": 7, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1006, "n_models": 2, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1007, "n_models": 15, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.3199286353466923, "reason": null }, "Lmax": { "spearman": -0.8285714285714284, "reason": null }, "Lavg": { "spearman": -0.682142857142857, "reason": null }, "Lavg_flip": { "spearman": -0.11428571428571425, "reason": null }, "band_mass_0.1": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_0.2": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_0.5": { "spearman": -0.6607142857142856, "reason": null }, "band_mass_1.0": { "spearman": -0.6607142857142856, "reason": null }, "flat_rel": { "spearman": -0.7892857142857141, "reason": null }, "sam_sharp": { "spearman": -0.43214285714285705, "reason": null }, "hess_power": { "spearman": -0.5821428571428571, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6214285714285712, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.7321428571428571, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.6392857142857141, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.7285714285714285, "reason": null }, "normalized_Lmax": { "spearman": -0.7107142857142856, "reason": null }, "normalized_Lavg": { "spearman": 0.024999999999999998, "reason": null }, "normalized_margin_min": { "spearman": 0.6892857142857142, "reason": null }, "normalized_margin_q10": { "spearman": 0.14285714285714282, "reason": null } } }, { "dataset_seed": 1009, "n_models": 15, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.37115374447904503, "reason": null }, "nb_regions": { "spearman": -0.2486587163042157, "reason": null }, "Lmax": { "spearman": 0.5142857142857142, "reason": null }, "Lavg": { "spearman": 0.3, "reason": null }, "Lavg_flip": { "spearman": 0.37857142857142856, "reason": null }, "band_mass_0.1": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_0.2": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_0.5": { "spearman": -0.20357142857142854, "reason": null }, "band_mass_1.0": { "spearman": -0.23571428571428568, "reason": null }, "flat_rel": { "spearman": -0.3607142857142856, "reason": null }, "sam_sharp": { "spearman": 0.12142857142857141, "reason": null }, "hess_power": { "spearman": 0.27499999999999997, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.09999999999999999, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.05357142857142857, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.010714285714285713, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.16428571428571426, "reason": null }, "normalized_Lmax": { "spearman": 0.3428571428571428, "reason": null }, "normalized_Lavg": { "spearman": 0.0, "reason": null }, "normalized_margin_min": { "spearman": -0.24642857142857136, "reason": null }, "normalized_margin_q10": { "spearman": 0.18928571428571428, "reason": null } } }, { "dataset_seed": 1010, "n_models": 6, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1011, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.01978021978021978, "reason": null }, "Lmax": { "spearman": 0.3010989010989011, "reason": null }, "Lavg": { "spearman": 0.4153846153846154, "reason": null }, "Lavg_flip": { "spearman": 0.7142857142857143, "reason": null }, "band_mass_0.1": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_0.2": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_0.5": { "spearman": -0.8021978021978022, "reason": null }, "band_mass_1.0": { "spearman": -0.8021978021978022, "reason": null }, "flat_rel": { "spearman": -0.5076923076923077, "reason": null }, "sam_sharp": { "spearman": -0.3494505494505495, "reason": null }, "hess_power": { "spearman": 0.08131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.3538461538461538, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.4241758241758242, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.3494505494505495, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3142857142857143, "reason": null }, "normalized_Lmax": { "spearman": 0.06813186813186813, "reason": null }, "normalized_Lavg": { "spearman": 0.45494505494505494, "reason": null }, "normalized_margin_min": { "spearman": -0.49890109890109896, "reason": null }, "normalized_margin_q10": { "spearman": 0.5076923076923077, "reason": null } } }, { "dataset_seed": 1012, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.4472135954999579, "reason": null }, "nb_regions": { "spearman": 0.0022002213534027884, "reason": null }, "Lmax": { "spearman": -0.2527472527472528, "reason": null }, "Lavg": { "spearman": -0.410989010989011, "reason": null }, "Lavg_flip": { "spearman": -0.0857142857142857, "reason": null }, "band_mass_0.1": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_0.2": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_0.5": { "spearman": -0.07252747252747253, "reason": null }, "band_mass_1.0": { "spearman": -0.07252747252747253, "reason": null }, "flat_rel": { "spearman": 0.1252747252747253, "reason": null }, "sam_sharp": { "spearman": 0.221978021978022, "reason": null }, "hess_power": { "spearman": 0.08131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.3098901098901099, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.367032967032967, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.26153846153846155, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3450549450549451, "reason": null }, "normalized_Lmax": { "spearman": 0.16923076923076924, "reason": null }, "normalized_Lavg": { "spearman": 0.3934065934065934, "reason": null }, "normalized_margin_min": { "spearman": 0.44175824175824174, "reason": null }, "normalized_margin_q10": { "spearman": 0.27912087912087913, "reason": null } } }, { "dataset_seed": 1013, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": 0.02857142857142857, "reason": null }, "Lmax": { "spearman": -0.26593406593406593, "reason": null }, "Lavg": { "spearman": -0.3362637362637363, "reason": null }, "Lavg_flip": { "spearman": -0.08131868131868132, "reason": null }, "band_mass_0.1": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_0.2": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_0.5": { "spearman": -0.4945054945054946, "reason": null }, "band_mass_1.0": { "spearman": -0.4945054945054946, "reason": null }, "flat_rel": { "spearman": -0.6527472527472528, "reason": null }, "sam_sharp": { "spearman": 0.0857142857142857, "reason": null }, "hess_power": { "spearman": -0.14285714285714285, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.1384615384615385, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.14725274725274726, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.032967032967032975, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.1384615384615385, "reason": null }, "normalized_Lmax": { "spearman": 0.221978021978022, "reason": null }, "normalized_Lavg": { "spearman": 0.04175824175824176, "reason": null }, "normalized_margin_min": { "spearman": 0.3934065934065934, "reason": null }, "normalized_margin_q10": { "spearman": 0.3010989010989011, "reason": null } } }, { "dataset_seed": 1014, "n_models": 2, "eligible": false, "metrics": { "Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "label_changes": { "spearman": null, "reason": "too_few_models" }, "wrong_label_count": { "spearman": null, "reason": "too_few_models" }, "nn_Leb_wrong": { "spearman": null, "reason": "too_few_models" }, "nb_flips": { "spearman": null, "reason": "too_few_models" }, "nb_regions": { "spearman": null, "reason": "too_few_models" }, "Lmax": { "spearman": null, "reason": "too_few_models" }, "Lavg": { "spearman": null, "reason": "too_few_models" }, "Lavg_flip": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "flat_rel": { "spearman": null, "reason": "too_few_models" }, "sam_sharp": { "spearman": null, "reason": "too_few_models" }, "hess_power": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.1": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.2": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_0.5": { "spearman": null, "reason": "too_few_models" }, "normalized_band_mass_1.0": { "spearman": null, "reason": "too_few_models" }, "normalized_Lmax": { "spearman": null, "reason": "too_few_models" }, "normalized_Lavg": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_min": { "spearman": null, "reason": "too_few_models" }, "normalized_margin_q10": { "spearman": null, "reason": "too_few_models" } } }, { "dataset_seed": 1015, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.0058823529411764705, "reason": null }, "Lmax": { "spearman": 0.36470588235294116, "reason": null }, "Lavg": { "spearman": 0.23823529411764705, "reason": null }, "Lavg_flip": { "spearman": 0.22058823529411767, "reason": null }, "band_mass_0.1": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.2": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.5": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_1.0": { "spearman": -0.18529411764705883, "reason": null }, "flat_rel": { "spearman": 0.09705882352941177, "reason": null }, "sam_sharp": { "spearman": 0.023529411764705882, "reason": null }, "hess_power": { "spearman": 0.6029411764705882, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.4176470588235294, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.32647058823529407, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.7029411764705883, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.38823529411764707, "reason": null }, "normalized_Lmax": { "spearman": 0.15294117647058825, "reason": null }, "normalized_Lavg": { "spearman": -0.4205882352941176, "reason": null }, "normalized_margin_min": { "spearman": -0.02941176470588235, "reason": null }, "normalized_margin_q10": { "spearman": -0.4794117647058824, "reason": null } } }, { "dataset_seed": 1016, "n_models": 13, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.13186813186813187, "reason": null }, "Lmax": { "spearman": 0.46703296703296704, "reason": null }, "Lavg": { "spearman": 0.7032967032967032, "reason": null }, "Lavg_flip": { "spearman": 0.5384615384615384, "reason": null }, "band_mass_0.1": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_0.2": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_0.5": { "spearman": -0.32967032967032966, "reason": null }, "band_mass_1.0": { "spearman": -0.32967032967032966, "reason": null }, "flat_rel": { "spearman": 0.40659340659340654, "reason": null }, "sam_sharp": { "spearman": -0.027472527472527472, "reason": null }, "hess_power": { "spearman": 0.14835164835164835, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.07692307692307691, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.07692307692307691, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.23626373626373626, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.016483516483516484, "reason": null }, "normalized_Lmax": { "spearman": 0.1043956043956044, "reason": null }, "normalized_Lavg": { "spearman": 0.1758241758241758, "reason": null }, "normalized_margin_min": { "spearman": -0.510989010989011, "reason": null }, "normalized_margin_q10": { "spearman": -0.1868131868131868, "reason": null } } }, { "dataset_seed": 1018, "n_models": 13, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.41208791208791207, "reason": null }, "Lmax": { "spearman": 0.054945054945054944, "reason": null }, "Lavg": { "spearman": 0.17032967032967034, "reason": null }, "Lavg_flip": { "spearman": -0.027472527472527472, "reason": null }, "band_mass_0.1": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_0.2": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_0.5": { "spearman": 0.1758241758241758, "reason": null }, "band_mass_1.0": { "spearman": 0.17032967032967034, "reason": null }, "flat_rel": { "spearman": 0.027472527472527472, "reason": null }, "sam_sharp": { "spearman": 0.2857142857142857, "reason": null }, "hess_power": { "spearman": 0.3131868131868132, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.532967032967033, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.5549450549450549, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.6923076923076923, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.7747252747252747, "reason": null }, "normalized_Lmax": { "spearman": -0.41208791208791207, "reason": null }, "normalized_Lavg": { "spearman": -0.48901098901098894, "reason": null }, "normalized_margin_min": { "spearman": -0.25274725274725274, "reason": null }, "normalized_margin_q10": { "spearman": -0.6923076923076923, "reason": null } } }, { "dataset_seed": 1019, "n_models": 14, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.14725274725274726, "reason": null }, "Lmax": { "spearman": 0.6307692307692307, "reason": null }, "Lavg": { "spearman": 0.7714285714285714, "reason": null }, "Lavg_flip": { "spearman": 0.5384615384615385, "reason": null }, "band_mass_0.1": { "spearman": -0.5780219780219781, "reason": null }, "band_mass_0.2": { "spearman": -0.5560439560439561, "reason": null }, "band_mass_0.5": { "spearman": -0.5560439560439561, "reason": null }, "band_mass_1.0": { "spearman": -0.5560439560439561, "reason": null }, "flat_rel": { "spearman": 0.22637362637362637, "reason": null }, "sam_sharp": { "spearman": 0.3450549450549451, "reason": null }, "hess_power": { "spearman": 0.5208791208791209, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.22637362637362637, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.16043956043956045, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.3142857142857143, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.006593406593406593, "reason": null }, "normalized_Lmax": { "spearman": 0.3142857142857143, "reason": null }, "normalized_Lavg": { "spearman": -0.2, "reason": null }, "normalized_margin_min": { "spearman": -0.6351648351648352, "reason": null }, "normalized_margin_q10": { "spearman": -0.6131868131868132, "reason": null } } } ], "metrics": { "Leb_wrong": { "mean_spearman": 1.0, "ci95": [ 1.0, 1.0 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "label_changes": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 13, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 13, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "wrong_label_count": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 13, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 13, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nn_Leb_wrong": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 13, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 13, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nb_flips": { "mean_spearman": 0.4091836699895015, "ci95": [ 0.37115374447904503, 0.4472135954999579 ], "n_eligible_datasets": 13, "n_informative_datasets": 2, "informative_dataset_seeds": [ 1009, 1012 ], "n_constant_proxy_datasets": 11, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "nb_regions": { "mean_spearman": -0.06738627264537869, "ci95": [ -0.21832523918501068, 0.08637288151377469 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lmax": { "mean_spearman": 0.16042735544997994, "ci95": [ -0.07789834599789347, 0.3752153276587664 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lavg": { "mean_spearman": 0.16982485839951902, "ci95": [ -0.08745260960758697, 0.4241190229453804 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "Lavg_flip": { "mean_spearman": 0.28691421700471476, "ci95": [ 0.11122551091442497, 0.46748370972918474 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.1": { "mean_spearman": -0.3822067073198294, "ci95": [ -0.5518316920635923, -0.21050830391328143 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.2": { "mean_spearman": -0.38085421365964345, "ci95": [ -0.550843189615814, -0.20967191462383786 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_0.5": { "mean_spearman": -0.38085421365964345, "ci95": [ -0.550843189615814, -0.20967191462383786 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "band_mass_1.0": { "mean_spearman": -0.38171319630595646, "ci95": [ -0.5505391186867431, -0.21040242280344096 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "flat_rel": { "mean_spearman": -0.0568578480343186, "ci95": [ -0.2876334270932687, 0.18018612609110346 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "sam_sharp": { "mean_spearman": 0.04733852120729949, "ci95": [ -0.10411422853164935, 0.19000985157376554 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "hess_power": { "mean_spearman": 0.13067724131072544, "ci95": [ -0.05930377032469792, 0.31226673213664163 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.1": { "mean_spearman": -0.08537005980444895, "ci95": [ -0.2987743534745797, 0.12305075580528071 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.2": { "mean_spearman": -0.09218157408202662, "ci95": [ -0.2959427083097897, 0.10677028288453605 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_0.5": { "mean_spearman": -0.057629248579474816, "ci95": [ -0.29607722594147934, 0.1921091408591408 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_band_mass_1.0": { "mean_spearman": 0.0667090827724312, "ci95": [ -0.1337514408668255, 0.25746590513558837 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_Lmax": { "mean_spearman": 0.01668953218726975, "ci95": [ -0.16691977988075277, 0.1701315313645629 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_Lavg": { "mean_spearman": 0.09503878474466712, "ci95": [ -0.08241124260355027, 0.26144550867684335 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_margin_min": { "mean_spearman": -0.10050628104926748, "ci95": [ -0.33620483871332285, 0.14070951107715798 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 }, "normalized_margin_q10": { "mean_spearman": -0.052683176551954834, "ci95": [ -0.30813312253357494, 0.19543530281935229 ], "n_eligible_datasets": 13, "n_informative_datasets": 13, "informative_dataset_seeds": [ 1000, 1001, 1002, 1004, 1007, 1009, 1011, 1012, 1013, 1015, 1016, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 5 } }, "excluded_empty_dataset_seeds": [ 1008, 1017 ] }, "pooled_descriptive": { "Leb_wrong": { "n_models": 206, "pearson": 1.0, "spearman": 1.0 }, "label_changes": { "n_models": 206, "pearson": 0.5144382187481837, "spearman": 0.5473224266602511 }, "wrong_label_count": { "n_models": 206, "pearson": 0.8363543780278361, "spearman": 0.8276366669278616 }, "nn_Leb_wrong": { "n_models": 206, "pearson": 0.9728336666687026, "spearman": 0.9550762974435357 }, "nb_flips": { "n_models": 206, "pearson": 0.5157662493100771, "spearman": 0.5494272330925158 }, "nb_regions": { "n_models": 206, "pearson": 0.015551708228429216, "spearman": 0.009091358377391499 }, "Lmax": { "n_models": 206, "pearson": -0.02141315692636344, "spearman": -0.026395824110203955 }, "Lavg": { "n_models": 206, "pearson": -0.3345828058033604, "spearman": -0.10868364065658384 }, "Lavg_flip": { "n_models": 206, "pearson": 0.03739084566816466, "spearman": 0.16665053691482462 }, "band_mass_0.1": { "n_models": 206, "pearson": 0.1783532286866621, "spearman": 0.14351292267671514 }, "band_mass_0.2": { "n_models": 206, "pearson": 0.17829454593085553, "spearman": 0.14352665012509136 }, "band_mass_0.5": { "n_models": 206, "pearson": 0.17823602952644585, "spearman": 0.14345938562804791 }, "band_mass_1.0": { "n_models": 206, "pearson": 0.17794149558813532, "spearman": 0.1421923421429233 }, "flat_rel": { "n_models": 206, "pearson": 0.2237709678667447, "spearman": 0.33114243257248954 }, "sam_sharp": { "n_models": 206, "pearson": 0.0906704885576113, "spearman": 0.05335104174173865 }, "hess_power": { "n_models": 206, "pearson": 0.24966603914877605, "spearman": 0.3497623435499868 }, "normalized_band_mass_0.1": { "n_models": 206, "pearson": -0.004052305775959339, "spearman": -0.09317986046048725 }, "normalized_band_mass_0.2": { "n_models": 206, "pearson": 0.025944144444497073, "spearman": -0.04675637554180523 }, "normalized_band_mass_0.5": { "n_models": 206, "pearson": 0.03030658063854019, "spearman": -0.021467670143142967 }, "normalized_band_mass_1.0": { "n_models": 206, "pearson": 0.16990637731212224, "spearman": 0.0677950629231915 }, "normalized_Lmax": { "n_models": 206, "pearson": 0.3223725789321024, "spearman": 0.3907374042081493 }, "normalized_Lavg": { "n_models": 206, "pearson": 0.4554717624981588, "spearman": 0.44581604532803454 }, "normalized_margin_min": { "n_models": 206, "pearson": 0.18183082439530351, "spearman": 0.33492297185529896 }, "normalized_margin_q10": { "n_models": 206, "pearson": 0.12289500189904168, "spearman": 0.2574507441992951 } } }, "secondary_all_models": { "n_models": 320, "within_dataset": { "target": "population_error", "min_models": 8, "estimand": "equal-weight mean over eligible datasets with defined Spearman correlation", "bootstrap": "percentile 95%; resample informative datasets with replacement", "per_dataset": [ { "dataset_seed": 1000, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.5294117647058824, "reason": null }, "Lmax": { "spearman": 0.37941176470588234, "reason": null }, "Lavg": { "spearman": 0.7205882352941175, "reason": null }, "Lavg_flip": { "spearman": 0.7058823529411764, "reason": null }, "band_mass_0.1": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_0.2": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_0.5": { "spearman": -0.7235294117647059, "reason": null }, "band_mass_1.0": { "spearman": -0.7264705882352941, "reason": null }, "flat_rel": { "spearman": 0.7382352941176471, "reason": null }, "sam_sharp": { "spearman": 0.27941176470588236, "reason": null }, "hess_power": { "spearman": 0.4617647058823529, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.049999999999999996, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.2911764705882353, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.07058823529411765, "reason": null }, "normalized_Lmax": { "spearman": -0.2352941176470588, "reason": null }, "normalized_Lavg": { "spearman": 0.2588235294117647, "reason": null }, "normalized_margin_min": { "spearman": -0.5529411764705882, "reason": null }, "normalized_margin_q10": { "spearman": -0.7852941176470588, "reason": null } } }, { "dataset_seed": 1001, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": -0.17431911788999158, "reason": null }, "nb_regions": { "spearman": -0.2411764705882353, "reason": null }, "Lmax": { "spearman": 0.5411764705882353, "reason": null }, "Lavg": { "spearman": 0.44117647058823534, "reason": null }, "Lavg_flip": { "spearman": 0.6882352941176471, "reason": null }, "band_mass_0.1": { "spearman": -0.7764705882352941, "reason": null }, "band_mass_0.2": { "spearman": -0.7764705882352941, "reason": null }, "band_mass_0.5": { "spearman": -0.7764705882352941, "reason": null }, "band_mass_1.0": { "spearman": -0.7176470588235294, "reason": null }, "flat_rel": { "spearman": -0.06764705882352941, "reason": null }, "sam_sharp": { "spearman": 0.1411764705882353, "reason": null }, "hess_power": { "spearman": 0.09705882352941177, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.4176470588235294, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.3411764705882353, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.5852941176470587, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.29705882352941176, "reason": null }, "normalized_Lmax": { "spearman": 0.12941176470588234, "reason": null }, "normalized_Lavg": { "spearman": 0.29705882352941176, "reason": null }, "normalized_margin_min": { "spearman": -0.1411764705882353, "reason": null }, "normalized_margin_q10": { "spearman": -0.3764705882352941, "reason": null } } }, { "dataset_seed": 1002, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.03834824944236852, "reason": null }, "nb_regions": { "spearman": 0.5, "reason": null }, "Lmax": { "spearman": -0.11470588235294117, "reason": null }, "Lavg": { "spearman": -0.2088235294117647, "reason": null }, "Lavg_flip": { "spearman": -0.02941176470588235, "reason": null }, "band_mass_0.1": { "spearman": -0.21764705882352942, "reason": null }, "band_mass_0.2": { "spearman": -0.22058823529411767, "reason": null }, "band_mass_0.5": { "spearman": -0.1764705882352941, "reason": null }, "band_mass_1.0": { "spearman": -0.1764705882352941, "reason": null }, "flat_rel": { "spearman": 0.03823529411764706, "reason": null }, "sam_sharp": { "spearman": -0.0411764705882353, "reason": null }, "hess_power": { "spearman": 0.07058823529411765, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.4441176470588235, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.36764705882352944, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.3147058823529411, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.02647058823529412, "reason": null }, "normalized_Lmax": { "spearman": 0.15882352941176472, "reason": null }, "normalized_Lavg": { "spearman": 0.3941176470588235, "reason": null }, "normalized_margin_min": { "spearman": 0.03823529411764706, "reason": null }, "normalized_margin_q10": { "spearman": 0.5029411764705883, "reason": null } } }, { "dataset_seed": 1003, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.4518868007280904, "reason": null }, "nb_regions": { "spearman": 0.44624496112623724, "reason": null }, "Lmax": { "spearman": 0.4735294117647058, "reason": null }, "Lavg": { "spearman": 0.4941176470588235, "reason": null }, "Lavg_flip": { "spearman": 0.4705882352941176, "reason": null }, "band_mass_0.1": { "spearman": -0.4794117647058824, "reason": null }, "band_mass_0.2": { "spearman": -0.4794117647058824, "reason": null }, "band_mass_0.5": { "spearman": -0.4794117647058824, "reason": null }, "band_mass_1.0": { "spearman": -0.4794117647058824, "reason": null }, "flat_rel": { "spearman": -0.10882352941176471, "reason": null }, "sam_sharp": { "spearman": -0.26176470588235295, "reason": null }, "hess_power": { "spearman": -0.10882352941176471, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.07352941176470588, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.1264705882352941, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.09999999999999999, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.22352941176470587, "reason": null }, "normalized_Lmax": { "spearman": 0.0823529411764706, "reason": null }, "normalized_Lavg": { "spearman": 0.21764705882352942, "reason": null }, "normalized_margin_min": { "spearman": 0.4676470588235294, "reason": null }, "normalized_margin_q10": { "spearman": 0.011764705882352941, "reason": null } } }, { "dataset_seed": 1004, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.023529411764705882, "reason": null }, "Lmax": { "spearman": -0.13529411764705881, "reason": null }, "Lavg": { "spearman": -0.2411764705882353, "reason": null }, "Lavg_flip": { "spearman": -0.0029411764705882353, "reason": null }, "band_mass_0.1": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.2": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_0.5": { "spearman": 0.06470588235294117, "reason": null }, "band_mass_1.0": { "spearman": 0.06470588235294117, "reason": null }, "flat_rel": { "spearman": -0.02647058823529412, "reason": null }, "sam_sharp": { "spearman": -0.3941176470588235, "reason": null }, "hess_power": { "spearman": -0.37941176470588234, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0823529411764706, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.047058823529411764, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.13823529411764704, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.09411764705882353, "reason": null }, "normalized_Lmax": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_Lavg": { "spearman": 0.08823529411764705, "reason": null }, "normalized_margin_min": { "spearman": 0.22352941176470587, "reason": null }, "normalized_margin_q10": { "spearman": 0.25, "reason": null } } }, { "dataset_seed": 1005, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.4509560339299332, "reason": null }, "nb_regions": { "spearman": 0.3764705882352941, "reason": null }, "Lmax": { "spearman": -0.6441176470588235, "reason": null }, "Lavg": { "spearman": -0.3205882352941176, "reason": null }, "Lavg_flip": { "spearman": 0.2764705882352941, "reason": null }, "band_mass_0.1": { "spearman": -0.5617647058823529, "reason": null }, "band_mass_0.2": { "spearman": -0.5617647058823529, "reason": null }, "band_mass_0.5": { "spearman": -0.5852941176470587, "reason": null }, "band_mass_1.0": { "spearman": -0.5617647058823529, "reason": null }, "flat_rel": { "spearman": -0.6176470588235293, "reason": null }, "sam_sharp": { "spearman": -0.0823529411764706, "reason": null }, "hess_power": { "spearman": 0.17352941176470588, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6735294117647058, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.7000000000000001, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.6852941176470588, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.6411764705882352, "reason": null }, "normalized_Lmax": { "spearman": -0.6147058823529412, "reason": null }, "normalized_Lavg": { "spearman": 0.7794117647058825, "reason": null }, "normalized_margin_min": { "spearman": 0.6588235294117648, "reason": null }, "normalized_margin_q10": { "spearman": 0.6205882352941177, "reason": null } } }, { "dataset_seed": 1006, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.05210501057189061, "reason": null }, "nb_regions": { "spearman": -0.011764705882352941, "reason": null }, "Lmax": { "spearman": -0.22941176470588234, "reason": null }, "Lavg": { "spearman": -0.26176470588235295, "reason": null }, "Lavg_flip": { "spearman": -0.39999999999999997, "reason": null }, "band_mass_0.1": { "spearman": 0.3029411764705882, "reason": null }, "band_mass_0.2": { "spearman": 0.33529411764705885, "reason": null }, "band_mass_0.5": { "spearman": 0.33529411764705885, "reason": null }, "band_mass_1.0": { "spearman": 0.35000000000000003, "reason": null }, "flat_rel": { "spearman": -0.3147058823529411, "reason": null }, "sam_sharp": { "spearman": -0.6411764705882352, "reason": null }, "hess_power": { "spearman": -0.36470588235294116, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.4823529411764706, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.538235294117647, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.3852941176470588, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.07647058823529412, "reason": null }, "normalized_Lmax": { "spearman": 0.3941176470588235, "reason": null }, "normalized_Lavg": { "spearman": 0.45294117647058824, "reason": null }, "normalized_margin_min": { "spearman": -0.15882352941176472, "reason": null }, "normalized_margin_q10": { "spearman": -0.023529411764705882, "reason": null } } }, { "dataset_seed": 1007, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.42008402520840293, "reason": null }, "nb_regions": { "spearman": 0.4329901603007055, "reason": null }, "Lmax": { "spearman": -0.85, "reason": null }, "Lavg": { "spearman": -0.7205882352941175, "reason": null }, "Lavg_flip": { "spearman": 0.023529411764705882, "reason": null }, "band_mass_0.1": { "spearman": -0.36764705882352944, "reason": null }, "band_mass_0.2": { "spearman": -0.36764705882352944, "reason": null }, "band_mass_0.5": { "spearman": -0.36764705882352944, "reason": null }, "band_mass_1.0": { "spearman": -0.36764705882352944, "reason": null }, "flat_rel": { "spearman": -0.8264705882352942, "reason": null }, "sam_sharp": { "spearman": -0.17941176470588235, "reason": null }, "hess_power": { "spearman": -0.3941176470588235, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.6882352941176471, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.7794117647058825, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.6941176470588235, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.7764705882352941, "reason": null }, "normalized_Lmax": { "spearman": -0.7529411764705882, "reason": null }, "normalized_Lavg": { "spearman": 0.19705882352941176, "reason": null }, "normalized_margin_min": { "spearman": 0.7441176470588236, "reason": null }, "normalized_margin_q10": { "spearman": 0.29411764705882354, "reason": null } } }, { "dataset_seed": 1008, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.3090522029191526, "reason": null }, "nb_regions": { "spearman": -0.02058823529411765, "reason": null }, "Lmax": { "spearman": 0.18823529411764706, "reason": null }, "Lavg": { "spearman": 0.22352941176470587, "reason": null }, "Lavg_flip": { "spearman": 0.3852941176470588, "reason": null }, "band_mass_0.1": { "spearman": -0.3323529411764706, "reason": null }, "band_mass_0.2": { "spearman": -0.32647058823529407, "reason": null }, "band_mass_0.5": { "spearman": -0.31764705882352945, "reason": null }, "band_mass_1.0": { "spearman": -0.3029411764705882, "reason": null }, "flat_rel": { "spearman": 0.15882352941176472, "reason": null }, "sam_sharp": { "spearman": 0.28529411764705886, "reason": null }, "hess_power": { "spearman": 0.1264705882352941, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.0058823529411764705, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.008823529411764706, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.014705882352941175, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.11470588235294117, "reason": null }, "normalized_Lmax": { "spearman": 0.17058823529411765, "reason": null }, "normalized_Lavg": { "spearman": 0.18529411764705883, "reason": null }, "normalized_margin_min": { "spearman": 0.15882352941176472, "reason": null }, "normalized_margin_q10": { "spearman": 0.17352941176470588, "reason": null } } }, { "dataset_seed": 1009, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.11504474832710555, "reason": null }, "nb_regions": { "spearman": -0.3107514415763566, "reason": null }, "Lmax": { "spearman": 0.5705882352941177, "reason": null }, "Lavg": { "spearman": 0.37941176470588234, "reason": null }, "Lavg_flip": { "spearman": 0.4441176470588235, "reason": null }, "band_mass_0.1": { "spearman": -0.2764705882352941, "reason": null }, "band_mass_0.2": { "spearman": -0.2735294117647059, "reason": null }, "band_mass_0.5": { "spearman": -0.2735294117647059, "reason": null }, "band_mass_1.0": { "spearman": -0.3029411764705882, "reason": null }, "flat_rel": { "spearman": -0.3941176470588235, "reason": null }, "sam_sharp": { "spearman": 0.032352941176470584, "reason": null }, "hess_power": { "spearman": 0.3323529411764706, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.2058823529411765, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.15588235294117647, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.1176470588235294, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.26176470588235295, "reason": null }, "normalized_Lmax": { "spearman": 0.19411764705882353, "reason": null }, "normalized_Lavg": { "spearman": -0.10588235294117648, "reason": null }, "normalized_margin_min": { "spearman": -0.3294117647058824, "reason": null }, "normalized_margin_q10": { "spearman": 0.044117647058823525, "reason": null } } }, { "dataset_seed": 1010, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.6782510669390219, "reason": null }, "nb_regions": { "spearman": 0.21502232450307138, "reason": null }, "Lmax": { "spearman": 0.15, "reason": null }, "Lavg": { "spearman": 0.05294117647058824, "reason": null }, "Lavg_flip": { "spearman": 0.14705882352941177, "reason": null }, "band_mass_0.1": { "spearman": -0.3529411764705882, "reason": null }, "band_mass_0.2": { "spearman": -0.2911764705882353, "reason": null }, "band_mass_0.5": { "spearman": -0.17058823529411765, "reason": null }, "band_mass_1.0": { "spearman": -0.17352941176470588, "reason": null }, "flat_rel": { "spearman": -0.6352941176470589, "reason": null }, "sam_sharp": { "spearman": -0.1411764705882353, "reason": null }, "hess_power": { "spearman": -0.4323529411764706, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.15, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.3529411764705882, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.2411764705882353, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.18823529411764706, "reason": null }, "normalized_Lmax": { "spearman": -0.01764705882352941, "reason": null }, "normalized_Lavg": { "spearman": 0.24705882352941175, "reason": null }, "normalized_margin_min": { "spearman": 0.6147058823529412, "reason": null }, "normalized_margin_q10": { "spearman": 0.1264705882352941, "reason": null } } }, { "dataset_seed": 1011, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.4509560339299332, "reason": null }, "nb_regions": { "spearman": 0.17941176470588235, "reason": null }, "Lmax": { "spearman": 0.044117647058823525, "reason": null }, "Lavg": { "spearman": 0.1647058823529412, "reason": null }, "Lavg_flip": { "spearman": 0.3411764705882353, "reason": null }, "band_mass_0.1": { "spearman": -0.31764705882352945, "reason": null }, "band_mass_0.2": { "spearman": -0.31764705882352945, "reason": null }, "band_mass_0.5": { "spearman": -0.31764705882352945, "reason": null }, "band_mass_1.0": { "spearman": -0.4588235294117647, "reason": null }, "flat_rel": { "spearman": -0.6235294117647059, "reason": null }, "sam_sharp": { "spearman": -0.02647058823529412, "reason": null }, "hess_power": { "spearman": -0.21470588235294116, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.2676470588235294, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.29411764705882354, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.25588235294117645, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3911764705882353, "reason": null }, "normalized_Lmax": { "spearman": -0.17941176470588235, "reason": null }, "normalized_Lavg": { "spearman": 0.3411764705882353, "reason": null }, "normalized_margin_min": { "spearman": -0.10294117647058824, "reason": null }, "normalized_margin_q10": { "spearman": 0.6147058823529412, "reason": null } } }, { "dataset_seed": 1012, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.2286693027873004, "reason": null }, "nb_regions": { "spearman": -0.0029455112945626217, "reason": null }, "Lmax": { "spearman": -0.14705882352941177, "reason": null }, "Lavg": { "spearman": -0.2529411764705882, "reason": null }, "Lavg_flip": { "spearman": -0.03529411764705882, "reason": null }, "band_mass_0.1": { "spearman": -0.02647058823529412, "reason": null }, "band_mass_0.2": { "spearman": -0.02647058823529412, "reason": null }, "band_mass_0.5": { "spearman": -0.07058823529411765, "reason": null }, "band_mass_1.0": { "spearman": -0.07058823529411765, "reason": null }, "flat_rel": { "spearman": 0.31764705882352945, "reason": null }, "sam_sharp": { "spearman": 0.22058823529411767, "reason": null }, "hess_power": { "spearman": 0.1323529411764706, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.25588235294117645, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.3529411764705882, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.3029411764705882, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.17941176470588235, "reason": null }, "normalized_Lmax": { "spearman": 0.2352941176470588, "reason": null }, "normalized_Lavg": { "spearman": 0.4058823529411764, "reason": null }, "normalized_margin_min": { "spearman": 0.2735294117647059, "reason": null }, "normalized_margin_q10": { "spearman": 0.3529411764705882, "reason": null } } }, { "dataset_seed": 1013, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.5739440431835515, "reason": null }, "nb_regions": { "spearman": 0.14411764705882352, "reason": null }, "Lmax": { "spearman": -0.19999999999999998, "reason": null }, "Lavg": { "spearman": -0.3, "reason": null }, "Lavg_flip": { "spearman": -0.22352941176470587, "reason": null }, "band_mass_0.1": { "spearman": -0.6617647058823529, "reason": null }, "band_mass_0.2": { "spearman": -0.6617647058823529, "reason": null }, "band_mass_0.5": { "spearman": -0.6588235294117648, "reason": null }, "band_mass_1.0": { "spearman": -0.6588235294117648, "reason": null }, "flat_rel": { "spearman": -0.7647058823529411, "reason": null }, "sam_sharp": { "spearman": 0.049999999999999996, "reason": null }, "hess_power": { "spearman": -0.3558823529411765, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.09117647058823529, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.09705882352941177, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.011764705882352941, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.03823529411764706, "reason": null }, "normalized_Lmax": { "spearman": -0.06764705882352941, "reason": null }, "normalized_Lavg": { "spearman": 0.0823529411764706, "reason": null }, "normalized_margin_min": { "spearman": 0.5911764705882353, "reason": null }, "normalized_margin_q10": { "spearman": 0.5323529411764706, "reason": null } } }, { "dataset_seed": 1014, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": -0.0079275249025749, "reason": null }, "nb_regions": { "spearman": 0.2411764705882353, "reason": null }, "Lmax": { "spearman": 0.049999999999999996, "reason": null }, "Lavg": { "spearman": 0.1323529411764706, "reason": null }, "Lavg_flip": { "spearman": 0.34705882352941175, "reason": null }, "band_mass_0.1": { "spearman": -0.34705882352941175, "reason": null }, "band_mass_0.2": { "spearman": -0.34411764705882353, "reason": null }, "band_mass_0.5": { "spearman": -0.3382352941176471, "reason": null }, "band_mass_1.0": { "spearman": -0.15588235294117647, "reason": null }, "flat_rel": { "spearman": 0.3529411764705882, "reason": null }, "sam_sharp": { "spearman": -0.09999999999999999, "reason": null }, "hess_power": { "spearman": 0.12058823529411765, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.23823529411764705, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.29411764705882354, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.11470588235294117, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.05294117647058824, "reason": null }, "normalized_Lmax": { "spearman": -0.23823529411764705, "reason": null }, "normalized_Lavg": { "spearman": 0.21176470588235297, "reason": null }, "normalized_margin_min": { "spearman": 0.07941176470588236, "reason": null }, "normalized_margin_q10": { "spearman": 0.09999999999999999, "reason": null } } }, { "dataset_seed": 1015, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": null, "reason": "constant_proxy" }, "nb_regions": { "spearman": -0.0058823529411764705, "reason": null }, "Lmax": { "spearman": 0.36470588235294116, "reason": null }, "Lavg": { "spearman": 0.23823529411764705, "reason": null }, "Lavg_flip": { "spearman": 0.22058823529411767, "reason": null }, "band_mass_0.1": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.2": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_0.5": { "spearman": -0.21176470588235297, "reason": null }, "band_mass_1.0": { "spearman": -0.18529411764705883, "reason": null }, "flat_rel": { "spearman": 0.09705882352941177, "reason": null }, "sam_sharp": { "spearman": 0.023529411764705882, "reason": null }, "hess_power": { "spearman": 0.6029411764705882, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.4176470588235294, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.32647058823529407, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.7029411764705883, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.38823529411764707, "reason": null }, "normalized_Lmax": { "spearman": 0.15294117647058825, "reason": null }, "normalized_Lavg": { "spearman": -0.4205882352941176, "reason": null }, "normalized_margin_min": { "spearman": -0.02941176470588235, "reason": null }, "normalized_margin_q10": { "spearman": -0.4794117647058824, "reason": null } } }, { "dataset_seed": 1016, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.25928148942086576, "reason": null }, "nb_regions": { "spearman": 0.02941176470588235, "reason": null }, "Lmax": { "spearman": 0.0588235294117647, "reason": null }, "Lavg": { "spearman": 0.5617647058823529, "reason": null }, "Lavg_flip": { "spearman": 0.29411764705882354, "reason": null }, "band_mass_0.1": { "spearman": -0.0588235294117647, "reason": null }, "band_mass_0.2": { "spearman": -0.0588235294117647, "reason": null }, "band_mass_0.5": { "spearman": -0.0588235294117647, "reason": null }, "band_mass_1.0": { "spearman": -0.30882352941176466, "reason": null }, "flat_rel": { "spearman": 0.22352941176470587, "reason": null }, "sam_sharp": { "spearman": 0.15588235294117647, "reason": null }, "hess_power": { "spearman": 0.07352941176470588, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.3235294117647059, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.3411764705882353, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.4441176470588235, "reason": null }, "normalized_band_mass_1.0": { "spearman": -0.07941176470588236, "reason": null }, "normalized_Lmax": { "spearman": -0.19999999999999998, "reason": null }, "normalized_Lavg": { "spearman": 0.27058823529411763, "reason": null }, "normalized_margin_min": { "spearman": -0.14705882352941177, "reason": null }, "normalized_margin_q10": { "spearman": 0.15588235294117647, "reason": null } } }, { "dataset_seed": 1017, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": -0.28221988653166646, "reason": null }, "nb_regions": { "spearman": 0.4647058823529412, "reason": null }, "Lmax": { "spearman": -0.25588235294117645, "reason": null }, "Lavg": { "spearman": -0.21176470588235297, "reason": null }, "Lavg_flip": { "spearman": -0.09705882352941177, "reason": null }, "band_mass_0.1": { "spearman": 0.4323529411764706, "reason": null }, "band_mass_0.2": { "spearman": 0.44705882352941173, "reason": null }, "band_mass_0.5": { "spearman": 0.5205882352941177, "reason": null }, "band_mass_1.0": { "spearman": 0.38823529411764707, "reason": null }, "flat_rel": { "spearman": -0.5147058823529412, "reason": null }, "sam_sharp": { "spearman": -0.1411764705882353, "reason": null }, "hess_power": { "spearman": 0.0029411764705882353, "reason": null }, "normalized_band_mass_0.1": { "spearman": -0.15882352941176472, "reason": null }, "normalized_band_mass_0.2": { "spearman": -0.09705882352941177, "reason": null }, "normalized_band_mass_0.5": { "spearman": -0.2088235294117647, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.02647058823529412, "reason": null }, "normalized_Lmax": { "spearman": -0.18235294117647058, "reason": null }, "normalized_Lavg": { "spearman": 0.09999999999999999, "reason": null }, "normalized_margin_min": { "spearman": -0.22058823529411767, "reason": null }, "normalized_margin_q10": { "spearman": -0.15, "reason": null } } }, { "dataset_seed": 1018, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": 0.583383351196948, "reason": null }, "nb_regions": { "spearman": 0.0029411764705882353, "reason": null }, "Lmax": { "spearman": 0.044117647058823525, "reason": null }, "Lavg": { "spearman": 0.1323529411764706, "reason": null }, "Lavg_flip": { "spearman": 0.02941176470588235, "reason": null }, "band_mass_0.1": { "spearman": -0.16176470588235295, "reason": null }, "band_mass_0.2": { "spearman": -0.16176470588235295, "reason": null }, "band_mass_0.5": { "spearman": -0.15588235294117647, "reason": null }, "band_mass_1.0": { "spearman": -0.22058823529411767, "reason": null }, "flat_rel": { "spearman": 0.011764705882352941, "reason": null }, "sam_sharp": { "spearman": 0.07352941176470588, "reason": null }, "hess_power": { "spearman": -0.047058823529411764, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.3294117647058824, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.36764705882352944, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.36470588235294116, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.3529411764705882, "reason": null }, "normalized_Lmax": { "spearman": -0.3735294117647059, "reason": null }, "normalized_Lavg": { "spearman": -0.45294117647058824, "reason": null }, "normalized_margin_min": { "spearman": 0.27941176470588236, "reason": null }, "normalized_margin_q10": { "spearman": -0.27941176470588236, "reason": null } } }, { "dataset_seed": 1019, "n_models": 16, "eligible": true, "metrics": { "Leb_wrong": { "spearman": 1.0, "reason": null }, "label_changes": { "spearman": null, "reason": "constant_proxy" }, "wrong_label_count": { "spearman": null, "reason": "constant_proxy" }, "nn_Leb_wrong": { "spearman": null, "reason": "constant_proxy" }, "nb_flips": { "spearman": -0.4509560339299332, "reason": null }, "nb_regions": { "spearman": -0.3058823529411765, "reason": null }, "Lmax": { "spearman": 0.7088235294117646, "reason": null }, "Lavg": { "spearman": 0.8058823529411765, "reason": null }, "Lavg_flip": { "spearman": 0.4588235294117647, "reason": null }, "band_mass_0.1": { "spearman": -0.4794117647058824, "reason": null }, "band_mass_0.2": { "spearman": -0.4647058823529412, "reason": null }, "band_mass_0.5": { "spearman": -0.4647058823529412, "reason": null }, "band_mass_1.0": { "spearman": -0.11176470588235293, "reason": null }, "flat_rel": { "spearman": 0.411764705882353, "reason": null }, "sam_sharp": { "spearman": 0.19411764705882353, "reason": null }, "hess_power": { "spearman": 0.6294117647058822, "reason": null }, "normalized_band_mass_0.1": { "spearman": 0.4264705882352941, "reason": null }, "normalized_band_mass_0.2": { "spearman": 0.3735294117647059, "reason": null }, "normalized_band_mass_0.5": { "spearman": 0.4441176470588235, "reason": null }, "normalized_band_mass_1.0": { "spearman": 0.10588235294117648, "reason": null }, "normalized_Lmax": { "spearman": 0.38235294117647056, "reason": null }, "normalized_Lavg": { "spearman": -0.39999999999999997, "reason": null }, "normalized_margin_min": { "spearman": -0.7000000000000001, "reason": null }, "normalized_margin_q10": { "spearman": -0.6911764705882353, "reason": null } } } ], "metrics": { "Leb_wrong": { "mean_spearman": 1.0, "ci95": [ 1.0, 1.0 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "label_changes": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 20, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 20, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "wrong_label_count": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 20, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 20, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "nn_Leb_wrong": { "mean_spearman": null, "ci95": null, "n_eligible_datasets": 20, "n_informative_datasets": 0, "informative_dataset_seeds": [], "n_constant_proxy_datasets": 20, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "nb_flips": { "mean_spearman": 0.21744351737237638, "ci95": [ 0.06530122011416918, 0.3607449036916169 ], "n_eligible_datasets": 20, "n_informative_datasets": 17, "informative_dataset_seeds": [ 1001, 1002, 1003, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 3, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "nb_regions": { "mean_spearman": 0.07902802465295472, "ci95": [ -0.04571211229081402, 0.19851655366890333 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "Lmax": { "mean_spearman": 0.0498529411764706, "ci95": [ -0.12118014705882352, 0.21764705882352936 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "Lavg": { "mean_spearman": 0.09147058823529411, "ci95": [ -0.07764705882352939, 0.26426838235294114 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "Lavg_flip": { "mean_spearman": 0.20220588235294118, "ci95": [ 0.07837867647058823, 0.32720955882352937 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "band_mass_0.1": { "mean_spearman": -0.2776470588235294, "ci95": [ -0.40529779411764705, -0.13705514705882355 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "band_mass_0.2": { "mean_spearman": -0.27102941176470585, "ci95": [ -0.3995625, -0.12999632352941182 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "band_mass_0.5": { "mean_spearman": -0.26132352941176473, "ci95": [ -0.39397794117647056, -0.11646691176470594 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "band_mass_1.0": { "mean_spearman": -0.2588235294117647, "ci95": [ -0.3855919117647059, -0.12705514705882356 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "flat_rel": { "mean_spearman": -0.1272058823529412, "ci95": [ -0.3132463235294117, 0.06044117647058826 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "sam_sharp": { "mean_spearman": -0.027647058823529413, "ci95": [ -0.13015073529411764, 0.0639742647058823 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "hess_power": { "mean_spearman": 0.026323529411764697, "ci95": [ -0.10500000000000001, 0.16455882352941176 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_band_mass_0.1": { "mean_spearman": -0.1325, "ci95": [ -0.2730919117647059, 0.01176838235294113 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_band_mass_0.2": { "mean_spearman": -0.16352941176470587, "ci95": [ -0.3048566176470588, -0.01882352941176472 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_band_mass_0.5": { "mean_spearman": -0.12735294117647059, "ci95": [ -0.2845625, 0.03735661764705879 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_band_mass_1.0": { "mean_spearman": -0.027794117647058823, "ci95": [ -0.16456249999999997, 0.09823529411764703 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_Lmax": { "mean_spearman": -0.048382352941176474, "ci95": [ -0.17971323529411762, 0.07838235294117647 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_Lavg": { "mean_spearman": 0.1575, "ci95": [ 0.021761029411764703, 0.28559191176470583 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_margin_min": { "mean_spearman": 0.08735294117647056, "ci95": [ -0.08382352941176469, 0.2555919117647058 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 }, "normalized_margin_q10": { "mean_spearman": 0.04970588235294119, "ci95": [ -0.1307463235294118, 0.2167683823529411 ], "n_eligible_datasets": 20, "n_informative_datasets": 20, "informative_dataset_seeds": [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019 ], "n_constant_proxy_datasets": 0, "n_constant_target_datasets": 0, "n_undefined_measure_datasets": 0, "n_too_few_models_datasets": 0 } }, "excluded_empty_dataset_seeds": [] }, "pooled_descriptive": { "Leb_wrong": { "n_models": 320, "pearson": 0.9999999999999999, "spearman": 1.0 }, "label_changes": { "n_models": 320, "pearson": 0.48246425188419195, "spearman": 0.4617724525138099 }, "wrong_label_count": { "n_models": 320, "pearson": 0.7119527372506803, "spearman": 0.7178220373381922 }, "nn_Leb_wrong": { "n_models": 320, "pearson": 0.9478297335651131, "spearman": 0.9383325225004142 }, "nb_flips": { "n_models": 320, "pearson": 0.4863623315760469, "spearman": 0.4977409539396755 }, "nb_regions": { "n_models": 320, "pearson": -0.008924514277794813, "spearman": -0.012926245949186205 }, "Lmax": { "n_models": 320, "pearson": -0.04444317865646095, "spearman": -0.07372300999033195 }, "Lavg": { "n_models": 320, "pearson": -0.3786487562297641, "spearman": -0.2325048340315823 }, "Lavg_flip": { "n_models": 320, "pearson": -0.09234903809598406, "spearman": -0.01086045762165646 }, "band_mass_0.1": { "n_models": 320, "pearson": 0.11132516476314491, "spearman": 0.22249985351419452 }, "band_mass_0.2": { "n_models": 320, "pearson": 0.13327162338186804, "spearman": 0.22483556968329768 }, "band_mass_0.5": { "n_models": 320, "pearson": 0.14145984953326893, "spearman": 0.22898661119737496 }, "band_mass_1.0": { "n_models": 320, "pearson": 0.19769498247291922, "spearman": 0.23973354231974925 }, "flat_rel": { "n_models": 320, "pearson": 0.2602826662269818, "spearman": 0.3400067383470542 }, "sam_sharp": { "n_models": 320, "pearson": 0.17262952532234666, "spearman": 0.10252578150177248 }, "hess_power": { "n_models": 320, "pearson": 0.22804122410932426, "spearman": 0.37699403802771514 }, "normalized_band_mass_0.1": { "n_models": 320, "pearson": -0.07698427441941812, "spearman": -0.16145189406146543 }, "normalized_band_mass_0.2": { "n_models": 320, "pearson": -0.07485303436676949, "spearman": -0.12406835027685817 }, "normalized_band_mass_0.5": { "n_models": 320, "pearson": -0.04746487424750173, "spearman": -0.09568013359505464 }, "normalized_band_mass_1.0": { "n_models": 320, "pearson": 0.14031544181797864, "spearman": 0.06951813494272405 }, "normalized_Lmax": { "n_models": 320, "pearson": 0.24878659420990268, "spearman": 0.35868001640641023 }, "normalized_Lavg": { "n_models": 320, "pearson": 0.4348054100494276, "spearman": 0.4161134386077989 }, "normalized_margin_min": { "n_models": 320, "pearson": 0.003690147958631405, "spearman": 0.1204424603755896 }, "normalized_margin_q10": { "n_models": 320, "pearson": 0.12251937743768426, "spearman": 0.21727031025693608 } } } }, "notes": [ "Only completed dataset blocks enter correlations; all attempted fits remain recorded.", "Intervals resample datasets with defined correlations; inspect coverage for each proxy.", "No multiple-comparison correction; measures and thresholds fixed before the suite.", "High pooled correlation can reflect dataset differences. Dataset-only measures cannot rank fits within a dataset.", "Wrong-label count, nearest-neighbour disagreement and model disagreement use the known clean rule.", "Positive normalized training margins suggest better separation: a useful margin may correlate negatively with error.", "Interpolation and training error use float32 counts; float64 count mismatches are reported separately." ] }