{
  "run": {
    "run_id": "20260923T115815829404Z-human-review-baseline",
    "git_commit": "210d9857220b23d175d4aff5a54ab221ad86280b",
    "git_dirty": false,
    "finished_utc": "2026-09-23T11:59:24+00:00",
    "data_sha256": {
      "train": "bd4084611bd27c939ba98e5e63bc3e5a2c1a4e99477dcba46c829e4c986c429d",
      "test": "c2513ce4abb98c4d1d216e3ca0d4377d57589a0989aa8c06a840509a16c786e8",
      "test_labels": "2a56dcbeba5c05f965a636f56cb5ae972bad60c3b952c239b49be18d7ab70f49"
    },
    "split_label_counts": {
      "train": {
        "toxic": 9182,
        "severe_toxic": 983,
        "obscene": 5140,
        "threat": 293,
        "insult": 4776,
        "identity_hate": 854,
        "rows": 95743
      },
      "calib": {
        "toxic": 3108,
        "severe_toxic": 303,
        "obscene": 1667,
        "threat": 106,
        "insult": 1558,
        "identity_hate": 281,
        "rows": 31914
      },
      "thresh": {
        "toxic": 3004,
        "severe_toxic": 309,
        "obscene": 1642,
        "threat": 79,
        "insult": 1543,
        "identity_hate": 270,
        "rows": 31914
      },
      "test_scored": {
        "toxic": 6090,
        "severe_toxic": 367,
        "obscene": 3691,
        "threat": 211,
        "insult": 3427,
        "identity_hate": 712,
        "rows": 63978
      }
    },
    "dependencies": {
      "python": "3.12.14",
      "review_router": "0.1.0",
      "numpy": "2.5.3",
      "pandas": "3.0.6",
      "scikit-learn": "1.9.1",
      "scipy": "1.18.1",
      "pyyaml": "6.0.3"
    },
    "policy_snapshot": {
      "version": 3,
      "decision_mode": "human_confirmation",
      "tiers": {
        "allow": 1,
        "human_review": 2,
        "priority_review": 3
      },
      "severity_weights": {
        "threat": 10,
        "identity_hate": 6,
        "severe_toxic": 5,
        "insult": 2,
        "obscene": 2,
        "toxic": 1
      },
      "tier_precision_floors": {
        "human_review": 0.9,
        "priority_review": 0.95
      },
      "tier_selection_rules": {
        "priority_review": "segment_agreement",
        "human_review": "cumulative_precision"
      },
      "agreement_segments": {
        "edges": [
          0.5,
          0.8,
          0.9,
          0.95,
          0.98,
          0.99,
          0.995,
          1.0
        ],
        "high_risk_edges": [
          0.1,
          0.2,
          0.3,
          0.5,
          0.8,
          0.9,
          1.0
        ],
        "min_rows": 30,
        "use_wilson_lower_bound": false
      },
      "subgroup_thresholds": {
        "priority_review": [
          {
            "signal": "identity_term_present",
            "rationale": "Benign identity mentions can attract false positives. Before the model assigns priority review to this slice, require its observed per-label precision to meet the same 0.95 selection target at or above the population threshold. Rows that do not qualify can still enter ordinary human review or be escalated by a high-risk rule. This does not certify fairness or test-set precision, and it never permits automatic action.\n"
          }
        ]
      },
      "rules": [
        {
          "id": "R101_high_risk_priority",
          "description": "Threat predictions with p_threat >= 0.30 receive priority human review because missing a threat can be costly. This is a high-risk exception to the confidence target, not a claim of 95% precision. A reviewer confirms the case before any moderation action.\n",
          "action": "priority_review",
          "conditions": [
            {
              "signal": "p_threat",
              "op": ">=",
              "threshold": 0.3
            }
          ]
        },
        {
          "id": "R102_hierarchy_violation",
          "description": "severe_toxic is a subset of toxic in the observed training labels: no row has severe_toxic=1 with toxic=0. Independently fitted heads can violate this relation. Escalate such predictions for priority human review; the inconsistency alone does not establish distribution shift or authorize moderation without confirmation.\n",
          "action": "priority_review",
          "conditions": [
            {
              "signal": "p_severe_toxic",
              "op": ">=",
              "threshold": 0.5
            },
            {
              "signal": "p_toxic",
              "op": "<",
              "threshold": 0.5
            }
          ]
        }
      ],
      "retired_rules": [
        {
          "id": "R103_identity_term_low_confidence",
          "retired_on": "2026-09-22",
          "was": "identity_term_present == 1 AND p_max < 0.95 -> human_review, meant to suppress auto-action on borderline identity mentions.\n",
          "why": "Measured on the threshold-selection split (31,914 rows). From run 20260922T100649877069Z, made with the rule still active: the auto-action thresholds chosen from the 0.99 floor were 0.989 (toxic), 0.998 (obscene) and 0.9998 (insult), all above the rule's 0.95 band, so the rule could never intercept an auto-action, and because rules override the model tier in both directions its only effect was to move 1,825 allow-tier comments with an identity term into the human queue, of which 1,670 (92%) had no positive label, giving a queue precision of 0.404 and concentrating human scrutiny on the speech it meant to protect. From the run named in gates.eval_slice (same split, same population thresholds, rule removed): the premise check on the population tier (classifier alone, false positive per each tier's own definition), where the false discovery rate among predicted positives was 0.071 with an identity term (15 of 210) and 0.068 without (128 of 1,869), ratio 1.04, 95% interval 0.62 to 1.75: neither confirmed nor excluded for that FDR comparison. It does not establish parity in FPR among truly negative comments. Two replacements were considered. An unconditional cap (identity mentions never auto-actioned) would have turned all 102 identity-term auto-action rows on that split, every one truly positive by some label, into human reviews to prevent about two wrong-trigger enforcements; rejected as a blunt override of the floor-defined tiers. Instead the floor is required on the slice itself (subgroup_thresholds above) and the concentration is measured and gated on every run.\n",
          "replaced_by": "subgroup_thresholds.priority_review[identity_term_present] and gates.fairness"
        }
      ],
      "gates": {
        "measured_on": "2026-09-22",
        "eval_slice": "Classification references use the unchanged classifier heads: clean run 20260922T112845423138Z-baseline at commit 3429f1fe41701d31de68ac936530d6aebfee1132 supplies AP; feature-branch run 20260922T131021772191Z-baseline on a dirty worktree supplies the human operating-point precision and volume diagnostics. V2 human-review routing was measured separately in 20260922T130141204182Z-human-review-baseline on a dirty worktree. The v2 no-worse-FIFO criteria were declared before that run. The absolute capacity limits below are adopted before the new merged v2 run and are not measurements or fitted tolerances from that run.\n",
        "classification": {
          "per_label_average_precision": {
            "toxic": {
              "floor": 0.732,
              "measured": 0.7522
            },
            "obscene": {
              "floor": 0.753,
              "measured": 0.7734
            },
            "insult": {
              "floor": 0.676,
              "measured": 0.6961
            },
            "identity_hate": {
              "floor": 0.46,
              "measured": 0.48
            },
            "severe_toxic": {
              "floor": 0.291,
              "measured": 0.3113
            },
            "threat": {
              "floor": 0.438,
              "measured": 0.458
            }
          },
          "per_label_precision_at_human_threshold": {
            "toxic": {
              "floor": 0.643,
              "measured": 0.6631
            },
            "obscene": {
              "floor": 0.753,
              "measured": 0.7731
            },
            "insult": {
              "floor": 0.858,
              "measured": 0.8788
            },
            "severe_toxic": {
              "floor": null,
              "measured": null
            },
            "threat": {
              "floor": null,
              "measured": null
            },
            "identity_hate": {
              "floor": null,
              "measured": null
            }
          },
          "hierarchy_violation_rate_max": 0.005,
          "volume_overshoot_max": {
            "severe_toxic": {
              "ceiling": 1.0,
              "measured": 0.916
            }
          }
        },
        "routing": {
          "priority_review_precision_floor": null,
          "human_review_precision_floor": null
        },
        "capacity": {
          "harm_per_reviewer_hour_vs_fifo_min": 1.0,
          "high_risk_handled_vs_fifo_min": 1.0,
          "high_risk_wait_p90_vs_fifo_max": 1.0,
          "completion_ratio_min": 0.97,
          "backlog_end_max": 10,
          "wait_p50_max_min": 1.0,
          "queue_depth_p95_max": 39,
          "reviewer_utilization_min": 0.6,
          "ordering_vs_alternatives": {
            "alternatives": [
              "fifo",
              "prob",
              "priority"
            ],
            "high_risk_handled_tolerance": 1.0,
            "harm_per_reviewer_hour_tolerance": 0.5
          }
        },
        "reproducibility": {
          "float_tolerance": 1e-09,
          "records_file": "simulation_jobs.csv",
          "scenario": {
            "load": "primary",
            "seed": "first"
          }
        },
        "fairness": {
          "identity_false_discovery_rate_ratio_max": 1.25
        },
        "min_predicted_positives_for_precision": 30
      }
    },
    "decision_contract": {
      "mode": "human_confirmation",
      "requires_human_confirmation": true,
      "automatic_actions": 0
    },
    "review_workload": {
      "n_total": 63978,
      "n_requires_human_review": 6312,
      "n_priority_review": 2823,
      "n_human_review": 3489,
      "review_fraction": 0.09865891400168808,
      "n_truly_positive": 4235,
      "precision": 0.6709442332065906,
      "precision_ci95": [
        0.6592517416596075,
        0.6824287793049322
      ],
      "high_risk_by_band": {
        "priority_review": {
          "n": 2823,
          "n_high_risk": 714,
          "high_risk_share": 0.25292242295430395
        },
        "human_review": {
          "n": 3489,
          "n_high_risk": 202,
          "high_risk_share": 0.057896245342505016
        },
        "allow": {
          "n": 57666,
          "n_high_risk": 164,
          "high_risk_share": 0.002843963514029064
        }
      },
      "n_high_risk_total": 1080,
      "high_risk_recall_into_queue": 0.8481481481481481,
      "high_risk_recall_into_priority": 0.6611111111111111
    },
    "tiers": {
      "priority_review": {
        "n_predicted_positive": 2823,
        "coverage": 0.044124542811591484,
        "precision": 0.8742472546935883,
        "precision_ci95": [
          0.8615052343086654,
          0.8859721289376644
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 3489,
        "coverage": 0.054534371190096596,
        "precision": 0.5064488392089423,
        "precision_ci95": [
          0.4898614393838784,
          0.523022054049692
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57666,
        "coverage": 0.901341085998312,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "tiers_model_only": {
      "priority_review": {
        "n_predicted_positive": 2724,
        "coverage": 0.042577135890462345,
        "precision": 0.8865638766519823,
        "precision_ci95": [
          0.8741064468079365,
          0.8979325560979299
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 3570,
        "coverage": 0.05580043139829316,
        "precision": 0.5078431372549019,
        "precision_ci95": [
          0.4914440228145776,
          0.5242253907966041
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57684,
        "coverage": 0.9016224327112445,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "threshold_selection_tiers": {
      "priority_review": {
        "n_predicted_positive": 1060,
        "coverage": 0.03321426333270665,
        "precision": 0.9952830188679245,
        "precision_ci95": [
          0.989005616202792,
          0.9979835551951235
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1023,
        "coverage": 0.03205489753713104,
        "precision": 0.8709677419354839,
        "precision_ci95": [
          0.8490286461543156,
          0.8901312245689615
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 29831,
        "coverage": 0.9347308391301623,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "per_label": {
      "toxic": {
        "positives": 6090,
        "negatives": 57888,
        "average_precision": 0.7521890826317615,
        "roc_auc": 0.9569544433688153,
        "at_priority_review": {
          "threshold": 0.98,
          "n_predicted_positive": 2722,
          "precision": 0.8802351212343865,
          "recall": 0.393431855500821
        },
        "at_human_review": {
          "threshold": 0.6575178598345869,
          "n_predicted_positive": 6250,
          "precision": 0.66304,
          "recall": 0.6804597701149425
        }
      },
      "severe_toxic": {
        "positives": 367,
        "negatives": 63611,
        "average_precision": 0.31133157532520817,
        "roc_auc": 0.9819112138377519,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 3691,
        "negatives": 60287,
        "average_precision": 0.7733820574973245,
        "roc_auc": 0.971523804829942,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 1247,
          "precision": 0.9398556535685646,
          "recall": 0.3175291248984015
        },
        "at_human_review": {
          "threshold": 0.6575248417421482,
          "n_predicted_positive": 2938,
          "precision": 0.7733151803948264,
          "recall": 0.6155513410999729
        }
      },
      "threat": {
        "positives": 211,
        "negatives": 63767,
        "average_precision": 0.45793356475781777,
        "roc_auc": 0.9917276961437735,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 3427,
        "negatives": 60551,
        "average_precision": 0.696041642970931,
        "roc_auc": 0.9654014764914653,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 437,
          "precision": 0.9427917620137299,
          "recall": 0.12022176831047564
        },
        "at_human_review": {
          "threshold": 0.917856881186332,
          "n_predicted_positive": 1126,
          "precision": 0.8783303730017762,
          "recall": 0.28859060402684567
        }
      },
      "identity_hate": {
        "positives": 712,
        "negatives": 63266,
        "average_precision": 0.4799206224609285,
        "roc_auc": 0.9739760950465255,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "threshold_selection_per_label": {
      "toxic": {
        "positives": 3004,
        "negatives": 28910,
        "average_precision": 0.8591605013116437,
        "roc_auc": 0.9711163047448322,
        "at_priority_review": {
          "threshold": 0.98,
          "n_predicted_positive": 1039,
          "precision": 0.9865255052935515,
          "recall": 0.34121171770972036
        },
        "at_human_review": {
          "threshold": 0.6575178598345869,
          "n_predicted_positive": 2074,
          "precision": 0.9001928640308583,
          "recall": 0.6215046604527297
        }
      },
      "severe_toxic": {
        "positives": 309,
        "negatives": 31605,
        "average_precision": 0.49208182986820953,
        "roc_auc": 0.9875912162110272,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 1642,
        "negatives": 30272,
        "average_precision": 0.8739265303249879,
        "roc_auc": 0.9852955312354345,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 572,
          "precision": 0.986013986013986,
          "recall": 0.34348355663824603
        },
        "at_human_review": {
          "threshold": 0.6575248417421482,
          "n_predicted_positive": 1177,
          "precision": 0.9005947323704333,
          "recall": 0.6455542021924482
        }
      },
      "threat": {
        "positives": 79,
        "negatives": 31835,
        "average_precision": 0.45854647319377284,
        "roc_auc": 0.9861334054350657,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 1543,
        "negatives": 30371,
        "average_precision": 0.7743441564479363,
        "roc_auc": 0.9778964622274467,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 209,
          "precision": 0.9617224880382775,
          "recall": 0.13026571613739468
        },
        "at_human_review": {
          "threshold": 0.917856881186332,
          "n_predicted_positive": 525,
          "precision": 0.900952380952381,
          "recall": 0.30654569021386907
        }
      },
      "identity_hate": {
        "positives": 270,
        "negatives": 31644,
        "average_precision": 0.4145628903623871,
        "roc_auc": 0.9700846687921647,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "rules": {
      "matched_per_rule": {
        "R101_high_risk_priority": 267,
        "R102_hierarchy_violation": 0
      },
      "n_rows_with_any_rule": 267,
      "n_added_to_queue_from_allow": 18,
      "n_added_to_queue_from_allow_true_positive": 7,
      "n_promoted_to_priority_review": 99,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "consistency": {
      "hierarchy_violation_rate": 0.0,
      "n_violations": 0,
      "n_rows": 63978
    },
    "allow_false_negatives": {
      "toxic": 1913,
      "severe_toxic": 6,
      "obscene": 721,
      "threat": 24,
      "insult": 694,
      "identity_hate": 140
    },
    "prevalence_shift": {
      "definition": "volume_overshoot = model-expected positives (sum of calibrated p on the test rows) / actual positives - 1; prevalence ratio = training-file prevalence / test prevalence",
      "n_train_file_rows": 159571,
      "n_test_rows": 63978,
      "toxic": {
        "prevalence_train_file": 0.09584448302009764,
        "prevalence_test": 0.09518897120885304,
        "prevalence_ratio_train_over_test": 1.0068864260525134,
        "actual_positives_test": 6090,
        "model_expected_positives_test": 8715.25371014754,
        "train_prevalence_expected_positives_test": 6131.938334659807,
        "volume_overshoot_model": 0.4310761428813694,
        "volume_overshoot_train_prevalence": 0.006886426052513439
      },
      "severe_toxic": {
        "prevalence_train_file": 0.009995550569965721,
        "prevalence_test": 0.0057363468692363,
        "prevalence_ratio_train_over_test": 1.7424940990879207,
        "actual_positives_test": 367,
        "model_expected_positives_test": 702.9369953219017,
        "train_prevalence_expected_positives_test": 639.4953343652669,
        "volume_overshoot_model": 0.9153596602776612,
        "volume_overshoot_train_prevalence": 0.7424940990879207
      },
      "obscene": {
        "prevalence_train_file": 0.052948217407925,
        "prevalence_test": 0.05769170652411767,
        "prevalence_ratio_train_over_test": 0.9177786652192429,
        "actual_positives_test": 3691,
        "model_expected_positives_test": 4359.793338728759,
        "train_prevalence_expected_positives_test": 3387.5210533242257,
        "volume_overshoot_model": 0.18119570271708474,
        "volume_overshoot_train_prevalence": -0.0822213347807571
      },
      "threat": {
        "prevalence_train_file": 0.002995531769557125,
        "prevalence_test": 0.003298008690487355,
        "prevalence_ratio_train_over_test": 0.9082849836622073,
        "actual_positives_test": 211,
        "model_expected_positives_test": 307.63654287949777,
        "train_prevalence_expected_positives_test": 191.64813155272574,
        "volume_overshoot_model": 0.45799309421562917,
        "volume_overshoot_train_prevalence": -0.09171501633779267
      },
      "insult": {
        "prevalence_train_file": 0.04936360616904074,
        "prevalence_test": 0.053565288067773296,
        "prevalence_ratio_train_over_test": 0.9215596135053657,
        "actual_positives_test": 3427,
        "model_expected_positives_test": 3778.64329101567,
        "train_prevalence_expected_positives_test": 3158.184795482888,
        "volume_overshoot_model": 0.10260965597189076,
        "volume_overshoot_train_prevalence": -0.07844038649463436
      },
      "identity_hate": {
        "prevalence_train_file": 0.00880485802558109,
        "prevalence_test": 0.011128825533777236,
        "prevalence_ratio_train_over_test": 0.7911758521918918,
        "actual_positives_test": 712,
        "model_expected_positives_test": 776.4930010909766,
        "train_prevalence_expected_positives_test": 563.317206760627,
        "volume_overshoot_model": 0.09058005771204569,
        "volume_overshoot_train_prevalence": -0.2088241478081082
      }
    },
    "simulation": {
      "note": null,
      "assumptions": {
        "reviewers": 4,
        "handle_minutes": 2.0,
        "horizon_hours": 8.0,
        "capacity_per_hour": 120.0,
        "arrivals": "Post-admission Poisson review arrivals; jobs sampled with replacement from the combined priority_review and human_review pool",
        "arrival_rates_are": "post-admission review demand, not all incoming comments",
        "queue_tiers": [
          "priority_review",
          "human_review"
        ],
        "queue_fraction": 0.09865891400168808,
        "equivalent_input_loads_per_hour": [
          608.1558935361217,
          1094.680608365019,
          1824.467680608365
        ],
        "priority_order": "priority_review first, then human_review; within each tier, higher max predicted probability times severity weight first; ties use arrival order",
        "primary_order": "highest max predicted probability times severity weight first; ties use arrival order",
        "handle_time": "deterministic",
        "review_completion_definition": "simulated reviewer service finished; moderation requires human confirmation; no enforcement outcome is observed",
        "wait_definition": "start of review minus arrival, over jobs that started (completed or in progress); never-started jobs are counted, not timed",
        "completion_latency_definition": "completion minus arrival, over jobs completed within the horizon",
        "p99_min_samples": 100,
        "harm_proxy": "max severity weight over TRUE labels (0 if clean); not real-world harm",
        "high_risk": "harm proxy >= 5.0",
        "loads_per_hour": [
          60.0,
          108.0,
          180.0
        ],
        "seeds": [
          1,
          2,
          3,
          4,
          5
        ],
        "queued_jobs": 6312,
        "queued_high_risk": 916,
        "queued_priority_review": 2823,
        "queued_human_review": 3489,
        "per_job_records": "simulation_jobs.csv in the run directory; every time metric here is recomputable from it"
      },
      "summary": {
        "fifo@60": {
          "strategy": "fifo",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "n_not_started": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "n_in_progress": {
            "mean": 1.6,
            "std": 1.3564659966250536
          },
          "high_risk_arrived": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_handled": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 881.4,
            "std": 80.05897825978046
          },
          "harm_handled": {
            "mean": 878.8,
            "std": 79.45665485030187
          },
          "harm_per_reviewer_hour": {
            "mean": 27.4625,
            "std": 2.4830204640719336
          },
          "completion_ratio": {
            "mean": 0.9959142142714512,
            "std": 0.004082943871166383
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.3028671220757165,
            "std": 0.14791052737540858
          },
          "wait_p99": {
            "mean": 1.419295950982092,
            "std": 0.3738634836049926
          },
          "n_wait_samples": {
            "mean": 469.4,
            "std": 27.39781013146854
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.3169906926896496,
            "std": 0.2059577923472864
          },
          "high_risk_wait_p99": {
            "mean": 1.3114052063239892,
            "std": 0.12076194444667335
          },
          "n_high_risk_wait_samples": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "completion_latency_p50": {
            "mean": 2.0,
            "std": 0.0
          },
          "completion_latency_p90": {
            "mean": 2.2973983211717988,
            "std": 0.15731102575811837
          },
          "completion_latency_p99": {
            "mean": 3.4205222295946016,
            "std": 0.37486940307942956
          },
          "n_completion_samples": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "unfinished_age_p50": {
            "mean": 1.1027421610170123,
            "std": 0.34420918537509
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "prob@60": {
          "strategy": "prob",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "n_not_started": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "n_in_progress": {
            "mean": 1.6,
            "std": 1.3564659966250536
          },
          "high_risk_arrived": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_handled": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 881.4,
            "std": 80.05897825978046
          },
          "harm_handled": {
            "mean": 878.8,
            "std": 79.45665485030187
          },
          "harm_per_reviewer_hour": {
            "mean": 27.4625,
            "std": 2.4830204640719336
          },
          "completion_ratio": {
            "mean": 0.9959142142714512,
            "std": 0.004082943871166383
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.26168513651839725,
            "std": 0.14239717249384154
          },
          "wait_p99": {
            "mean": 1.4610233376344521,
            "std": 0.2579004773888722
          },
          "n_wait_samples": {
            "mean": 469.4,
            "std": 27.39781013146854
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.174572628949454,
            "std": 0.11082679717123527
          },
          "high_risk_wait_p99": {
            "mean": 0.9339724361086521,
            "std": 0.26031288060457286
          },
          "n_high_risk_wait_samples": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "completion_latency_p50": {
            "mean": 2.0,
            "std": 0.0
          },
          "completion_latency_p90": {
            "mean": 2.2561645504130405,
            "std": 0.1481910309910408
          },
          "completion_latency_p99": {
            "mean": 3.462230579436259,
            "std": 0.25862682256109437
          },
          "n_completion_samples": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "unfinished_age_p50": {
            "mean": 1.1027421610170123,
            "std": 0.34420918537509
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "severity@60": {
          "strategy": "severity",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "n_not_started": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "n_in_progress": {
            "mean": 1.6,
            "std": 1.3564659966250536
          },
          "high_risk_arrived": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_handled": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 881.4,
            "std": 80.05897825978046
          },
          "harm_handled": {
            "mean": 878.8,
            "std": 79.45665485030187
          },
          "harm_per_reviewer_hour": {
            "mean": 27.4625,
            "std": 2.4830204640719336
          },
          "completion_ratio": {
            "mean": 0.9959142142714512,
            "std": 0.004082943871166383
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.258364660209897,
            "std": 0.14078894805810244
          },
          "wait_p99": {
            "mean": 1.3820592130413816,
            "std": 0.21454578859671472
          },
          "n_wait_samples": {
            "mean": 469.4,
            "std": 27.39781013146854
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.17666131386969464,
            "std": 0.11404568518850416
          },
          "high_risk_wait_p99": {
            "mean": 0.8254183250093152,
            "std": 0.22325591296290487
          },
          "n_high_risk_wait_samples": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "completion_latency_p50": {
            "mean": 2.0,
            "std": 0.0
          },
          "completion_latency_p90": {
            "mean": 2.2524652812083055,
            "std": 0.14588358612406174
          },
          "completion_latency_p99": {
            "mean": 3.382553155769434,
            "std": 0.21450805128102238
          },
          "n_completion_samples": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "unfinished_age_p50": {
            "mean": 1.1027421610170123,
            "std": 0.34420918537509
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "priority@60": {
          "strategy": "priority",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "n_not_started": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "n_in_progress": {
            "mean": 1.6,
            "std": 1.3564659966250536
          },
          "high_risk_arrived": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_handled": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 881.4,
            "std": 80.05897825978046
          },
          "harm_handled": {
            "mean": 878.8,
            "std": 79.45665485030187
          },
          "harm_per_reviewer_hour": {
            "mean": 27.4625,
            "std": 2.4830204640719336
          },
          "completion_ratio": {
            "mean": 0.9959142142714512,
            "std": 0.004082943871166383
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.25865059699765036,
            "std": 0.1393929564983064
          },
          "wait_p99": {
            "mean": 1.3820592130413816,
            "std": 0.21454578859671472
          },
          "n_wait_samples": {
            "mean": 469.4,
            "std": 27.39781013146854
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.17666131386969464,
            "std": 0.11404568518850416
          },
          "high_risk_wait_p99": {
            "mean": 0.8664621551052782,
            "std": 0.2581168785572972
          },
          "n_high_risk_wait_samples": {
            "mean": 66.2,
            "std": 8.657944328765344
          },
          "completion_latency_p50": {
            "mean": 2.0,
            "std": 0.0
          },
          "completion_latency_p90": {
            "mean": 2.25260057614087,
            "std": 0.14449908259042715
          },
          "completion_latency_p99": {
            "mean": 3.382553155769434,
            "std": 0.21450805128102238
          },
          "n_completion_samples": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "unfinished_age_p50": {
            "mean": 1.1027421610170123,
            "std": 0.34420918537509
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "fifo@108": {
          "strategy": "fifo",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "n_not_started": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "n_in_progress": {
            "mean": 3.8,
            "std": 0.39999999999999997
          },
          "high_risk_arrived": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "high_risk_handled": {
            "mean": 122.6,
            "std": 13.215142829345432
          },
          "high_risk_unhandled": {
            "mean": 0.4,
            "std": 0.48989794855663565
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.4,
            "std": 0.48989794855663565
          },
          "harm_arrived": {
            "mean": 1626.4,
            "std": 98.66225215349587
          },
          "harm_handled": {
            "mean": 1619.8,
            "std": 98.91491293025537
          },
          "harm_per_reviewer_hour": {
            "mean": 50.61875,
            "std": 3.0910910290704803
          },
          "completion_ratio": {
            "mean": 0.994271372623818,
            "std": 0.002060062206193563
          },
          "wait_p50": {
            "mean": 1.6389207147892448,
            "std": 0.9728608303463531
          },
          "wait_p90": {
            "mean": 5.975672164523138,
            "std": 2.8710305362700184
          },
          "wait_p99": {
            "mean": 8.772770425121845,
            "std": 3.264002671028306
          },
          "n_wait_samples": {
            "mean": 871.2,
            "std": 20.478281177872326
          },
          "high_risk_wait_p50": {
            "mean": 1.5856383492670318,
            "std": 0.8404862579633662
          },
          "high_risk_wait_p90": {
            "mean": 5.945219363009528,
            "std": 2.734276976471336
          },
          "high_risk_wait_p99": {
            "mean": 8.741860239279971,
            "std": 3.14395173846065
          },
          "n_high_risk_wait_samples": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "completion_latency_p50": {
            "mean": 3.6465033420211457,
            "std": 0.9788027642000939
          },
          "completion_latency_p90": {
            "mean": 7.989271534385042,
            "std": 2.8671244892364833
          },
          "completion_latency_p99": {
            "mean": 10.776882230883988,
            "std": 3.264786257620957
          },
          "n_completion_samples": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "unfinished_age_p50": {
            "mean": 1.3518858971024827,
            "std": 0.468387208093191
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "prob@108": {
          "strategy": "prob",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "n_not_started": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "n_in_progress": {
            "mean": 3.8,
            "std": 0.39999999999999997
          },
          "high_risk_arrived": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "high_risk_handled": {
            "mean": 122.8,
            "std": 13.075167302944921
          },
          "high_risk_unhandled": {
            "mean": 0.2,
            "std": 0.4
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.2,
            "std": 0.4
          },
          "harm_arrived": {
            "mean": 1626.4,
            "std": 98.66225215349587
          },
          "harm_handled": {
            "mean": 1621.2,
            "std": 98.82388375286614
          },
          "harm_per_reviewer_hour": {
            "mean": 50.6625,
            "std": 3.088246367277067
          },
          "completion_ratio": {
            "mean": 0.994271372623818,
            "std": 0.002060062206193563
          },
          "wait_p50": {
            "mean": 0.41262580704952684,
            "std": 0.07604298693702577
          },
          "wait_p90": {
            "mean": 3.94783153240163,
            "std": 0.9795740989081109
          },
          "wait_p99": {
            "mean": 40.27008781987795,
            "std": 29.276575072222606
          },
          "n_wait_samples": {
            "mean": 871.2,
            "std": 20.478281177872326
          },
          "high_risk_wait_p50": {
            "mean": 0.251077097821608,
            "std": 0.0373543699058921
          },
          "high_risk_wait_p90": {
            "mean": 1.1891239385917647,
            "std": 0.11488940301034165
          },
          "high_risk_wait_p99": {
            "mean": 12.093075659819744,
            "std": 10.018514882672644
          },
          "n_high_risk_wait_samples": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "completion_latency_p50": {
            "mean": 2.413570737982442,
            "std": 0.07544709519386232
          },
          "completion_latency_p90": {
            "mean": 5.96697274349836,
            "std": 0.9859785336956857
          },
          "completion_latency_p99": {
            "mean": 42.33668886080274,
            "std": 29.323875560548274
          },
          "n_completion_samples": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "unfinished_age_p50": {
            "mean": 1.4723602926751482,
            "std": 0.601214289819207
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "severity@108": {
          "strategy": "severity",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "n_not_started": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "n_in_progress": {
            "mean": 3.8,
            "std": 0.39999999999999997
          },
          "high_risk_arrived": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "high_risk_handled": {
            "mean": 122.8,
            "std": 13.075167302944921
          },
          "high_risk_unhandled": {
            "mean": 0.2,
            "std": 0.4
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.2,
            "std": 0.4
          },
          "harm_arrived": {
            "mean": 1626.4,
            "std": 98.66225215349587
          },
          "harm_handled": {
            "mean": 1620.8,
            "std": 98.2270838414742
          },
          "harm_per_reviewer_hour": {
            "mean": 50.65,
            "std": 3.0695963700460687
          },
          "completion_ratio": {
            "mean": 0.994271372623818,
            "std": 0.002060062206193563
          },
          "wait_p50": {
            "mean": 0.4116613934067381,
            "std": 0.07640714604576315
          },
          "wait_p90": {
            "mean": 4.407067368426992,
            "std": 1.9783330561563401
          },
          "wait_p99": {
            "mean": 39.490760110590124,
            "std": 28.21117781809085
          },
          "n_wait_samples": {
            "mean": 871.2,
            "std": 20.478281177872326
          },
          "high_risk_wait_p50": {
            "mean": 0.2170008342457706,
            "std": 0.038684304518486354
          },
          "high_risk_wait_p90": {
            "mean": 1.0002904639523633,
            "std": 0.10465381175589338
          },
          "high_risk_wait_p99": {
            "mean": 10.235683456426015,
            "std": 9.031593173606987
          },
          "n_high_risk_wait_samples": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "completion_latency_p50": {
            "mean": 2.4137096564685208,
            "std": 0.07676081506697482
          },
          "completion_latency_p90": {
            "mean": 6.457001344304855,
            "std": 2.0175700874777798
          },
          "completion_latency_p99": {
            "mean": 41.5329642234095,
            "std": 28.250686774187187
          },
          "n_completion_samples": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "unfinished_age_p50": {
            "mean": 1.4723602926751482,
            "std": 0.601214289819207
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "priority@108": {
          "strategy": "priority",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "n_not_started": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "n_in_progress": {
            "mean": 3.8,
            "std": 0.39999999999999997
          },
          "high_risk_arrived": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "high_risk_handled": {
            "mean": 122.8,
            "std": 13.075167302944921
          },
          "high_risk_unhandled": {
            "mean": 0.2,
            "std": 0.4
          },
          "high_risk_not_started": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_in_progress": {
            "mean": 0.2,
            "std": 0.4
          },
          "harm_arrived": {
            "mean": 1626.4,
            "std": 98.66225215349587
          },
          "harm_handled": {
            "mean": 1621.2,
            "std": 98.82388375286614
          },
          "harm_per_reviewer_hour": {
            "mean": 50.6625,
            "std": 3.088246367277067
          },
          "completion_ratio": {
            "mean": 0.994271372623818,
            "std": 0.002060062206193563
          },
          "wait_p50": {
            "mean": 0.4094986402851468,
            "std": 0.07789204736910166
          },
          "wait_p90": {
            "mean": 4.402187984896975,
            "std": 1.96718215110001
          },
          "wait_p99": {
            "mean": 39.490760110590124,
            "std": 28.21117781809085
          },
          "n_wait_samples": {
            "mean": 871.2,
            "std": 20.478281177872326
          },
          "high_risk_wait_p50": {
            "mean": 0.22078731802748522,
            "std": 0.03768871478776951
          },
          "high_risk_wait_p90": {
            "mean": 1.0561364877842803,
            "std": 0.08896516076554545
          },
          "high_risk_wait_p99": {
            "mean": 10.235683456426015,
            "std": 9.031593173606987
          },
          "n_high_risk_wait_samples": {
            "mean": 123.0,
            "std": 12.96148139681572
          },
          "completion_latency_p50": {
            "mean": 2.4113586527038002,
            "std": 0.07877140351541745
          },
          "completion_latency_p90": {
            "mean": 6.436857693945361,
            "std": 2.0191042633885896
          },
          "completion_latency_p99": {
            "mean": 41.5329642234095,
            "std": 28.250686774187187
          },
          "n_completion_samples": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "unfinished_age_p50": {
            "mean": 1.4723602926751482,
            "std": 0.601214289819207
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "fifo@180": {
          "strategy": "fifo",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "n_not_started": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "n_in_progress": {
            "mean": 4.0,
            "std": 0.0
          },
          "high_risk_arrived": {
            "mean": 200.8,
            "std": 14.372195378577345
          },
          "high_risk_handled": {
            "mean": 134.0,
            "std": 9.055385138137417
          },
          "high_risk_unhandled": {
            "mean": 66.8,
            "std": 9.57914401186244
          },
          "high_risk_not_started": {
            "mean": 66.4,
            "std": 8.822698000045111
          },
          "high_risk_in_progress": {
            "mean": 0.4,
            "std": 0.8
          },
          "harm_arrived": {
            "mean": 2666.0,
            "std": 127.42370266163199
          },
          "harm_handled": {
            "mean": 1784.2,
            "std": 68.51394018738084
          },
          "harm_per_reviewer_hour": {
            "mean": 55.75625,
            "std": 2.1410606308556512
          },
          "completion_ratio": {
            "mean": 0.6709026783635214,
            "std": 0.018797071090613824
          },
          "wait_p50": {
            "mean": 79.17617573141584,
            "std": 2.500599098867411
          },
          "wait_p90": {
            "mean": 140.38217390295523,
            "std": 7.920355762876184
          },
          "wait_p99": {
            "mean": 154.63036998207275,
            "std": 7.511653382607061
          },
          "n_wait_samples": {
            "mean": 957.2,
            "std": 3.9191835884530852
          },
          "high_risk_wait_p50": {
            "mean": 82.18092859999722,
            "std": 5.848254152942134
          },
          "high_risk_wait_p90": {
            "mean": 140.03966011303706,
            "std": 8.540235050400238
          },
          "high_risk_wait_p99": {
            "mean": 153.50243890704115,
            "std": 8.203500895811251
          },
          "n_high_risk_wait_samples": {
            "mean": 134.4,
            "std": 9.43610088966836
          },
          "completion_latency_p50": {
            "mean": 80.75870506496214,
            "std": 2.4070498005840677
          },
          "completion_latency_p90": {
            "mean": 141.86341084347154,
            "std": 7.777914356926516
          },
          "completion_latency_p99": {
            "mean": 155.92826700199763,
            "std": 7.661522457419192
          },
          "n_completion_samples": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "unfinished_age_p50": {
            "mean": 79.00371900277469,
            "std": 4.0760856901089815
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "prob@180": {
          "strategy": "prob",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "n_not_started": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "n_in_progress": {
            "mean": 4.0,
            "std": 0.0
          },
          "high_risk_arrived": {
            "mean": 200.8,
            "std": 14.372195378577345
          },
          "high_risk_handled": {
            "mean": 176.2,
            "std": 10.906878563548783
          },
          "high_risk_unhandled": {
            "mean": 24.6,
            "std": 4.127953488110059
          },
          "high_risk_not_started": {
            "mean": 23.8,
            "std": 4.069397989875161
          },
          "high_risk_in_progress": {
            "mean": 0.8,
            "std": 0.7483314773547883
          },
          "harm_arrived": {
            "mean": 2666.0,
            "std": 127.42370266163199
          },
          "harm_handled": {
            "mean": 2191.6,
            "std": 81.98682821038999
          },
          "harm_per_reviewer_hour": {
            "mean": 68.4875,
            "std": 2.562088381574687
          },
          "completion_ratio": {
            "mean": 0.6709026783635214,
            "std": 0.018797071090613824
          },
          "wait_p50": {
            "mean": 0.7146777601585569,
            "std": 0.23027037164871222
          },
          "wait_p90": {
            "mean": 11.680128325261979,
            "std": 1.5080455229190297
          },
          "wait_p99": {
            "mean": 119.2418261269891,
            "std": 24.05571398621287
          },
          "n_wait_samples": {
            "mean": 957.2,
            "std": 3.9191835884530852
          },
          "high_risk_wait_p50": {
            "mean": 0.5140003746662842,
            "std": 0.24696735250061475
          },
          "high_risk_wait_p90": {
            "mean": 2.2865115802600866,
            "std": 0.663439828196621
          },
          "high_risk_wait_p99": {
            "mean": 30.57707075295692,
            "std": 11.044669415582353
          },
          "n_high_risk_wait_samples": {
            "mean": 177.0,
            "std": 11.260550608207398
          },
          "completion_latency_p50": {
            "mean": 2.7148593213436882,
            "std": 0.22993636424312258
          },
          "completion_latency_p90": {
            "mean": 13.410996889317506,
            "std": 1.331016040798225
          },
          "completion_latency_p99": {
            "mean": 120.29714915000275,
            "std": 22.814544072218585
          },
          "n_completion_samples": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "unfinished_age_p50": {
            "mean": 221.859350819784,
            "std": 7.65336021881225
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "severity@180": {
          "strategy": "severity",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "n_not_started": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "n_in_progress": {
            "mean": 4.0,
            "std": 0.0
          },
          "high_risk_arrived": {
            "mean": 200.8,
            "std": 14.372195378577345
          },
          "high_risk_handled": {
            "mean": 184.6,
            "std": 10.011992808627062
          },
          "high_risk_unhandled": {
            "mean": 16.2,
            "std": 4.534313619501853
          },
          "high_risk_not_started": {
            "mean": 15.8,
            "std": 4.66476151587624
          },
          "high_risk_in_progress": {
            "mean": 0.4,
            "std": 0.48989794855663565
          },
          "harm_arrived": {
            "mean": 2666.0,
            "std": 127.42370266163199
          },
          "harm_handled": {
            "mean": 2273.4,
            "std": 66.85686202627222
          },
          "harm_per_reviewer_hour": {
            "mean": 71.04375,
            "std": 2.089276938321007
          },
          "completion_ratio": {
            "mean": 0.6709026783635214,
            "std": 0.018797071090613824
          },
          "wait_p50": {
            "mean": 0.7016711115172711,
            "std": 0.23234004179630757
          },
          "wait_p90": {
            "mean": 11.118244255110334,
            "std": 1.6835765739949913
          },
          "wait_p99": {
            "mean": 114.09032813381116,
            "std": 50.07421921852515
          },
          "n_wait_samples": {
            "mean": 957.2,
            "std": 3.9191835884530852
          },
          "high_risk_wait_p50": {
            "mean": 0.48522631771927394,
            "std": 0.25276053792925407
          },
          "high_risk_wait_p90": {
            "mean": 1.6394874852877745,
            "std": 0.27884376868732297
          },
          "high_risk_wait_p99": {
            "mean": 23.232417986309166,
            "std": 11.432859098372287
          },
          "n_high_risk_wait_samples": {
            "mean": 185.0,
            "std": 10.039920318408907
          },
          "completion_latency_p50": {
            "mean": 2.7010137986980327,
            "std": 0.23228825317917642
          },
          "completion_latency_p90": {
            "mean": 13.027675763145817,
            "std": 1.616637866034761
          },
          "completion_latency_p99": {
            "mean": 115.16829708998678,
            "std": 50.77526332922067
          },
          "n_completion_samples": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "unfinished_age_p50": {
            "mean": 219.26287640821343,
            "std": 6.9052412203838305
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "priority@180": {
          "strategy": "priority",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "n_not_started": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "n_in_progress": {
            "mean": 4.0,
            "std": 0.0
          },
          "high_risk_arrived": {
            "mean": 200.8,
            "std": 14.372195378577345
          },
          "high_risk_handled": {
            "mean": 184.2,
            "std": 10.087616170334794
          },
          "high_risk_unhandled": {
            "mean": 16.6,
            "std": 4.409081537009721
          },
          "high_risk_not_started": {
            "mean": 16.2,
            "std": 4.534313619501853
          },
          "high_risk_in_progress": {
            "mean": 0.4,
            "std": 0.48989794855663565
          },
          "harm_arrived": {
            "mean": 2666.0,
            "std": 127.42370266163199
          },
          "harm_handled": {
            "mean": 2268.8,
            "std": 68.63060541769975
          },
          "harm_per_reviewer_hour": {
            "mean": 70.9,
            "std": 2.1447064193031173
          },
          "completion_ratio": {
            "mean": 0.6709026783635214,
            "std": 0.018797071090613824
          },
          "wait_p50": {
            "mean": 0.7097105932328048,
            "std": 0.23208072313527775
          },
          "wait_p90": {
            "mean": 11.318699997516916,
            "std": 1.176879331227942
          },
          "wait_p99": {
            "mean": 113.1084310632472,
            "std": 49.26874831145469
          },
          "n_wait_samples": {
            "mean": 957.2,
            "std": 3.9191835884530852
          },
          "high_risk_wait_p50": {
            "mean": 0.5009022844652058,
            "std": 0.24606886548216308
          },
          "high_risk_wait_p90": {
            "mean": 2.0096949018676824,
            "std": 0.28890174048803635
          },
          "high_risk_wait_p99": {
            "mean": 21.85632336905597,
            "std": 6.990127280638906
          },
          "n_high_risk_wait_samples": {
            "mean": 184.6,
            "std": 10.11137972781163
          },
          "completion_latency_p50": {
            "mean": 2.7103375426795937,
            "std": 0.2314452483013916
          },
          "completion_latency_p90": {
            "mean": 13.176508424135756,
            "std": 1.1466635656947386
          },
          "completion_latency_p99": {
            "mean": 115.37745894995408,
            "std": 49.362764675124524
          },
          "n_completion_samples": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "unfinished_age_p50": {
            "mean": 219.26287640821343,
            "std": 6.9052412203838305
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        }
      },
      "paired": {
        "severity_vs_fifo@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.14032937881995491,
            "min_diff": -0.3333859982813291,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          },
          "wait_p90": {
            "mean_diff": -0.04450246186581946,
            "min_diff": -0.08368705798288811,
            "max_diff": -0.014550602187256345,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_prob@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": 0.0020886849202406466,
            "min_diff": 0.0,
            "max_diff": 0.010443424601203233,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "wait_p90": {
            "mean_diff": -0.003320476308500281,
            "min_diff": -0.009080314906818598,
            "max_diff": 0.00640504172395126,
            "n_seeds": 5,
            "n_seeds_first_better": 3,
            "n_seeds_tied": 1
          }
        },
        "severity_vs_priority@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "wait_p90": {
            "mean_diff": -0.00028593678775339917,
            "min_diff": -0.007834725662718256,
            "max_diff": 0.00640504172395126,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 3
          }
        },
        "prob_vs_fifo@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.14241806374019556,
            "min_diff": -0.3333859982813291,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          },
          "wait_p90": {
            "mean_diff": -0.04118198555731918,
            "min_diff": -0.08368705798288811,
            "max_diff": -0.006715876524538089,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_fifo@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.14032937881995491,
            "min_diff": -0.3333859982813291,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          },
          "wait_p90": {
            "mean_diff": -0.04421652507806606,
            "min_diff": -0.08368705798288811,
            "max_diff": -0.006715876524538089,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_prob@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": 0.0020886849202406466,
            "min_diff": 0.0,
            "max_diff": 0.010443424601203233,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "wait_p90": {
            "mean_diff": -0.0030345395207468815,
            "min_diff": -0.009080314906818598,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          }
        },
        "severity_vs_fifo@108": {
          "high_risk_handled": {
            "mean_diff": 0.2,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.03125,
            "min_diff": 0.0,
            "max_diff": 0.15625,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": -0.2,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "high_risk_wait_p90": {
            "mean_diff": -4.944928899057165,
            "min_diff": -10.21262843772127,
            "max_diff": -2.871438586130459,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -1.5686047960961458,
            "min_diff": -3.324446662323494,
            "max_diff": -0.10316822734513575,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_prob@108": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": -0.0125,
            "min_diff": -0.0625,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.1888334746394015,
            "min_diff": -0.2912093152582371,
            "max_diff": -0.08939328694048232,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": 0.45923583602536155,
            "min_diff": -0.47971690440871395,
            "max_diff": 2.420588249429752,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_priority@108": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": -0.0125,
            "min_diff": -0.0625,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.055846023831916905,
            "min_diff": -0.15324190905586477,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          },
          "wait_p90": {
            "mean_diff": 0.004879383530015424,
            "min_diff": -0.19463737793329727,
            "max_diff": 0.16667059090026592,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 2
          }
        },
        "prob_vs_fifo@108": {
          "high_risk_handled": {
            "mean_diff": 0.2,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.04375,
            "min_diff": 0.0,
            "max_diff": 0.15625,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          },
          "high_risk_unhandled": {
            "mean_diff": -0.2,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "high_risk_wait_p90": {
            "mean_diff": -4.7560954244177625,
            "min_diff": -10.051852988073634,
            "max_diff": -2.6089047042437326,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -2.0278406321215074,
            "min_diff": -5.745034911753246,
            "max_diff": -0.17234304949597945,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_fifo@108": {
          "high_risk_handled": {
            "mean_diff": 0.2,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.04375,
            "min_diff": 0.0,
            "max_diff": 0.15625,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          },
          "high_risk_unhandled": {
            "mean_diff": -0.2,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "high_risk_wait_p90": {
            "mean_diff": -4.889082875225247,
            "min_diff": -10.21262843772127,
            "max_diff": -2.7181966770745944,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -1.5734841796261612,
            "min_diff": -3.324446662323494,
            "max_diff": -0.10316822734513575,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_prob@108": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.1329874508074846,
            "min_diff": -0.16522110515451738,
            "max_diff": -0.08939328694048232,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": 0.4543564524953461,
            "min_diff": -0.2850795264754167,
            "max_diff": 2.420588249429752,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_fifo@180": {
          "high_risk_handled": {
            "mean_diff": 50.6,
            "min_diff": 43.0,
            "max_diff": 60.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 15.2875,
            "min_diff": 13.21875,
            "max_diff": 17.53125,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -50.6,
            "min_diff": -60.0,
            "max_diff": -43.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -138.40017262774927,
            "min_diff": -155.19807945227384,
            "max_diff": -132.00950756426417,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -129.2639296478449,
            "min_diff": -144.48713018876592,
            "max_diff": -123.3091348944762,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_prob@180": {
          "high_risk_handled": {
            "mean_diff": 8.4,
            "min_diff": 6.0,
            "max_diff": 12.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 2.55625,
            "min_diff": 1.34375,
            "max_diff": 3.0625,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -8.4,
            "min_diff": -12.0,
            "max_diff": -6.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.647024094972312,
            "min_diff": -1.725661936481346,
            "max_diff": 0.1285889781780163,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -0.5618840701516451,
            "min_diff": -1.0525338707670073,
            "max_diff": -0.07603175262210726,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_priority@180": {
          "high_risk_handled": {
            "mean_diff": 0.4,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.14375,
            "min_diff": 0.0,
            "max_diff": 0.375,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          },
          "high_risk_unhandled": {
            "mean_diff": -0.4,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 3
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.37020741657990774,
            "min_diff": -1.1478436993958172,
            "max_diff": -0.032093299353744165,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -0.20045574240658331,
            "min_diff": -0.943300975029711,
            "max_diff": 0.39982615461524595,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 0
          }
        },
        "prob_vs_fifo@180": {
          "high_risk_handled": {
            "mean_diff": 42.2,
            "min_diff": 34.0,
            "max_diff": 54.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 12.73125,
            "min_diff": 10.28125,
            "max_diff": 16.1875,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -42.2,
            "min_diff": -54.0,
            "max_diff": -34.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -137.75314853277695,
            "min_diff": -153.9777941755602,
            "max_diff": -130.28384562778282,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -128.70204557769324,
            "min_diff": -144.41109843614382,
            "max_diff": -123.14483849254255,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_fifo@180": {
          "high_risk_handled": {
            "mean_diff": 50.2,
            "min_diff": 43.0,
            "max_diff": 60.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 15.14375,
            "min_diff": 13.21875,
            "max_diff": 17.46875,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -50.2,
            "min_diff": -60.0,
            "max_diff": -43.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -138.02996521116935,
            "min_diff": -154.05023575287802,
            "max_diff": -131.60109958036858,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -129.0634739054383,
            "min_diff": -144.71306934907756,
            "max_diff": -123.50575903702975,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "priority_vs_prob@180": {
          "high_risk_handled": {
            "mean_diff": 8.0,
            "min_diff": 6.0,
            "max_diff": 11.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 2.4125,
            "min_diff": 1.28125,
            "max_diff": 2.9375,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -8.0,
            "min_diff": -11.0,
            "max_diff": -6.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.27681667839240415,
            "min_diff": -1.6935686371276018,
            "max_diff": 0.21185157287073442,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -0.36142832774506173,
            "min_diff": -1.0837004826123824,
            "max_diff": 0.11061697759138056,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 0
          }
        }
      },
      "harm_per_reviewer_hour": {
        "router": 71.04375,
        "fifo": 55.75625,
        "load_per_hour": 180.0
      },
      "high_risk_wait_p90": {
        "router": 1.0002904639523633,
        "fifo": 5.945219363009528,
        "load_per_hour": 108.0
      },
      "high_risk_handled": {
        "primary": {
          "router": 122.8,
          "fifo": 122.6,
          "load_per_hour": 108.0
        },
        "thesis": {
          "router": 184.6,
          "fifo": 134.0,
          "load_per_hour": 180.0
        }
      },
      "high_risk_arrived": {
        "primary": {
          "router": 123.0,
          "fifo": 123.0,
          "load_per_hour": 108.0
        },
        "thesis": {
          "router": 200.8,
          "fifo": 200.8,
          "load_per_hour": 180.0
        }
      },
      "headline": {
        "metric": "wait",
        "percentile": 50,
        "load_per_hour": 108.0,
        "router_strategy": "severity",
        "population": "jobs that started, including in-progress reviews; pooled over seeds; populations may differ by strategy; read with completion and unfinished counts",
        "selected": {
          "router": 0.40263116281190037,
          "fifo": 1.4173549491476933,
          "n_router": 4356,
          "n_fifo": 4356,
          "absolute_difference_min": 1.014723786335793,
          "reduction": 0.715927782907156
        },
        "supplementary_high_risk": {
          "note": "reported alongside, never a substitute for the selected metric",
          "wait_p90": {
            "router": 1.004987364263349,
            "fifo": 6.3037574850354305,
            "n_router": 615,
            "n_fifo": 615,
            "absolute_difference_min": 5.298770120772081,
            "reduction": 0.8405732824193981,
            "percentile": 90
          }
        }
      },
      "time_metrics": {
        "fifo@60": {
          "pooled_over_seeds": {
            "n_arrivals": 2349,
            "status": {
              "not_started": 2,
              "in_progress": 8,
              "completed": 2339
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 0,
              "completed": 331
            },
            "completion_ratio": 0.9957428693060877,
            "wait": {
              "n": 2347,
              "p50": 0.0,
              "p90": 0.34783024911243443,
              "p99": 1.4696809585511497,
              "p99_reliable": true,
              "mean": 0.09510419526541297,
              "max": 2.7211235993694913
            },
            "high_risk_wait": {
              "n": 331,
              "p50": 0.0,
              "p90": 0.352413698279463,
              "p99": 1.246566234173218,
              "p99_reliable": true,
              "mean": 0.10177040705944515,
              "max": 2.7211235993694913
            },
            "completion_latency": {
              "n": 2339,
              "p50": 2.0,
              "p90": 2.32556965563164,
              "p99": 3.4713534645370827,
              "p99_reliable": true,
              "mean": 2.0941182837627355,
              "max": 4.721123599369491
            },
            "high_risk_completion_latency": {
              "n": 331,
              "p50": 2.0,
              "p90": 2.352413698279463,
              "p99": 3.246566234173218,
              "p99_reliable": true,
              "mean": 2.101770407059445,
              "max": 4.721123599369491
            },
            "unfinished_age_at_horizon": {
              "n": 10,
              "p50": 1.2026948818665346,
              "p90": 2.129641730156578,
              "p99": 2.290669844457139,
              "p99_reliable": false,
              "mean": 1.3159431660228904,
              "max": 2.3085618571572013
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 0,
              "p50": null,
              "p90": null,
              "p99": null,
              "p99_reliable": false,
              "mean": null,
              "max": null
            }
          },
          "n_seeds": 5
        },
        "prob@60": {
          "pooled_over_seeds": {
            "n_arrivals": 2349,
            "status": {
              "not_started": 2,
              "in_progress": 8,
              "completed": 2339
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 0,
              "completed": 331
            },
            "completion_ratio": 0.9957428693060877,
            "wait": {
              "n": 2347,
              "p50": 0.0,
              "p90": 0.2701938137892659,
              "p99": 1.555190534193808,
              "p99_reliable": true,
              "mean": 0.09492957405032418,
              "max": 5.948259947308685
            },
            "high_risk_wait": {
              "n": 331,
              "p50": 0.0,
              "p90": 0.2447700913483004,
              "p99": 1.0465783765060195,
              "p99_reliable": true,
              "mean": 0.07138739026582423,
              "max": 1.8704167656832738
            },
            "completion_latency": {
              "n": 2339,
              "p50": 2.0,
              "p90": 2.2618503485649004,
              "p99": 3.557272327414809,
              "p99_reliable": true,
              "mean": 2.0941182837627355,
              "max": 7.948259947308685
            },
            "high_risk_completion_latency": {
              "n": 331,
              "p50": 2.0,
              "p90": 2.2447700913483004,
              "p99": 3.0465783765060195,
              "p99_reliable": true,
              "mean": 2.071387390265824,
              "max": 3.8704167656832738
            },
            "unfinished_age_at_horizon": {
              "n": 10,
              "p50": 1.2026948818665346,
              "p90": 2.129641730156578,
              "p99": 2.290669844457139,
              "p99_reliable": false,
              "mean": 1.3159431660228904,
              "max": 2.3085618571572013
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 0,
              "p50": null,
              "p90": null,
              "p99": null,
              "p99_reliable": false,
              "mean": null,
              "max": null
            }
          },
          "n_seeds": 5
        },
        "severity@60": {
          "pooled_over_seeds": {
            "n_arrivals": 2349,
            "status": {
              "not_started": 2,
              "in_progress": 8,
              "completed": 2339
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 0,
              "completed": 331
            },
            "completion_ratio": 0.9957428693060877,
            "wait": {
              "n": 2347,
              "p50": 0.0,
              "p90": 0.2654712766065019,
              "p99": 1.4958083384842575,
              "p99_reliable": true,
              "mean": 0.09492957405032418,
              "max": 5.948259947308685
            },
            "high_risk_wait": {
              "n": 331,
              "p50": 0.0,
              "p90": 0.21612389138317667,
              "p99": 0.854934620218887,
              "p99_reliable": true,
              "mean": 0.06448004713043425,
              "max": 1.8704167656832738
            },
            "completion_latency": {
              "n": 2339,
              "p50": 2.0,
              "p90": 2.2609765035245917,
              "p99": 3.4963772071508727,
              "p99_reliable": true,
              "mean": 2.094118283762736,
              "max": 7.948259947308685
            },
            "high_risk_completion_latency": {
              "n": 331,
              "p50": 2.0,
              "p90": 2.2161238913831767,
              "p99": 2.854934620218887,
              "p99_reliable": true,
              "mean": 2.0644800471304343,
              "max": 3.8704167656832738
            },
            "unfinished_age_at_horizon": {
              "n": 10,
              "p50": 1.2026948818665346,
              "p90": 2.129641730156578,
              "p99": 2.290669844457139,
              "p99_reliable": false,
              "mean": 1.3159431660228904,
              "max": 2.3085618571572013
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 0,
              "p50": null,
              "p90": null,
              "p99": null,
              "p99_reliable": false,
              "mean": null,
              "max": null
            }
          },
          "n_seeds": 5
        },
        "priority@60": {
          "pooled_over_seeds": {
            "n_arrivals": 2349,
            "status": {
              "not_started": 2,
              "in_progress": 8,
              "completed": 2339
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 0,
              "completed": 331
            },
            "completion_ratio": 0.9957428693060877,
            "wait": {
              "n": 2347,
              "p50": 0.0,
              "p90": 0.26456161055531907,
              "p99": 1.4958083384842575,
              "p99_reliable": true,
              "mean": 0.09492957405032418,
              "max": 5.948259947308685
            },
            "high_risk_wait": {
              "n": 331,
              "p50": 0.0,
              "p90": 0.23187070419373867,
              "p99": 0.864232406239563,
              "p99_reliable": true,
              "mean": 0.06695976648680152,
              "max": 1.8704167656832738
            },
            "completion_latency": {
              "n": 2339,
              "p50": 2.0,
              "p90": 2.2582911166074275,
              "p99": 3.4963772071508727,
              "p99_reliable": true,
              "mean": 2.094118283762736,
              "max": 7.948259947308685
            },
            "high_risk_completion_latency": {
              "n": 331,
              "p50": 2.0,
              "p90": 2.2318707041937387,
              "p99": 2.8642324062395628,
              "p99_reliable": true,
              "mean": 2.066959766486802,
              "max": 3.8704167656832738
            },
            "unfinished_age_at_horizon": {
              "n": 10,
              "p50": 1.2026948818665346,
              "p90": 2.129641730156578,
              "p99": 2.290669844457139,
              "p99_reliable": false,
              "mean": 1.3159431660228904,
              "max": 2.3085618571572013
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 0,
              "p50": null,
              "p90": null,
              "p99": null,
              "p99_reliable": false,
              "mean": null,
              "max": null
            }
          },
          "n_seeds": 5
        },
        "fifo@108": {
          "pooled_over_seeds": {
            "n_arrivals": 4362,
            "status": {
              "not_started": 6,
              "in_progress": 19,
              "completed": 4337
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 2,
              "completed": 613
            },
            "completion_ratio": 0.994268684089867,
            "wait": {
              "n": 4356,
              "p50": 1.4173549491476933,
              "p90": 6.442540900804133,
              "p99": 13.525724968102882,
              "p99_reliable": true,
              "mean": 2.3735803547492322,
              "max": 15.83196024181342
            },
            "high_risk_wait": {
              "n": 615,
              "p50": 1.4285397172496026,
              "p90": 6.3037574850354305,
              "p99": 12.517014533077308,
              "p99_reliable": true,
              "mean": 2.3850647184951126,
              "max": 15.246971486806729
            },
            "completion_latency": {
              "n": 4337,
              "p50": 3.419545378048099,
              "p90": 8.464705547151432,
              "p99": 15.528315051464148,
              "p99_reliable": true,
              "mean": 4.380993763868377,
              "max": 17.83196024181342
            },
            "high_risk_completion_latency": {
              "n": 613,
              "p50": 3.4285397172496026,
              "p90": 8.30710909708197,
              "p99": 14.5180881629283,
              "p99_reliable": true,
              "mean": 4.389139491739891,
              "max": 17.24697148680673
            },
            "unfinished_age_at_horizon": {
              "n": 25,
              "p50": 1.2369406609570888,
              "p90": 3.1454303229097613,
              "p99": 3.490061138175592,
              "p99_reliable": false,
              "mean": 1.4553530266725647,
              "max": 3.5107849108978826
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 2,
              "p50": 2.267594529659334,
              "p90": 3.1930675922425396,
              "p99": 3.4012990313237608,
              "p99_reliable": false,
              "mean": 2.267594529659334,
              "max": 3.4244358578883407
            }
          },
          "n_seeds": 5
        },
        "prob@108": {
          "pooled_over_seeds": {
            "n_arrivals": 4362,
            "status": {
              "not_started": 6,
              "in_progress": 19,
              "completed": 4337
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 1,
              "completed": 614
            },
            "completion_ratio": 0.994268684089867,
            "wait": {
              "n": 4356,
              "p50": 0.4001767555209135,
              "p90": 3.7493883195952833,
              "p99": 42.42437574667875,
              "p99_reliable": true,
              "mean": 2.370670036171434,
              "max": 130.109811670253
            },
            "high_risk_wait": {
              "n": 615,
              "p50": 0.2597253806104902,
              "p90": 1.2180020917129128,
              "p99": 12.537342998457584,
              "p99_reliable": true,
              "mean": 0.9201818620713773,
              "max": 127.57457078656498
            },
            "completion_latency": {
              "n": 4337,
              "p50": 2.401141488207294,
              "p90": 5.750691070990138,
              "p99": 44.65855800883954,
              "p99_reliable": true,
              "mean": 4.379819421874729,
              "max": 132.109811670253
            },
            "high_risk_completion_latency": {
              "n": 614,
              "p50": 2.2611943423641065,
              "p90": 3.218693665268933,
              "p99": 14.585431531668824,
              "p99_reliable": true,
              "mean": 2.921680529599181,
              "max": 129.57457078656498
            },
            "unfinished_age_at_horizon": {
              "n": 25,
              "p50": 1.2369406609570888,
              "p90": 3.5169726877114273,
              "p99": 5.756618262206671,
              "p99_reliable": false,
              "mean": 1.6590778757305134,
              "max": 6.321862843395934
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 1,
              "p50": 1.1107532014303274,
              "p90": 1.1107532014303274,
              "p99": 1.1107532014303274,
              "p99_reliable": false,
              "mean": 1.1107532014303274,
              "max": 1.1107532014303274
            }
          },
          "n_seeds": 5
        },
        "severity@108": {
          "pooled_over_seeds": {
            "n_arrivals": 4362,
            "status": {
              "not_started": 6,
              "in_progress": 19,
              "completed": 4337
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 1,
              "completed": 614
            },
            "completion_ratio": 0.994268684089867,
            "wait": {
              "n": 4356,
              "p50": 0.40263116281190037,
              "p90": 3.799464659717513,
              "p99": 38.223281247964835,
              "p99_reliable": true,
              "mean": 2.371561246671204,
              "max": 151.72859540760882
            },
            "high_risk_wait": {
              "n": 615,
              "p50": 0.21911251923256714,
              "p90": 1.004987364263349,
              "p99": 6.215561624566652,
              "p99_reliable": true,
              "mean": 0.8207229445392829,
              "max": 151.72859540760882
            },
            "completion_latency": {
              "n": 4337,
              "p50": 2.4059263867594325,
              "p90": 5.8060948259924405,
              "p99": 40.35137543534614,
              "p99_reliable": true,
              "mean": 4.379834580315732,
              "max": 153.72859540760882
            },
            "high_risk_completion_latency": {
              "n": 614,
              "p50": 2.2206769730060643,
              "p90": 3.0066637784653287,
              "p99": 8.216948002325731,
              "p99_reliable": true,
              "mean": 2.822059626859379,
              "max": 153.72859540760882
            },
            "unfinished_age_at_horizon": {
              "n": 25,
              "p50": 1.2369406609570888,
              "p90": 3.5169726877114273,
              "p99": 5.925801977370706,
              "p99_reliable": false,
              "mean": 1.6564481893851757,
              "max": 6.54447299492756
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 1,
              "p50": 1.1107532014303274,
              "p90": 1.1107532014303274,
              "p99": 1.1107532014303274,
              "p99_reliable": false,
              "mean": 1.1107532014303274,
              "max": 1.1107532014303274
            }
          },
          "n_seeds": 5
        },
        "priority@108": {
          "pooled_over_seeds": {
            "n_arrivals": 4362,
            "status": {
              "not_started": 6,
              "in_progress": 19,
              "completed": 4337
            },
            "high_risk_status": {
              "not_started": 0,
              "in_progress": 1,
              "completed": 614
            },
            "completion_ratio": 0.994268684089867,
            "wait": {
              "n": 4356,
              "p50": 0.4006220267968956,
              "p90": 3.7609328693494604,
              "p99": 38.223281247964835,
              "p99_reliable": true,
              "mean": 2.371561246671204,
              "max": 151.72859540760882
            },
            "high_risk_wait": {
              "n": 615,
              "p50": 0.21911251923256714,
              "p90": 1.0559692033328703,
              "p99": 6.656447067986001,
              "p99_reliable": true,
              "mean": 0.8306614572521003,
              "max": 151.72859540760882
            },
            "completion_latency": {
              "n": 4337,
              "p50": 2.4020694531041045,
              "p90": 5.773231823573942,
              "p99": 40.35137543534614,
              "p99_reliable": true,
              "mean": 4.379768093732803,
              "max": 153.72859540760882
            },
            "high_risk_completion_latency": {
              "n": 614,
              "p50": 2.2206769730060643,
              "p90": 3.0567146701470493,
              "p99": 8.657252944408809,
              "p99_reliable": true,
              "mean": 2.8320143260749866,
              "max": 153.72859540760882
            },
            "unfinished_age_at_horizon": {
              "n": 25,
              "p50": 1.2369406609570888,
              "p90": 3.5169726877114273,
              "p99": 5.925801977370706,
              "p99_reliable": false,
              "mean": 1.6679822817917784,
              "max": 6.54447299492756
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 1,
              "p50": 1.1107532014303274,
              "p90": 1.1107532014303274,
              "p99": 1.1107532014303274,
              "p99_reliable": false,
              "mean": 1.1107532014303274,
              "max": 1.1107532014303274
            }
          },
          "n_seeds": 5
        },
        "fifo@180": {
          "pooled_over_seeds": {
            "n_arrivals": 7110,
            "status": {
              "not_started": 2324,
              "in_progress": 20,
              "completed": 4766
            },
            "high_risk_status": {
              "not_started": 332,
              "in_progress": 2,
              "completed": 670
            },
            "completion_ratio": 0.6703234880450071,
            "wait": {
              "n": 4786,
              "p50": 78.87252652584435,
              "p90": 140.27151586072847,
              "p99": 165.95016891212,
              "p99_reliable": true,
              "mean": 77.53299363257301,
              "max": 169.74183453725038
            },
            "high_risk_wait": {
              "n": 672,
              "p50": 81.90977293487249,
              "p90": 139.01082894369523,
              "p99": 162.04814983157593,
              "p99_reliable": true,
              "mean": 78.35625316265086,
              "max": 168.84924794814322
            },
            "completion_latency": {
              "n": 4766,
              "p50": 80.39186054627456,
              "p90": 141.59044675441356,
              "p99": 167.25062344348652,
              "p99_reliable": true,
              "mean": 79.2033089175938,
              "max": 171.74183453725038
            },
            "high_risk_completion_latency": {
              "n": 670,
              "p50": 83.82312229038843,
              "p90": 140.75278533314767,
              "p99": 164.0707940875022,
              "p99_reliable": true,
              "mean": 80.11811849787382,
              "max": 170.84924794814322
            },
            "unfinished_age_at_horizon": {
              "n": 2344,
              "p50": 79.36956353967992,
              "p90": 141.79487625365437,
              "p99": 163.08940773329135,
              "p99_reliable": true,
              "mean": 79.31850882731416,
              "max": 171.12460717667818
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 334,
              "p50": 78.27313326101313,
              "p90": 139.11402592204718,
              "p99": 159.4435789378222,
              "p99_reliable": true,
              "mean": 79.246369564342,
              "max": 169.6780291450429
            }
          },
          "n_seeds": 5
        },
        "prob@180": {
          "pooled_over_seeds": {
            "n_arrivals": 7110,
            "status": {
              "not_started": 2324,
              "in_progress": 20,
              "completed": 4766
            },
            "high_risk_status": {
              "not_started": 119,
              "in_progress": 4,
              "completed": 881
            },
            "completion_ratio": 0.6703234880450071,
            "wait": {
              "n": 4786,
              "p50": 0.7057169348460519,
              "p90": 11.211968277147989,
              "p99": 118.12467568819235,
              "p99_reliable": true,
              "mean": 6.131936524310395,
              "max": 370.1124701865657
            },
            "high_risk_wait": {
              "n": 885,
              "p50": 0.4458682263217497,
              "p90": 2.2111111040846825,
              "p99": 33.0709975457287,
              "p99_reliable": true,
              "mean": 2.1210098253106318,
              "max": 162.05583476731
            },
            "completion_latency": {
              "n": 4766,
              "p50": 2.705716934846052,
              "p90": 13.146352056186217,
              "p99": 119.75021361222366,
              "p99_reliable": true,
              "mean": 8.087156214225685,
              "max": 372.1124701865657
            },
            "high_risk_completion_latency": {
              "n": 881,
              "p50": 2.442625719803317,
              "p90": 4.081525049487951,
              "p99": 35.31487793865782,
              "p99_reliable": true,
              "mean": 4.100500227324477,
              "max": 164.05583476731
            },
            "unfinished_age_at_horizon": {
              "n": 2344,
              "p50": 219.8356750220175,
              "p90": 413.9640050211456,
              "p99": 464.45302028687905,
              "p99_reliable": true,
              "mean": 223.91730737008396,
              "max": 476.57265471311126
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 123,
              "p50": 226.60840577978314,
              "p90": 412.08408806809933,
              "p99": 459.960581016927,
              "p99_reliable": true,
              "mean": 224.94700365245797,
              "max": 468.7510840044234
            }
          },
          "n_seeds": 5
        },
        "severity@180": {
          "pooled_over_seeds": {
            "n_arrivals": 7110,
            "status": {
              "not_started": 2324,
              "in_progress": 20,
              "completed": 4766
            },
            "high_risk_status": {
              "not_started": 79,
              "in_progress": 2,
              "completed": 923
            },
            "completion_ratio": 0.6703234880450071,
            "wait": {
              "n": 4786,
              "p50": 0.6897688139825675,
              "p90": 11.055070945116274,
              "p99": 102.44741537421622,
              "p99_reliable": true,
              "mean": 5.774469405219291,
              "max": 352.69702303068385
            },
            "high_risk_wait": {
              "n": 925,
              "p50": 0.41307187500299847,
              "p90": 1.6859068686938643,
              "p99": 22.419571127080346,
              "p99_reliable": true,
              "mean": 1.6969698416911332,
              "max": 263.2009515463099
            },
            "completion_latency": {
              "n": 4766,
              "p50": 2.6897688139825675,
              "p90": 13.025723395052271,
              "p99": 104.10814879544971,
              "p99_reliable": true,
              "mean": 7.718933840858238,
              "max": 354.69702303068385
            },
            "high_risk_completion_latency": {
              "n": 923,
              "p50": 2.4130718750029985,
              "p90": 3.6867558661569833,
              "p99": 24.435142291523775,
              "p99_reliable": true,
              "mean": 3.6991465660152794,
              "max": 265.2009515463099
            },
            "unfinished_age_at_horizon": {
              "n": 2344,
              "p50": 219.63027894636744,
              "p90": 414.24911101958276,
              "p99": 465.81463982341853,
              "p99_reliable": true,
              "mean": 224.66600525040366,
              "max": 477.955424527892
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 81,
              "p50": 241.71355506726283,
              "p90": 410.30295099976024,
              "p99": 464.50397086795476,
              "p99_reliable": false,
              "mean": 212.81279531990276,
              "max": 468.0811769981438
            }
          },
          "n_seeds": 5
        },
        "priority@180": {
          "pooled_over_seeds": {
            "n_arrivals": 7110,
            "status": {
              "not_started": 2324,
              "in_progress": 20,
              "completed": 4766
            },
            "high_risk_status": {
              "not_started": 81,
              "in_progress": 2,
              "completed": 921
            },
            "completion_ratio": 0.6703234880450071,
            "wait": {
              "n": 4786,
              "p50": 0.6962996880905177,
              "p90": 11.313066640295546,
              "p99": 100.32266039283667,
              "p99_reliable": true,
              "mean": 5.837572599873409,
              "max": 352.69702303068385
            },
            "high_risk_wait": {
              "n": 923,
              "p50": 0.42526902679139766,
              "p90": 1.905015364175725,
              "p99": 23.452719453767,
              "p99_reliable": true,
              "mean": 2.060260173997562,
              "max": 263.2009515463099
            },
            "completion_latency": {
              "n": 4766,
              "p50": 2.6953075451791193,
              "p90": 13.200831578051464,
              "p99": 102.36130907488314,
              "p99_reliable": true,
              "mean": 7.815035853098288,
              "max": 354.69702303068385
            },
            "high_risk_completion_latency": {
              "n": 921,
              "p50": 2.4252690267913977,
              "p90": 3.905957411501163,
              "p99": 25.460591811362626,
              "p99_reliable": true,
              "mean": 4.063230529280732,
              "max": 265.2009515463099
            },
            "unfinished_age_at_horizon": {
              "n": 2344,
              "p50": 219.8356750220175,
              "p90": 414.32942171648847,
              "p99": 465.81463982341853,
              "p99_reliable": true,
              "mean": 224.47060329206911,
              "max": 477.955424527892
            },
            "high_risk_unfinished_age_at_horizon": {
              "n": 83,
              "p50": 239.92897133917913,
              "p90": 409.1366131604837,
              "p99": 464.41454071469997,
              "p99_reliable": false,
              "mean": 209.34217184843752,
              "max": 468.0811769981438
            }
          },
          "n_seeds": 5
        }
      },
      "completion_ratio": 0.994271372623818,
      "backlog_end": 1.2,
      "high_risk_unfinished": 0.2,
      "queue_depth_p95": 14.219999999999981,
      "reviewer_utilization": 0.9054738607875965,
      "primary": {
        "load_per_hour": 108.0,
        "strategy": "severity",
        "note": "queue-clearing gates read this load; the router is the configured primary ordering (strategy), compared against fifo at the same load for time-to-action"
      },
      "thesis": {
        "load_per_hour": 180.0,
        "strategy": "severity",
        "note": "harm handled per reviewer-hour only differs between orderings when the queue does not clear, so the FIFO comparison and the paired comparison of the primary ordering against every alternative read this load"
      }
    },
    "identity_false_positives": {
      "threshold_selection": {
        "definition": "identity term = any whole-word match of review_router.signals.IDENTITY_TERMS; predicted positive = routed to priority_review or human_review; false positive = no ground-truth label is positive, for both review tiers; FDR = false positives / predicted positives; FDR ratio = with-term FDR / without-term FDR with a 95% log-ratio interval. clean_negative_false_positives separately reports FPR = clean comments routed to the tier / all clean comments in that subgroup. allow_false_omission = truly positive allowed comments / all allowed comments.",
        "identity_terms": [
          "african",
          "africans",
          "arab",
          "arabs",
          "asian",
          "asians",
          "atheist",
          "atheists",
          "autistic",
          "bisexual",
          "black",
          "blacks",
          "buddhist",
          "buddhists",
          "catholic",
          "catholics",
          "chinese",
          "christian",
          "christians",
          "disability",
          "disabled",
          "elderly",
          "female",
          "females",
          "feminist",
          "feminists",
          "gay",
          "gays",
          "heterosexual",
          "hindu",
          "hindus",
          "hispanic",
          "homosexual",
          "homosexuals",
          "immigrant",
          "immigrants",
          "indian",
          "indians",
          "islam",
          "islamic",
          "jew",
          "jewish",
          "jews",
          "latina",
          "latino",
          "lesbian",
          "lesbians",
          "lgbt",
          "male",
          "males",
          "mexican",
          "mexicans",
          "muslim",
          "muslims",
          "queer",
          "refugee",
          "refugees",
          "trans",
          "transgender",
          "white",
          "whites",
          "woman",
          "women"
        ],
        "population_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 13,
              "false_discovery_rate": 0.06190476190476191,
              "false_discovery_rate_ci95": [
                0.03653085811360575,
                0.10301859612885285
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1870,
              "n_false_positive": 122,
              "false_discovery_rate": 0.06524064171122995,
              "false_discovery_rate_ci95": [
                0.054915192884036464,
                0.07734864283516042
              ]
            },
            "false_discovery_rate_ratio": 0.9488680718188914,
            "false_discovery_rate_ratio_ci95": [
              0.5454010379857869,
              1.6508047381838027
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 109,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0,
              "false_discovery_rate_ci95": [
                0.0,
                0.03404297427009551
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 931,
              "n_false_positive": 3,
              "false_discovery_rate": 0.00322234156820623,
              "false_discovery_rate_ci95": [
                0.0010964789997296434,
                0.009430930528162986
              ]
            },
            "false_discovery_rate_ratio": 0.0,
            "false_discovery_rate_ratio_ci95": [
              0.06293522794081403,
              23.2785842980799
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 101,
              "n_false_positive": 13,
              "false_discovery_rate": 0.12871287128712872,
              "false_discovery_rate_ci95": [
                0.07678714412405785,
                0.20784699965567732
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 939,
              "n_false_positive": 119,
              "false_discovery_rate": 0.12673056443024494,
              "false_discovery_rate_ci95": [
                0.10696242545626841,
                0.14954035833969095
              ]
            },
            "false_discovery_rate_ratio": 1.0156419003244863,
            "false_discovery_rate_ratio_ci95": [
              0.5951486396975256,
              1.7332283078375024
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 122,
                "false_positive_rate": 0.004514672686230248,
                "false_positive_rate_ci95": [
                  0.0037827546821183845,
                  0.005387442259257935
                ]
              },
              "false_positive_rate_ratio": 1.7109328579916816,
              "false_positive_rate_ratio_ci95": [
                0.9678651372274237,
                3.024482577129694
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 3,
                "false_positive_rate": 0.00011101654146467824,
                "false_positive_rate_ci95": [
                  3.775634423773888e-05,
                  0.000326380134539066
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.11846534399266408,
                44.36369289464826
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 119,
                "false_positive_rate": 0.00440365614476557,
                "false_positive_rate_ci95": [
                  0.0036815580111,
                  0.005266637406009105
                ]
              },
              "false_positive_rate_ratio": 1.7540656191175223,
              "false_positive_rate_ratio_ci95": [
                0.9915744698525628,
                3.1028896867701907
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28009,
              "n_truly_positive": 1108,
              "false_omission_rate": 0.03955871327073441,
              "false_omission_rate_ci95": [
                0.03733840142090709,
                0.041905307652294034
              ]
            }
          }
        },
        "model_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 13,
              "false_discovery_rate": 0.06190476190476191,
              "false_discovery_rate_ci95": [
                0.03653085811360575,
                0.10301859612885285
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1870,
              "n_false_positive": 122,
              "false_discovery_rate": 0.06524064171122995,
              "false_discovery_rate_ci95": [
                0.054915192884036464,
                0.07734864283516042
              ]
            },
            "false_discovery_rate_ratio": 0.9488680718188914,
            "false_discovery_rate_ratio_ci95": [
              0.5454010379857869,
              1.6508047381838027
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 109,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0,
              "false_discovery_rate_ci95": [
                0.0,
                0.03404297427009551
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 931,
              "n_false_positive": 3,
              "false_discovery_rate": 0.00322234156820623,
              "false_discovery_rate_ci95": [
                0.0010964789997296434,
                0.009430930528162986
              ]
            },
            "false_discovery_rate_ratio": 0.0,
            "false_discovery_rate_ratio_ci95": [
              0.06293522794081403,
              23.2785842980799
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 101,
              "n_false_positive": 13,
              "false_discovery_rate": 0.12871287128712872,
              "false_discovery_rate_ci95": [
                0.07678714412405785,
                0.20784699965567732
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 939,
              "n_false_positive": 119,
              "false_discovery_rate": 0.12673056443024494,
              "false_discovery_rate_ci95": [
                0.10696242545626841,
                0.14954035833969095
              ]
            },
            "false_discovery_rate_ratio": 1.0156419003244863,
            "false_discovery_rate_ratio_ci95": [
              0.5951486396975256,
              1.7332283078375024
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 122,
                "false_positive_rate": 0.004514672686230248,
                "false_positive_rate_ci95": [
                  0.0037827546821183845,
                  0.005387442259257935
                ]
              },
              "false_positive_rate_ratio": 1.7109328579916816,
              "false_positive_rate_ratio_ci95": [
                0.9678651372274237,
                3.024482577129694
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 3,
                "false_positive_rate": 0.00011101654146467824,
                "false_positive_rate_ci95": [
                  3.775634423773888e-05,
                  0.000326380134539066
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.11846534399266408,
                44.36369289464826
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 119,
                "false_positive_rate": 0.00440365614476557,
                "false_positive_rate_ci95": [
                  0.0036815580111,
                  0.005266637406009105
                ]
              },
              "false_positive_rate_ratio": 1.7540656191175223,
              "false_positive_rate_ratio_ci95": [
                0.9915744698525628,
                3.1028896867701907
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28009,
              "n_truly_positive": 1108,
              "false_omission_rate": 0.03955871327073441,
              "false_omission_rate_ci95": [
                0.03733840142090709,
                0.041905307652294034
              ]
            }
          }
        },
        "final_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 13,
              "false_discovery_rate": 0.06190476190476191,
              "false_discovery_rate_ci95": [
                0.03653085811360575,
                0.10301859612885285
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1873,
              "n_false_positive": 124,
              "false_discovery_rate": 0.06620395088093967,
              "false_discovery_rate_ci95": [
                0.055808145817448404,
                0.07837551561520546
              ]
            },
            "false_discovery_rate_ratio": 0.9350614439324118,
            "false_discovery_rate_ratio_ci95": [
              0.537710132763461,
              1.626043198099057
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 109,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0,
              "false_discovery_rate_ci95": [
                0.0,
                0.03404297427009551
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 951,
              "n_false_positive": 5,
              "false_discovery_rate": 0.005257623554153523,
              "false_discovery_rate_ci95": [
                0.002247770999173177,
                0.012248309740254312
              ]
            },
            "false_discovery_rate_ratio": 0.0,
            "false_discovery_rate_ratio_ci95": [
              0.043799486521904823,
              14.132992777018156
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 101,
              "n_false_positive": 13,
              "false_discovery_rate": 0.12871287128712872,
              "false_discovery_rate_ci95": [
                0.07678714412405785,
                0.20784699965567732
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 922,
              "n_false_positive": 119,
              "false_discovery_rate": 0.12906724511930587,
              "false_discovery_rate_ci95": [
                0.10895519355564597,
                0.15225741100141893
              ]
            },
            "false_discovery_rate_ratio": 0.9972543472834677,
            "false_discovery_rate_ratio_ci95": [
              0.5844150921666221,
              1.7017292101214754
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 124,
                "false_positive_rate": 0.004588683713873367,
                "false_positive_rate_ci95": [
                  0.003850286352596687,
                  0.005467911604957776
                ]
              },
              "false_positive_rate_ratio": 1.6833371667337511,
              "false_positive_rate_ratio_ci95": [
                0.9526791141311439,
                2.974374030957439
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 5,
                "false_positive_rate": 0.00018502756910779706,
                "false_positive_rate_ci95": [
                  7.903522487927967e-05,
                  0.0004331022700756684
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.08070280330706606,
                26.37188049729835
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 119,
                "false_positive_rate": 0.00440365614476557,
                "false_positive_rate_ci95": [
                  0.0036815580111,
                  0.005266637406009105
                ]
              },
              "false_positive_rate_ratio": 1.7540656191175223,
              "false_positive_rate_ratio_ci95": [
                0.9915744698525628,
                3.1028896867701907
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28006,
              "n_truly_positive": 1107,
              "false_omission_rate": 0.03952724416196529,
              "false_omission_rate_ci95": [
                0.037307691311525754,
                0.041873101701426774
              ]
            }
          },
          "by_term": [
            {
              "term": "gay",
              "n_predicted_positive": 90,
              "n_false_positive": 6,
              "false_discovery_rate": 0.06666666666666667
            },
            {
              "term": "black",
              "n_predicted_positive": 16,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "jew",
              "n_predicted_positive": 16,
              "n_false_positive": 1,
              "false_discovery_rate": 0.0625
            },
            {
              "term": "white",
              "n_predicted_positive": 16,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "homosexual",
              "n_predicted_positive": 13,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "jews",
              "n_predicted_positive": 13,
              "n_false_positive": 3,
              "false_discovery_rate": 0.23076923076923078
            },
            {
              "term": "women",
              "n_predicted_positive": 10,
              "n_false_positive": 1,
              "false_discovery_rate": 0.1
            },
            {
              "term": "indian",
              "n_predicted_positive": 9,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "muslim",
              "n_predicted_positive": 7,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "woman",
              "n_predicted_positive": 6,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "arab",
              "n_predicted_positive": 5,
              "n_false_positive": 1,
              "false_discovery_rate": 0.2
            },
            {
              "term": "asian",
              "n_predicted_positive": 5,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            }
          ]
        }
      },
      "test": {
        "definition": "identity term = any whole-word match of review_router.signals.IDENTITY_TERMS; predicted positive = routed to priority_review or human_review; false positive = no ground-truth label is positive, for both review tiers; FDR = false positives / predicted positives; FDR ratio = with-term FDR / without-term FDR with a 95% log-ratio interval. clean_negative_false_positives separately reports FPR = clean comments routed to the tier / all clean comments in that subgroup. allow_false_omission = truly positive allowed comments / all allowed comments.",
        "identity_terms": [
          "african",
          "africans",
          "arab",
          "arabs",
          "asian",
          "asians",
          "atheist",
          "atheists",
          "autistic",
          "bisexual",
          "black",
          "blacks",
          "buddhist",
          "buddhists",
          "catholic",
          "catholics",
          "chinese",
          "christian",
          "christians",
          "disability",
          "disabled",
          "elderly",
          "female",
          "females",
          "feminist",
          "feminists",
          "gay",
          "gays",
          "heterosexual",
          "hindu",
          "hindus",
          "hispanic",
          "homosexual",
          "homosexuals",
          "immigrant",
          "immigrants",
          "indian",
          "indians",
          "islam",
          "islamic",
          "jew",
          "jewish",
          "jews",
          "latina",
          "latino",
          "lesbian",
          "lesbians",
          "lgbt",
          "male",
          "males",
          "mexican",
          "mexicans",
          "muslim",
          "muslims",
          "queer",
          "refugee",
          "refugees",
          "trans",
          "transgender",
          "white",
          "whites",
          "woman",
          "women"
        ],
        "population_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 809,
              "n_false_positive": 271,
              "false_discovery_rate": 0.33498145859085293,
              "false_discovery_rate_ci95": [
                0.303305117353319,
                0.3682175429604537
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5485,
              "n_false_positive": 1795,
              "false_discovery_rate": 0.3272561531449407,
              "false_discovery_rate_ci95": [
                0.31496346860304697,
                0.3397906331147628
              ]
            },
            "false_discovery_rate_ratio": 1.0236062954712137,
            "false_discovery_rate_ratio_ci95": [
              0.9222764145031342,
              1.136069221387143
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 362,
              "n_false_positive": 44,
              "false_discovery_rate": 0.12154696132596685,
              "false_discovery_rate_ci95": [
                0.09180219473987582,
                0.15923949736942566
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 2362,
              "n_false_positive": 265,
              "false_discovery_rate": 0.1121930567315834,
              "false_discovery_rate_ci95": [
                0.10008978941691218,
                0.12555570211730352
              ]
            },
            "false_discovery_rate_ratio": 1.0833732930261648,
            "false_discovery_rate_ratio_ci95": [
              0.8031670319613425,
              1.4613369888654093
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 447,
              "n_false_positive": 227,
              "false_discovery_rate": 0.5078299776286354,
              "false_discovery_rate_ci95": [
                0.4616152094413018,
                0.5539113129361424
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3123,
              "n_false_positive": 1530,
              "false_discovery_rate": 0.4899135446685879,
              "false_discovery_rate_ci95": [
                0.47240421004392447,
                0.5074476625794673
              ]
            },
            "false_discovery_rate_ratio": 1.0365706013949205,
            "false_discovery_rate_ratio_ci95": [
              0.9397789344284521,
              1.1433312370740634
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 271,
                "false_positive_rate": 0.06539575289575289,
                "false_positive_rate_ci95": [
                  0.05826389643505257,
                  0.07333261328991493
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1795,
                "false_positive_rate": 0.03349443003489392,
                "false_positive_rate_ci95": [
                  0.03200423665820344,
                  0.035051497831440334
                ]
              },
              "false_positive_rate_ratio": 1.9524366537249542,
              "false_positive_rate_ratio_ci95": [
                1.7251571551714946,
                2.2096589144829255
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 44,
                "false_positive_rate": 0.010617760617760617,
                "false_positive_rate_ci95": [
                  0.007919092518223264,
                  0.014222896222606361
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 265,
                "false_positive_rate": 0.0049448601444272355,
                "false_positive_rate_ci95": [
                  0.004385420294338417,
                  0.005575267047014511
                ]
              },
              "false_positive_rate_ratio": 2.14723173308079,
              "false_positive_rate_ratio_ci95": [
                1.563120466383857,
                2.9496153461641788
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 227,
                "false_positive_rate": 0.05477799227799228,
                "false_positive_rate_ci95": [
                  0.048253266304662465,
                  0.06212738913263076
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1530,
                "false_positive_rate": 0.028549569890466683,
                "false_positive_rate_ci95": [
                  0.027173029387708294,
                  0.029993693667754385
                ]
              },
              "false_positive_rate_ratio": 1.9186976367123427,
              "false_positive_rate_ratio_ci95": [
                1.6750982028407313,
                2.1977222677944437
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4113,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058351568198395334,
              "false_omission_rate_ci95": [
                0.05159143464612642,
                0.06593591337391959
              ]
            },
            "without_identity_term": {
              "n_allowed": 53571,
              "n_truly_positive": 1775,
              "false_omission_rate": 0.03313359840212055,
              "false_omission_rate_ci95": [
                0.03165110227102816,
                0.03468304566310751
              ]
            }
          }
        },
        "model_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 809,
              "n_false_positive": 271,
              "false_discovery_rate": 0.33498145859085293,
              "false_discovery_rate_ci95": [
                0.303305117353319,
                0.3682175429604537
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5485,
              "n_false_positive": 1795,
              "false_discovery_rate": 0.3272561531449407,
              "false_discovery_rate_ci95": [
                0.31496346860304697,
                0.3397906331147628
              ]
            },
            "false_discovery_rate_ratio": 1.0236062954712137,
            "false_discovery_rate_ratio_ci95": [
              0.9222764145031342,
              1.136069221387143
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 362,
              "n_false_positive": 44,
              "false_discovery_rate": 0.12154696132596685,
              "false_discovery_rate_ci95": [
                0.09180219473987582,
                0.15923949736942566
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 2362,
              "n_false_positive": 265,
              "false_discovery_rate": 0.1121930567315834,
              "false_discovery_rate_ci95": [
                0.10008978941691218,
                0.12555570211730352
              ]
            },
            "false_discovery_rate_ratio": 1.0833732930261648,
            "false_discovery_rate_ratio_ci95": [
              0.8031670319613425,
              1.4613369888654093
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 447,
              "n_false_positive": 227,
              "false_discovery_rate": 0.5078299776286354,
              "false_discovery_rate_ci95": [
                0.4616152094413018,
                0.5539113129361424
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3123,
              "n_false_positive": 1530,
              "false_discovery_rate": 0.4899135446685879,
              "false_discovery_rate_ci95": [
                0.47240421004392447,
                0.5074476625794673
              ]
            },
            "false_discovery_rate_ratio": 1.0365706013949205,
            "false_discovery_rate_ratio_ci95": [
              0.9397789344284521,
              1.1433312370740634
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 271,
                "false_positive_rate": 0.06539575289575289,
                "false_positive_rate_ci95": [
                  0.05826389643505257,
                  0.07333261328991493
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1795,
                "false_positive_rate": 0.03349443003489392,
                "false_positive_rate_ci95": [
                  0.03200423665820344,
                  0.035051497831440334
                ]
              },
              "false_positive_rate_ratio": 1.9524366537249542,
              "false_positive_rate_ratio_ci95": [
                1.7251571551714946,
                2.2096589144829255
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 44,
                "false_positive_rate": 0.010617760617760617,
                "false_positive_rate_ci95": [
                  0.007919092518223264,
                  0.014222896222606361
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 265,
                "false_positive_rate": 0.0049448601444272355,
                "false_positive_rate_ci95": [
                  0.004385420294338417,
                  0.005575267047014511
                ]
              },
              "false_positive_rate_ratio": 2.14723173308079,
              "false_positive_rate_ratio_ci95": [
                1.563120466383857,
                2.9496153461641788
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 227,
                "false_positive_rate": 0.05477799227799228,
                "false_positive_rate_ci95": [
                  0.048253266304662465,
                  0.06212738913263076
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1530,
                "false_positive_rate": 0.028549569890466683,
                "false_positive_rate_ci95": [
                  0.027173029387708294,
                  0.029993693667754385
                ]
              },
              "false_positive_rate_ratio": 1.9186976367123427,
              "false_positive_rate_ratio_ci95": [
                1.6750982028407313,
                2.1977222677944437
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4113,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058351568198395334,
              "false_omission_rate_ci95": [
                0.05159143464612642,
                0.06593591337391959
              ]
            },
            "without_identity_term": {
              "n_allowed": 53571,
              "n_truly_positive": 1775,
              "false_omission_rate": 0.03313359840212055,
              "false_omission_rate_ci95": [
                0.03165110227102816,
                0.03468304566310751
              ]
            }
          }
        },
        "final_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 809,
              "n_false_positive": 271,
              "false_discovery_rate": 0.33498145859085293,
              "false_discovery_rate_ci95": [
                0.303305117353319,
                0.3682175429604537
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5503,
              "n_false_positive": 1806,
              "false_discovery_rate": 0.3281846265673269,
              "false_discovery_rate_ci95": [
                0.3159022116479804,
                0.34070675119945326
              ]
            },
            "false_discovery_rate_ratio": 1.0207103912654838,
            "false_discovery_rate_ratio_ci95": [
              0.9197145916688699,
              1.132796752682641
            ]
          },
          "priority_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 369,
              "n_false_positive": 45,
              "false_discovery_rate": 0.12195121951219512,
              "false_discovery_rate_ci95": [
                0.09240340715743013,
                0.15928925374692743
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 2454,
              "n_false_positive": 310,
              "false_discovery_rate": 0.1263243683781581,
              "false_discovery_rate_ci95": [
                0.11376163890810129,
                0.1400551630776712
              ]
            },
            "false_discovery_rate_ratio": 0.9653815892997639,
            "false_discovery_rate_ratio_ci95": [
              0.7202787586985904,
              1.2938901803002203
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 440,
              "n_false_positive": 226,
              "false_discovery_rate": 0.5136363636363637,
              "false_discovery_rate_ci95": [
                0.4670193002493458,
                0.5600173808895593
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3049,
              "n_false_positive": 1496,
              "false_discovery_rate": 0.49065267300754345,
              "false_discovery_rate_ci95": [
                0.4729311103154587,
                0.5083977596020485
              ]
            },
            "false_discovery_rate_ratio": 1.0468430967428295,
            "false_discovery_rate_ratio_ci95": [
              0.9492600349562594,
              1.1544576078657032
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 271,
                "false_positive_rate": 0.06539575289575289,
                "false_positive_rate_ci95": [
                  0.05826389643505257,
                  0.07333261328991493
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1806,
                "false_positive_rate": 0.033699688380511655,
                "false_positive_rate_ci95": [
                  0.03220498369326171,
                  0.035261238063461285
                ]
              },
              "false_positive_rate_ratio": 1.9405447361219783,
              "false_positive_rate_ratio_ci95": [
                1.7147398717168183,
                2.1960846277636503
              ]
            },
            "priority_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 45,
                "false_positive_rate": 0.01085907335907336,
                "false_positive_rate_ci95": [
                  0.00812571092432013,
                  0.014498456323047552
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 310,
                "false_positive_rate": 0.005784553376499786,
                "false_positive_rate_ci95": [
                  0.005176960720109571,
                  0.006462992713840148
                ]
              },
              "false_positive_rate_ratio": 1.877253549632582,
              "false_positive_rate_ratio_ci95": [
                1.3754059634726177,
                2.5622114366223543
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 226,
                "false_positive_rate": 0.05453667953667954,
                "false_positive_rate_ci95": [
                  0.04802652599887612,
                  0.061871950931879055
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1496,
                "false_positive_rate": 0.027915135004011868,
                "false_positive_rate_ci95": [
                  0.026553933125083844,
                  0.02934401110486126
                ]
              },
              "false_positive_rate_ratio": 1.9536598884025356,
              "false_positive_rate_ratio_ci95": [
                1.704794560035958,
                2.2388544925158036
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4113,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058351568198395334,
              "false_omission_rate_ci95": [
                0.05159143464612642,
                0.06593591337391959
              ]
            },
            "without_identity_term": {
              "n_allowed": 53553,
              "n_truly_positive": 1768,
              "false_omission_rate": 0.03301402349074748,
              "false_omission_rate_ci95": [
                0.031533935671710894,
                0.034561102094828
              ]
            }
          },
          "by_term": [
            {
              "term": "gay",
              "n_predicted_positive": 360,
              "n_false_positive": 88,
              "false_discovery_rate": 0.24444444444444444
            },
            {
              "term": "black",
              "n_predicted_positive": 78,
              "n_false_positive": 34,
              "false_discovery_rate": 0.4358974358974359
            },
            {
              "term": "white",
              "n_predicted_positive": 70,
              "n_false_positive": 27,
              "false_discovery_rate": 0.38571428571428573
            },
            {
              "term": "woman",
              "n_predicted_positive": 47,
              "n_false_positive": 21,
              "false_discovery_rate": 0.44680851063829785
            },
            {
              "term": "women",
              "n_predicted_positive": 39,
              "n_false_positive": 14,
              "false_discovery_rate": 0.358974358974359
            },
            {
              "term": "jew",
              "n_predicted_positive": 35,
              "n_false_positive": 13,
              "false_discovery_rate": 0.37142857142857144
            },
            {
              "term": "homosexual",
              "n_predicted_positive": 32,
              "n_false_positive": 16,
              "false_discovery_rate": 0.5
            },
            {
              "term": "jews",
              "n_predicted_positive": 31,
              "n_false_positive": 18,
              "false_discovery_rate": 0.5806451612903226
            },
            {
              "term": "christian",
              "n_predicted_positive": 22,
              "n_false_positive": 12,
              "false_discovery_rate": 0.5454545454545454
            },
            {
              "term": "jewish",
              "n_predicted_positive": 21,
              "n_false_positive": 10,
              "false_discovery_rate": 0.47619047619047616
            },
            {
              "term": "male",
              "n_predicted_positive": 19,
              "n_false_positive": 6,
              "false_discovery_rate": 0.3157894736842105
            },
            {
              "term": "queer",
              "n_predicted_positive": 19,
              "n_false_positive": 8,
              "false_discovery_rate": 0.42105263157894735
            }
          ]
        }
      }
    },
    "subgroup_thresholds": {
      "thresholds": {
        "priority_review": {
          "identity_term_present": {
            "toxic": 0.9800699647700561,
            "severe_toxic": null,
            "obscene": 0.9952075374744895,
            "threat": null,
            "insult": null,
            "identity_hate": null
          }
        }
      },
      "threshold_selection": {
        "priority_review": {
          "n_population_tier": 1040,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "human_review": {
          "n_population_tier": 1040,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "declared": [
          {
            "tier": "priority_review",
            "signal": "identity_term_present"
          }
        ]
      },
      "test": {
        "priority_review": {
          "n_population_tier": 2724,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "human_review": {
          "n_population_tier": 3570,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "declared": [
          {
            "tier": "priority_review",
            "signal": "identity_term_present"
          }
        ]
      }
    },
    "thresholds": {
      "priority_review": {
        "toxic": 0.98,
        "severe_toxic": null,
        "obscene": 0.995,
        "threat": null,
        "insult": 0.995,
        "identity_hate": null
      },
      "human_review": {
        "toxic": 0.6575178598345869,
        "severe_toxic": null,
        "obscene": 0.6575248417421482,
        "threat": null,
        "insult": 0.917856881186332,
        "identity_hate": null
      },
      "subgroup": {
        "priority_review": {
          "identity_term_present": {
            "toxic": 0.9800699647700561,
            "severe_toxic": null,
            "obscene": 0.9952075374744895,
            "threat": null,
            "insult": null,
            "identity_hate": null
          }
        }
      }
    },
    "identity_summary": {
      "selection_identity_predicted_positive_model": 210,
      "selection_identity_predicted_positive_final": 210,
      "selection_identity_moved_to_positive_by_rules": 0,
      "selection_identity_allowed": 1825,
      "selection_identity_allowed_truly_positive": 155,
      "test_population_tier_predicted_positive": {
        "ratio": 1.0236062954712137,
        "ci95": [
          0.9222764145031342,
          1.136069221387143
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 809,
          "n_false_positive": 271,
          "false_discovery_rate": 0.33498145859085293,
          "false_discovery_rate_ci95": [
            0.303305117353319,
            0.3682175429604537
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5485,
          "n_false_positive": 1795,
          "false_discovery_rate": 0.3272561531449407,
          "false_discovery_rate_ci95": [
            0.31496346860304697,
            0.3397906331147628
          ]
        }
      },
      "test_population_tier_priority_review": {
        "ratio": 1.0833732930261648,
        "ci95": [
          0.8031670319613425,
          1.4613369888654093
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 362,
          "n_false_positive": 44,
          "false_discovery_rate": 0.12154696132596685,
          "false_discovery_rate_ci95": [
            0.09180219473987582,
            0.15923949736942566
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 2362,
          "n_false_positive": 265,
          "false_discovery_rate": 0.1121930567315834,
          "false_discovery_rate_ci95": [
            0.10008978941691218,
            0.12555570211730352
          ]
        }
      },
      "test_population_tier_human_review": {
        "ratio": 1.0365706013949205,
        "ci95": [
          0.9397789344284521,
          1.1433312370740634
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 447,
          "n_false_positive": 227,
          "false_discovery_rate": 0.5078299776286354,
          "false_discovery_rate_ci95": [
            0.4616152094413018,
            0.5539113129361424
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3123,
          "n_false_positive": 1530,
          "false_discovery_rate": 0.4899135446685879,
          "false_discovery_rate_ci95": [
            0.47240421004392447,
            0.5074476625794673
          ]
        }
      },
      "test_model_tier_predicted_positive": {
        "ratio": 1.0236062954712137,
        "ci95": [
          0.9222764145031342,
          1.136069221387143
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 809,
          "n_false_positive": 271,
          "false_discovery_rate": 0.33498145859085293,
          "false_discovery_rate_ci95": [
            0.303305117353319,
            0.3682175429604537
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5485,
          "n_false_positive": 1795,
          "false_discovery_rate": 0.3272561531449407,
          "false_discovery_rate_ci95": [
            0.31496346860304697,
            0.3397906331147628
          ]
        }
      },
      "test_model_tier_priority_review": {
        "ratio": 1.0833732930261648,
        "ci95": [
          0.8031670319613425,
          1.4613369888654093
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 362,
          "n_false_positive": 44,
          "false_discovery_rate": 0.12154696132596685,
          "false_discovery_rate_ci95": [
            0.09180219473987582,
            0.15923949736942566
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 2362,
          "n_false_positive": 265,
          "false_discovery_rate": 0.1121930567315834,
          "false_discovery_rate_ci95": [
            0.10008978941691218,
            0.12555570211730352
          ]
        }
      },
      "test_model_tier_human_review": {
        "ratio": 1.0365706013949205,
        "ci95": [
          0.9397789344284521,
          1.1433312370740634
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 447,
          "n_false_positive": 227,
          "false_discovery_rate": 0.5078299776286354,
          "false_discovery_rate_ci95": [
            0.4616152094413018,
            0.5539113129361424
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3123,
          "n_false_positive": 1530,
          "false_discovery_rate": 0.4899135446685879,
          "false_discovery_rate_ci95": [
            0.47240421004392447,
            0.5074476625794673
          ]
        }
      },
      "test_final_tier_predicted_positive": {
        "ratio": 1.0207103912654838,
        "ci95": [
          0.9197145916688699,
          1.132796752682641
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 809,
          "n_false_positive": 271,
          "false_discovery_rate": 0.33498145859085293,
          "false_discovery_rate_ci95": [
            0.303305117353319,
            0.3682175429604537
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5503,
          "n_false_positive": 1806,
          "false_discovery_rate": 0.3281846265673269,
          "false_discovery_rate_ci95": [
            0.3159022116479804,
            0.34070675119945326
          ]
        }
      },
      "test_final_tier_priority_review": {
        "ratio": 0.9653815892997639,
        "ci95": [
          0.7202787586985904,
          1.2938901803002203
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 369,
          "n_false_positive": 45,
          "false_discovery_rate": 0.12195121951219512,
          "false_discovery_rate_ci95": [
            0.09240340715743013,
            0.15928925374692743
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 2454,
          "n_false_positive": 310,
          "false_discovery_rate": 0.1263243683781581,
          "false_discovery_rate_ci95": [
            0.11376163890810129,
            0.1400551630776712
          ]
        }
      },
      "test_final_tier_human_review": {
        "ratio": 1.0468430967428295,
        "ci95": [
          0.9492600349562594,
          1.1544576078657032
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 440,
          "n_false_positive": 226,
          "false_discovery_rate": 0.5136363636363637,
          "false_discovery_rate_ci95": [
            0.4670193002493458,
            0.5600173808895593
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3049,
          "n_false_positive": 1496,
          "false_discovery_rate": 0.49065267300754345,
          "false_discovery_rate_ci95": [
            0.4729311103154587,
            0.5083977596020485
          ]
        }
      },
      "test_by_term": [
        {
          "term": "gay",
          "n_predicted_positive": 360,
          "n_false_positive": 88,
          "false_discovery_rate": 0.24444444444444444
        },
        {
          "term": "black",
          "n_predicted_positive": 78,
          "n_false_positive": 34,
          "false_discovery_rate": 0.4358974358974359
        },
        {
          "term": "white",
          "n_predicted_positive": 70,
          "n_false_positive": 27,
          "false_discovery_rate": 0.38571428571428573
        },
        {
          "term": "woman",
          "n_predicted_positive": 47,
          "n_false_positive": 21,
          "false_discovery_rate": 0.44680851063829785
        },
        {
          "term": "women",
          "n_predicted_positive": 39,
          "n_false_positive": 14,
          "false_discovery_rate": 0.358974358974359
        },
        {
          "term": "jew",
          "n_predicted_positive": 35,
          "n_false_positive": 13,
          "false_discovery_rate": 0.37142857142857144
        },
        {
          "term": "homosexual",
          "n_predicted_positive": 32,
          "n_false_positive": 16,
          "false_discovery_rate": 0.5
        },
        {
          "term": "jews",
          "n_predicted_positive": 31,
          "n_false_positive": 18,
          "false_discovery_rate": 0.5806451612903226
        },
        {
          "term": "christian",
          "n_predicted_positive": 22,
          "n_false_positive": 12,
          "false_discovery_rate": 0.5454545454545454
        },
        {
          "term": "jewish",
          "n_predicted_positive": 21,
          "n_false_positive": 10,
          "false_discovery_rate": 0.47619047619047616
        },
        {
          "term": "male",
          "n_predicted_positive": 19,
          "n_false_positive": 6,
          "false_discovery_rate": 0.3157894736842105
        },
        {
          "term": "queer",
          "n_predicted_positive": 19,
          "n_false_positive": 8,
          "false_discovery_rate": 0.42105263157894735
        }
      ],
      "identity_terms": [
        "african",
        "africans",
        "arab",
        "arabs",
        "asian",
        "asians",
        "atheist",
        "atheists",
        "autistic",
        "bisexual",
        "black",
        "blacks",
        "buddhist",
        "buddhists",
        "catholic",
        "catholics",
        "chinese",
        "christian",
        "christians",
        "disability",
        "disabled",
        "elderly",
        "female",
        "females",
        "feminist",
        "feminists",
        "gay",
        "gays",
        "heterosexual",
        "hindu",
        "hindus",
        "hispanic",
        "homosexual",
        "homosexuals",
        "immigrant",
        "immigrants",
        "indian",
        "indians",
        "islam",
        "islamic",
        "jew",
        "jewish",
        "jews",
        "latina",
        "latino",
        "lesbian",
        "lesbians",
        "lgbt",
        "male",
        "males",
        "mexican",
        "mexicans",
        "muslim",
        "muslims",
        "queer",
        "refugee",
        "refugees",
        "trans",
        "transgender",
        "white",
        "whites",
        "woman",
        "women"
      ]
    }
  },
  "threshold_selection": {
    "note": "development data; per-label precision targets do not guarantee tier precision",
    "per_label": {
      "toxic": {
        "positives": 3004,
        "negatives": 28910,
        "average_precision": 0.8591605013116437,
        "roc_auc": 0.9711163047448322,
        "at_priority_review": {
          "threshold": 0.98,
          "n_predicted_positive": 1039,
          "precision": 0.9865255052935515,
          "recall": 0.34121171770972036
        },
        "at_human_review": {
          "threshold": 0.6575178598345869,
          "n_predicted_positive": 2074,
          "precision": 0.9001928640308583,
          "recall": 0.6215046604527297
        }
      },
      "severe_toxic": {
        "positives": 309,
        "negatives": 31605,
        "average_precision": 0.49208182986820953,
        "roc_auc": 0.9875912162110272,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 1642,
        "negatives": 30272,
        "average_precision": 0.8739265303249879,
        "roc_auc": 0.9852955312354345,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 572,
          "precision": 0.986013986013986,
          "recall": 0.34348355663824603
        },
        "at_human_review": {
          "threshold": 0.6575248417421482,
          "n_predicted_positive": 1177,
          "precision": 0.9005947323704333,
          "recall": 0.6455542021924482
        }
      },
      "threat": {
        "positives": 79,
        "negatives": 31835,
        "average_precision": 0.45854647319377284,
        "roc_auc": 0.9861334054350657,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 1543,
        "negatives": 30371,
        "average_precision": 0.7743441564479363,
        "roc_auc": 0.9778964622274467,
        "at_priority_review": {
          "threshold": 0.995,
          "n_predicted_positive": 209,
          "precision": 0.9617224880382775,
          "recall": 0.13026571613739468
        },
        "at_human_review": {
          "threshold": 0.917856881186332,
          "n_predicted_positive": 525,
          "precision": 0.900952380952381,
          "recall": 0.30654569021386907
        }
      },
      "identity_hate": {
        "positives": 270,
        "negatives": 31644,
        "average_precision": 0.4145628903623871,
        "roc_auc": 0.9700846687921647,
        "at_priority_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "review_workload": {
      "n_total": 31914,
      "n_requires_human_review": 2083,
      "n_priority_review": 1060,
      "n_human_review": 1023,
      "review_fraction": 0.0652691608698377,
      "n_truly_positive": 1946,
      "precision": 0.9342294767162747,
      "precision_ci95": [
        0.922764947455796,
        0.9440953462675998
      ],
      "high_risk_by_band": {
        "priority_review": {
          "n": 1060,
          "n_high_risk": 385,
          "high_risk_share": 0.3632075471698113
        },
        "human_review": {
          "n": 1023,
          "n_high_risk": 95,
          "high_risk_share": 0.09286412512218964
        },
        "allow": {
          "n": 29831,
          "n_high_risk": 97,
          "high_risk_share": 0.0032516509671147464
        }
      },
      "n_high_risk_total": 577,
      "high_risk_recall_into_queue": 0.8318890814558059,
      "high_risk_recall_into_priority": 0.6672443674176777
    },
    "tiers": {
      "priority_review": {
        "n_predicted_positive": 1060,
        "coverage": 0.03321426333270665,
        "precision": 0.9952830188679245,
        "precision_ci95": [
          0.989005616202792,
          0.9979835551951235
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1023,
        "coverage": 0.03205489753713104,
        "precision": 0.8709677419354839,
        "precision_ci95": [
          0.8490286461543156,
          0.8901312245689615
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 29831,
        "coverage": 0.9347308391301623,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "tiers_model_only": {
      "priority_review": {
        "n_predicted_positive": 1040,
        "coverage": 0.03258757911888199,
        "precision": 0.9971153846153846,
        "precision_ci95": [
          0.9915533874584405,
          0.9990184960421026
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1040,
        "coverage": 0.03258757911888199,
        "precision": 0.8730769230769231,
        "precision_ci95": [
          0.8514631038990778,
          0.8919448086699951
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 29834,
        "coverage": 0.934824841762236,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "rules": {
      "matched_per_rule": {
        "R101_high_risk_priority": 67,
        "R102_hierarchy_violation": 0
      },
      "n_rows_with_any_rule": 67,
      "n_added_to_queue_from_allow": 3,
      "n_added_to_queue_from_allow_true_positive": 1,
      "n_promoted_to_priority_review": 20,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "allow_false_negatives": {
      "toxic": 1131,
      "severe_toxic": 13,
      "obscene": 296,
      "threat": 23,
      "insult": 292,
      "identity_hate": 69
    },
    "ranking": {
      "n_pool": 2083,
      "n_high_risk": 480,
      "fractions": [
        0.5,
        0.67,
        1.0
      ],
      "by_key": {
        "fifo": {
          "ap_vs_high_risk": 0.23043686989918388,
          "at_fraction": [
            {
              "fraction": 0.5,
              "k": 1042,
              "harm_share_at_k": 0.5077319587628866,
              "high_risk_share_at_k": 0.49583333333333335
            },
            {
              "fraction": 0.67,
              "k": 1396,
              "harm_share_at_k": 0.6712076583210603,
              "high_risk_share_at_k": 0.65
            },
            {
              "fraction": 1.0,
              "k": 2083,
              "harm_share_at_k": 1.0,
              "high_risk_share_at_k": 1.0
            }
          ]
        },
        "prob": {
          "ap_vs_high_risk": 0.5025469181519105,
          "at_fraction": [
            {
              "fraction": 0.5,
              "k": 1042,
              "harm_share_at_k": 0.6377025036818851,
              "high_risk_share_at_k": 0.7791666666666667
            },
            {
              "fraction": 0.67,
              "k": 1396,
              "harm_share_at_k": 0.7812960235640648,
              "high_risk_share_at_k": 0.8729166666666667
            },
            {
              "fraction": 1.0,
              "k": 2083,
              "harm_share_at_k": 1.0,
              "high_risk_share_at_k": 1.0
            }
          ]
        },
        "severity": {
          "ap_vs_high_risk": 0.5925123820800685,
          "at_fraction": [
            {
              "fraction": 0.5,
              "k": 1042,
              "harm_share_at_k": 0.6568483063328424,
              "high_risk_share_at_k": 0.81875
            },
            {
              "fraction": 0.67,
              "k": 1396,
              "harm_share_at_k": 0.7980486008836525,
              "high_risk_share_at_k": 0.9041666666666667
            },
            {
              "fraction": 1.0,
              "k": 2083,
              "harm_share_at_k": 1.0,
              "high_risk_share_at_k": 1.0
            }
          ]
        },
        "priority": {
          "ap_vs_high_risk": 0.588880730721175,
          "at_fraction": [
            {
              "fraction": 0.5,
              "k": 1042,
              "harm_share_at_k": 0.6478276877761414,
              "high_risk_share_at_k": 0.7958333333333333
            },
            {
              "fraction": 0.67,
              "k": 1396,
              "harm_share_at_k": 0.801178203240059,
              "high_risk_share_at_k": 0.9104166666666667
            },
            {
              "fraction": 1.0,
              "k": 2083,
              "harm_share_at_k": 1.0,
              "high_risk_share_at_k": 1.0
            }
          ]
        },
        "oracle_true_harm": {
          "ap_vs_high_risk": 1.0,
          "at_fraction": [
            {
              "fraction": 0.5,
              "k": 1042,
              "harm_share_at_k": 0.7354565537555229,
              "high_risk_share_at_k": 1.0
            },
            {
              "fraction": 0.67,
              "k": 1396,
              "harm_share_at_k": 0.8657952871870398,
              "high_risk_share_at_k": 1.0
            },
            {
              "fraction": 1.0,
              "k": 2083,
              "harm_share_at_k": 1.0,
              "high_risk_share_at_k": 1.0
            }
          ]
        }
      },
      "note": "oracle_true_harm ranks by the true-label harm proxy: it uses true labels and is not shippable; it is the ceiling for these shares. Shares are top-k over the whole review pool (both bands), k = ceil(fraction * n_pool); high risk = harm proxy >= 5; fifo scores are constant, so its order is pool row order."
    },
    "cross_fitted": {
      "k": 5,
      "seed": 20260922,
      "fold_sizes": [
        6383,
        6383,
        6383,
        6383,
        6382
      ],
      "tiers": {
        "priority_review": {
          "n_predicted_positive": 1032,
          "coverage": 0.03233690543335213,
          "precision": 0.9951550387596899,
          "precision_ci95": [
            0.9887086736847855,
            0.9979287998758454
          ],
          "precision_note": null
        },
        "human_review": {
          "n_predicted_positive": 1057,
          "coverage": 0.03312026070063295,
          "precision": 0.8713339640491958,
          "precision_ci95": [
            0.849795786891981,
            0.8901828342563827
          ],
          "precision_note": null
        },
        "allow": {
          "n_predicted_positive": 29825,
          "coverage": 0.9345428338660149,
          "precision": null,
          "precision_ci95": null,
          "precision_note": "allow is the fall-through; see false-negative counts"
        }
      },
      "review_workload": {
        "n_total": 31914,
        "n_requires_human_review": 2089,
        "n_priority_review": 1032,
        "n_human_review": 1057,
        "review_fraction": 0.06545716613398508,
        "n_truly_positive": 1948,
        "precision": 0.932503590234562,
        "precision_ci95": [
          0.9209320003748824,
          0.9424874394167542
        ],
        "high_risk_by_band": {
          "priority_review": {
            "n": 1032,
            "n_high_risk": 382,
            "high_risk_share": 0.3701550387596899
          },
          "human_review": {
            "n": 1057,
            "n_high_risk": 98,
            "high_risk_share": 0.09271523178807947
          },
          "allow": {
            "n": 29825,
            "n_high_risk": 97,
            "high_risk_share": 0.0032523051131601007
          }
        },
        "n_high_risk_total": 577,
        "high_risk_recall_into_queue": 0.8318890814558059,
        "high_risk_recall_into_priority": 0.6620450606585788
      },
      "thresholds_per_fold": [
        {
          "priority_review": {
            "toxic": 0.98,
            "severe_toxic": null,
            "obscene": 0.995,
            "threat": null,
            "insult": 0.995,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6440317478214106,
            "severe_toxic": null,
            "obscene": 0.6192878393880997,
            "threat": null,
            "insult": 0.8975968932080814,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.9800699647700561,
                "severe_toxic": null,
                "obscene": 0.9952075374744895,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        },
        {
          "priority_review": {
            "toxic": 0.98,
            "severe_toxic": null,
            "obscene": 0.99,
            "threat": null,
            "insult": 0.995,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6430854031501667,
            "severe_toxic": null,
            "obscene": 0.6352345096959657,
            "threat": null,
            "insult": 0.8781780887834073,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.9800699647700561,
                "severe_toxic": null,
                "obscene": 0.9923311406857134,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        },
        {
          "priority_review": {
            "toxic": 0.98,
            "severe_toxic": null,
            "obscene": 0.99,
            "threat": null,
            "insult": 0.995,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6594254978049771,
            "severe_toxic": null,
            "obscene": 0.6509424999759325,
            "threat": null,
            "insult": 0.8956093301846555,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.9811756266817863,
                "severe_toxic": null,
                "obscene": 0.9921862234599866,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        },
        {
          "priority_review": {
            "toxic": 0.99,
            "severe_toxic": null,
            "obscene": 0.99,
            "threat": null,
            "insult": 0.995,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6661983709931825,
            "severe_toxic": null,
            "obscene": 0.71009819362157,
            "threat": null,
            "insult": 0.9087201990937706,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.9902181204716101,
                "severe_toxic": null,
                "obscene": 0.9921862234599866,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        },
        {
          "priority_review": {
            "toxic": 0.98,
            "severe_toxic": null,
            "obscene": 0.995,
            "threat": null,
            "insult": 0.995,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6575973190444335,
            "severe_toxic": null,
            "obscene": 0.6933768503995272,
            "threat": null,
            "insult": 0.9259224377587684,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.9800699647700561,
                "severe_toxic": null,
                "obscene": 0.9952075374744895,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        }
      ],
      "note": "thresholds refit on k-1 folds; band membership is out-of-sample relative to the thresholds that define it; this estimates band quality in the same train.csv distribution, not distribution shift, and is not a formal bound on selection optimism"
    },
    "selection_rules": {
      "human_review": "cumulative_precision",
      "priority_review": "segment_agreement"
    },
    "segment_check": {
      "toxic": {
        "tier": "priority_review",
        "threshold": 0.98,
        "rule": "segment_agreement",
        "edges": [
          0.5,
          0.8,
          0.9,
          0.95,
          0.98,
          0.99,
          0.995,
          1.0
        ],
        "segments": [
          {
            "lo": 0.98,
            "hi": 0.99,
            "n": 129,
            "agreement": 0.9612403100775194,
            "meets_floor": true
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "n": 109,
            "agreement": 0.963302752293578,
            "meets_floor": true
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "n": 801,
            "agreement": 0.9937578027465668,
            "meets_floor": true
          }
        ],
        "all_meet_floor": true
      },
      "severe_toxic": {
        "tier": "priority_review",
        "threshold": null,
        "rule": "segment_agreement",
        "edges": [
          0.1,
          0.2,
          0.3,
          0.5,
          0.8,
          0.9,
          1.0
        ],
        "segments": [],
        "all_meet_floor": null
      },
      "obscene": {
        "tier": "priority_review",
        "threshold": 0.995,
        "rule": "segment_agreement",
        "edges": [
          0.5,
          0.8,
          0.9,
          0.95,
          0.98,
          0.99,
          0.995,
          1.0
        ],
        "segments": [
          {
            "lo": 0.995,
            "hi": 1.0,
            "n": 572,
            "agreement": 0.986013986013986,
            "meets_floor": true
          }
        ],
        "all_meet_floor": true
      },
      "threat": {
        "tier": "priority_review",
        "threshold": null,
        "rule": "segment_agreement",
        "edges": [
          0.1,
          0.2,
          0.3,
          0.5,
          0.8,
          0.9,
          1.0
        ],
        "segments": [],
        "all_meet_floor": null
      },
      "insult": {
        "tier": "priority_review",
        "threshold": 0.995,
        "rule": "segment_agreement",
        "edges": [
          0.5,
          0.8,
          0.9,
          0.95,
          0.98,
          0.99,
          0.995,
          1.0
        ],
        "segments": [
          {
            "lo": 0.995,
            "hi": 1.0,
            "n": 209,
            "agreement": 0.9617224880382775,
            "meets_floor": true
          }
        ],
        "all_meet_floor": true
      },
      "identity_hate": {
        "tier": "priority_review",
        "threshold": null,
        "rule": "segment_agreement",
        "edges": [
          0.1,
          0.2,
          0.3,
          0.5,
          0.8,
          0.9,
          1.0
        ],
        "segments": [],
        "all_meet_floor": null
      }
    },
    "identity_share": {
      "priority_review": {
        "n": 1060,
        "n_with_identity_term": 109,
        "share": 0.10283018867924529
      },
      "human_review": {
        "n": 1023,
        "n_with_identity_term": 101,
        "share": 0.09872922776148582
      }
    },
    "alternatives": {
      "cumulative_precision": {
        "thresholds": {
          "priority_review": {
            "toxic": 0.8473530547862892,
            "severe_toxic": null,
            "obscene": 0.9258484945343994,
            "threat": null,
            "insult": 0.990352467757599,
            "identity_hate": null
          },
          "human_review": {
            "toxic": 0.6575178598345869,
            "severe_toxic": null,
            "obscene": 0.6575248417421482,
            "threat": null,
            "insult": 0.917856881186332,
            "identity_hate": null
          },
          "subgroup": {
            "priority_review": {
              "identity_term_present": {
                "toxic": 0.8950082386695898,
                "severe_toxic": null,
                "obscene": 0.9447978435575853,
                "threat": null,
                "insult": null,
                "identity_hate": null
              }
            }
          }
        },
        "tiers": {
          "priority_review": {
            "n_predicted_positive": 1628,
            "coverage": 0.051012095005326816,
            "precision": 0.9723587223587223,
            "precision_ci95": [
              0.9632151228546284,
              0.9792783967391017
            ],
            "precision_note": null
          },
          "human_review": {
            "n_predicted_positive": 455,
            "coverage": 0.014257065864510874,
            "precision": 0.7978021978021979,
            "precision_ci95": [
              0.7584748970037196,
              0.8321430490309014
            ],
            "precision_note": null
          },
          "allow": {
            "n_predicted_positive": 29831,
            "coverage": 0.9347308391301623,
            "precision": null,
            "precision_ci95": null,
            "precision_note": "allow is the fall-through; see false-negative counts"
          }
        },
        "high_risk_by_band": {
          "priority_review": {
            "n": 1628,
            "n_high_risk": 446,
            "high_risk_share": 0.273955773955774
          },
          "human_review": {
            "n": 455,
            "n_high_risk": 34,
            "high_risk_share": 0.07472527472527472
          },
          "allow": {
            "n": 29831,
            "n_high_risk": 97,
            "high_risk_share": 0.0032516509671147464
          }
        },
        "n_high_risk_total": 577,
        "high_risk_recall_into_queue": 0.8318890814558059,
        "high_risk_recall_into_priority": 0.7729636048526863,
        "identity_share": {
          "priority_review": {
            "n": 1628,
            "n_with_identity_term": 150,
            "share": 0.09213759213759214
          },
          "human_review": {
            "n": 455,
            "n_with_identity_term": 60,
            "share": 0.13186813186813187
          }
        },
        "note": "both tiers selected by cumulative precision at the configured floors on the same selection split; in-sample development numbers for comparison only"
      }
    },
    "selection_notes": []
  },
  "agreement_by_confidence": {
    "calibration": {
      "n_rows": 31914,
      "min_rows": 30,
      "edges": [
        0.5,
        0.8,
        0.9,
        0.95,
        0.98,
        0.99,
        0.995,
        1.0
      ],
      "per_label": {
        "toxic": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 29452,
              "coverage": 0.9228551732781851,
              "mean_score": 0.03196342448754691,
              "n_positive": 978,
              "agreement": 0.03320657340757843,
              "agreement_ci95": [
                0.031220376663987772,
                0.035314523098326785
              ],
              "recall_share": 0.31467181467181465,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 652,
              "coverage": 0.02042990537068371,
              "mean_score": 0.6515618683219284,
              "n_positive": 415,
              "agreement": 0.6365030674846626,
              "agreement_ci95": [
                0.5988820098636819,
                0.6725250468106919
              ],
              "recall_share": 0.13352638352638352,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 278,
              "coverage": 0.008710910572162688,
              "mean_score": 0.8569667233997817,
              "n_positive": 226,
              "agreement": 0.8129496402877698,
              "agreement_ci95": [
                0.7629590828317611,
                0.854409279830895
              ],
              "recall_share": 0.07271557271557272,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 223,
              "coverage": 0.0069875289841448895,
              "mean_score": 0.9247196686304303,
              "n_positive": 207,
              "agreement": 0.9282511210762332,
              "agreement_ci95": [
                0.886641160666057,
                0.9553565982178184
              ],
              "recall_share": 0.06660231660231661,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 218,
              "coverage": 0.006830857930688726,
              "mean_score": 0.9671894103952766,
              "n_positive": 202,
              "agreement": 0.926605504587156,
              "agreement_ci95": [
                0.8841154711882634,
                0.9543211354675095
              ],
              "recall_share": 0.064993564993565,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 149,
              "coverage": 0.00466879739299367,
              "mean_score": 0.9857692032894073,
              "n_positive": 147,
              "agreement": 0.9865771812080537,
              "agreement_ci95": [
                0.9523842446572325,
                0.9963112275092164
              ],
              "recall_share": 0.0472972972972973,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 101,
              "coverage": 0.0031647552798145014,
              "mean_score": 0.9928050000943849,
              "n_positive": 97,
              "agreement": 0.9603960396039604,
              "agreement_ci95": [
                0.9025608078083471,
                0.984492853904781
              ],
              "recall_share": 0.03120978120978121,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 841,
              "coverage": 0.026352071191326692,
              "mean_score": 0.9992458334837921,
              "n_positive": 836,
              "agreement": 0.9940546967895363,
              "agreement_ci95": [
                0.9861585740409371,
                0.99745792733244
              ],
              "recall_share": 0.26898326898326896,
              "meets_min_rows": true
            }
          ]
        },
        "severe_toxic": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31377,
              "coverage": 0.983173528858808,
              "mean_score": 0.0030618016242436267,
              "n_positive": 106,
              "agreement": 0.003378270707843325,
              "agreement_ci95": [
                0.0027942023122708745,
                0.004083926152324817
              ],
              "recall_share": 0.34983498349834985,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 192,
              "coverage": 0.006016168452716676,
              "mean_score": 0.14526555918657366,
              "n_positive": 42,
              "agreement": 0.21875,
              "agreement_ci95": [
                0.16610635666863224,
                0.2824271631645646
              ],
              "recall_share": 0.13861386138613863,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 78,
              "coverage": 0.0024440684339161497,
              "mean_score": 0.24437132895576832,
              "n_positive": 23,
              "agreement": 0.2948717948717949,
              "agreement_ci95": [
                0.20524218405654404,
                0.4037579432943962
              ],
              "recall_share": 0.07590759075907591,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 93,
              "coverage": 0.00291408159428464,
              "mean_score": 0.39094203913950026,
              "n_positive": 36,
              "agreement": 0.3870967741935484,
              "agreement_ci95": [
                0.29446055280988226,
                0.48869017372201673
              ],
              "recall_share": 0.1188118811881188,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 118,
              "coverage": 0.003697436861565457,
              "mean_score": 0.6276276108935867,
              "n_positive": 61,
              "agreement": 0.5169491525423728,
              "agreement_ci95": [
                0.42768308187075155,
                0.6051464659426655
              ],
              "recall_share": 0.20132013201320131,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 31,
              "coverage": 0.0009713605314282133,
              "mean_score": 0.8564044681017464,
              "n_positive": 18,
              "agreement": 0.5806451612903226,
              "agreement_ci95": [
                0.4076626236738884,
                0.7358445703570756
              ],
              "recall_share": 0.0594059405940594,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 25,
              "coverage": 0.0007833552672808172,
              "mean_score": 0.9532097000049184,
              "n_positive": 17,
              "agreement": 0.68,
              "agreement_ci95": [
                0.4841026912048403,
                0.8279480978014191
              ],
              "recall_share": 0.056105610561056105,
              "meets_min_rows": false
            }
          ]
        },
        "obscene": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 30565,
              "coverage": 0.9577301497775271,
              "mean_score": 0.014576587122882623,
              "n_positive": 483,
              "agreement": 0.015802388352690988,
              "agreement_ci95": [
                0.014463899198885565,
                0.017262571697495885
              ],
              "recall_share": 0.2897420515896821,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 266,
              "coverage": 0.008334900043867895,
              "mean_score": 0.6528054091923227,
              "n_positive": 172,
              "agreement": 0.6466165413533834,
              "agreement_ci95": [
                0.587456285869917,
                0.701602335156559
              ],
              "recall_share": 0.10317936412717456,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 128,
              "coverage": 0.004010778968477784,
              "mean_score": 0.8501087004002563,
              "n_positive": 107,
              "agreement": 0.8359375,
              "agreement_ci95": [
                0.7621819233116779,
                0.8901166838922249
              ],
              "recall_share": 0.06418716256748651,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 112,
              "coverage": 0.003509431597418061,
              "mean_score": 0.9281817022600516,
              "n_positive": 97,
              "agreement": 0.8660714285714286,
              "agreement_ci95": [
                0.7907354328343577,
                0.9171285805498859
              ],
              "recall_share": 0.05818836232753449,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 121,
              "coverage": 0.0037914394936391553,
              "mean_score": 0.9677684288576278,
              "n_positive": 108,
              "agreement": 0.8925619834710744,
              "agreement_ci95": [
                0.8248348374317307,
                0.9361303167029194
              ],
              "recall_share": 0.0647870425914817,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 80,
              "coverage": 0.002506736855298615,
              "mean_score": 0.98517132584668,
              "n_positive": 72,
              "agreement": 0.9,
              "agreement_ci95": [
                0.8148931111226091,
                0.9484523846972116
              ],
              "recall_share": 0.04319136172765447,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 60,
              "coverage": 0.0018800526414739613,
              "mean_score": 0.9924783266581728,
              "n_positive": 56,
              "agreement": 0.9333333333333333,
              "agreement_ci95": [
                0.8407464259125857,
                0.9737713014577508
              ],
              "recall_share": 0.033593281343731254,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 582,
              "coverage": 0.018236510622297423,
              "mean_score": 0.9992909718642565,
              "n_positive": 572,
              "agreement": 0.9828178694158075,
              "agreement_ci95": [
                0.9686631751963131,
                0.9906407311424037
              ],
              "recall_share": 0.34313137372525493,
              "meets_min_rows": true
            }
          ]
        },
        "threat": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31783,
              "coverage": 0.9958952183994485,
              "mean_score": 0.001319789114780408,
              "n_positive": 47,
              "agreement": 0.0014787779630620144,
              "agreement_ci95": [
                0.0011123202256152714,
                0.001965728878692918
              ],
              "recall_share": 0.44339622641509435,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 37,
              "coverage": 0.0011593657955756094,
              "mean_score": 0.13639229835906513,
              "n_positive": 10,
              "agreement": 0.2702702702702703,
              "agreement_ci95": [
                0.15397461440966215,
                0.42978168456560734
              ],
              "recall_share": 0.09433962264150944,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 11,
              "coverage": 0.0003446763176035596,
              "mean_score": 0.24894093254131688,
              "n_positive": 6,
              "agreement": 0.5454545454545454,
              "agreement_ci95": [
                0.2800915357859759,
                0.7872872851236217
              ],
              "recall_share": 0.05660377358490566,
              "meets_min_rows": false
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 25,
              "coverage": 0.0007833552672808172,
              "mean_score": 0.39361621363711435,
              "n_positive": 9,
              "agreement": 0.36,
              "agreement_ci95": [
                0.20247880678156654,
                0.554815023991343
              ],
              "recall_share": 0.08490566037735849,
              "meets_min_rows": false
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 26,
              "coverage": 0.0008146894779720499,
              "mean_score": 0.6361381034628829,
              "n_positive": 13,
              "agreement": 0.5,
              "agreement_ci95": [
                0.32060593402470916,
                0.6793940659752908
              ],
              "recall_share": 0.12264150943396226,
              "meets_min_rows": false
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 10,
              "coverage": 0.0003133421069123269,
              "mean_score": 0.8540855357053116,
              "n_positive": 5,
              "agreement": 0.5,
              "agreement_ci95": [
                0.23659308901147935,
                0.7634069109885206
              ],
              "recall_share": 0.04716981132075472,
              "meets_min_rows": false
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 22,
              "coverage": 0.0006893526352071192,
              "mean_score": 0.9646608062962454,
              "n_positive": 16,
              "agreement": 0.7272727272727273,
              "agreement_ci95": [
                0.5184826582032107,
                0.8684923978853596
              ],
              "recall_share": 0.1509433962264151,
              "meets_min_rows": false
            }
          ]
        },
        "insult": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 30783,
              "coverage": 0.9645610077082158,
              "mean_score": 0.01958240740281177,
              "n_positive": 646,
              "agreement": 0.020985608939999352,
              "agreement_ci95": [
                0.019443154397917075,
                0.022647602479887795
              ],
              "recall_share": 0.4146341463414634,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 386,
              "coverage": 0.012095005326815818,
              "mean_score": 0.6476879881439428,
              "n_positive": 262,
              "agreement": 0.6787564766839378,
              "agreement_ci95": [
                0.630608527391035,
                0.7233815289667728
              ],
              "recall_share": 0.1681643132220796,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 164,
              "coverage": 0.0051388105533621605,
              "mean_score": 0.8542359007885804,
              "n_positive": 127,
              "agreement": 0.774390243902439,
              "agreement_ci95": [
                0.7045640999267878,
                0.8316562153628423
              ],
              "recall_share": 0.08151476251604621,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 137,
              "coverage": 0.004292786864698878,
              "mean_score": 0.9280714433294472,
              "n_positive": 117,
              "agreement": 0.8540145985401459,
              "agreement_ci95": [
                0.7852511722067829,
                0.9034664878520605
              ],
              "recall_share": 0.07509627727856226,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 127,
              "coverage": 0.003979444757786552,
              "mean_score": 0.9667342985258538,
              "n_positive": 113,
              "agreement": 0.889763779527559,
              "agreement_ci95": [
                0.823450907989796,
                0.933189998084297
              ],
              "recall_share": 0.07252888318356868,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 53,
              "coverage": 0.0016607131666353326,
              "mean_score": 0.9851429531093625,
              "n_positive": 49,
              "agreement": 0.9245283018867925,
              "agreement_ci95": [
                0.8214152241127329,
                0.9702604451197382
              ],
              "recall_share": 0.031450577663671375,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 64,
              "coverage": 0.002005389484238892,
              "mean_score": 0.992903565214475,
              "n_positive": 62,
              "agreement": 0.96875,
              "agreement_ci95": [
                0.8930270814228412,
                0.9913878617313676
              ],
              "recall_share": 0.03979460847240052,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 200,
              "coverage": 0.006266842138246537,
              "mean_score": 0.9983728575908407,
              "n_positive": 182,
              "agreement": 0.91,
              "agreement_ci95": [
                0.8622342807918277,
                0.9423125513396227
              ],
              "recall_share": 0.11681643132220795,
              "meets_min_rows": true
            }
          ]
        },
        "identity_hate": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31442,
              "coverage": 0.9852102525537382,
              "mean_score": 0.0034447720555972664,
              "n_positive": 125,
              "agreement": 0.003975574072896127,
              "agreement_ci95": [
                0.003338027504831002,
                0.004734310429366754
              ],
              "recall_share": 0.44483985765124556,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 182,
              "coverage": 0.005702826345804349,
              "mean_score": 0.14000795800842958,
              "n_positive": 33,
              "agreement": 0.1813186813186813,
              "agreement_ci95": [
                0.13212263570062185,
                0.24368941023206955
              ],
              "recall_share": 0.11743772241992882,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 84,
              "coverage": 0.0026320736980635457,
              "mean_score": 0.24222532023845583,
              "n_positive": 23,
              "agreement": 0.27380952380952384,
              "agreement_ci95": [
                0.18992825639172467,
                0.3774741924752613
              ],
              "recall_share": 0.08185053380782918,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 85,
              "coverage": 0.0026634079087547785,
              "mean_score": 0.3961258921028607,
              "n_positive": 28,
              "agreement": 0.32941176470588235,
              "agreement_ci95": [
                0.238777735509641,
                0.43479808860252434
              ],
              "recall_share": 0.099644128113879,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 63,
              "coverage": 0.0019740552735476595,
              "mean_score": 0.6512611303334834,
              "n_positive": 32,
              "agreement": 0.5079365079365079,
              "agreement_ci95": [
                0.3876288618517105,
                0.6273319127444259
              ],
              "recall_share": 0.11387900355871886,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 33,
              "coverage": 0.0010340289528106787,
              "mean_score": 0.8571046702129679,
              "n_positive": 20,
              "agreement": 0.6060606060606061,
              "agreement_ci95": [
                0.43683440692015163,
                0.753168926621936
              ],
              "recall_share": 0.0711743772241993,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 25,
              "coverage": 0.0007833552672808172,
              "mean_score": 0.9627504892352454,
              "n_positive": 20,
              "agreement": 0.8,
              "agreement_ci95": [
                0.6086904946512343,
                0.9113941536925313
              ],
              "recall_share": 0.0711743772241993,
              "meets_min_rows": false
            }
          ]
        }
      },
      "pooled_review": [
        {
          "lo": null,
          "hi": 0.5,
          "n": 29438,
          "coverage": 0.9224164943285079,
          "mean_score": 0.03286154702433298,
          "n_positive": 1078,
          "agreement": 0.036619335552687,
          "agreement_ci95": [
            0.03453348702025522,
            0.03882610435916809
          ],
          "recall_share": 0.3279586248859142,
          "meets_min_rows": true
        },
        {
          "lo": 0.5,
          "hi": 0.8,
          "n": 657,
          "coverage": 0.020586576424139875,
          "mean_score": 0.6513568559648688,
          "n_positive": 455,
          "agreement": 0.6925418569254186,
          "agreement_ci95": [
            0.6562232255335922,
            0.7266220039837725
          ],
          "recall_share": 0.13842409491937938,
          "meets_min_rows": true
        },
        {
          "lo": 0.8,
          "hi": 0.9,
          "n": 275,
          "coverage": 0.00861690794008899,
          "mean_score": 0.8566756572711491,
          "n_positive": 233,
          "agreement": 0.8472727272727273,
          "agreement_ci95": [
            0.799996287733429,
            0.8849807625455595
          ],
          "recall_share": 0.07088530574992394,
          "meets_min_rows": true
        },
        {
          "lo": 0.9,
          "hi": 0.95,
          "n": 223,
          "coverage": 0.0069875289841448895,
          "mean_score": 0.9249560823640465,
          "n_positive": 212,
          "agreement": 0.9506726457399103,
          "agreement_ci95": [
            0.9138451872318432,
            0.9722362239496174
          ],
          "recall_share": 0.06449650136902951,
          "meets_min_rows": true
        },
        {
          "lo": 0.95,
          "hi": 0.98,
          "n": 211,
          "coverage": 0.0066115184558500975,
          "mean_score": 0.9670367658257145,
          "n_positive": 203,
          "agreement": 0.9620853080568721,
          "agreement_ci95": [
            0.9269810200835585,
            0.9806650227956292
          ],
          "recall_share": 0.061758442348646185,
          "meets_min_rows": true
        },
        {
          "lo": 0.98,
          "hi": 0.99,
          "n": 154,
          "coverage": 0.004825468446449834,
          "mean_score": 0.9858709434856128,
          "n_positive": 154,
          "agreement": 1.0,
          "agreement_ci95": [
            0.9756625483030732,
            1.0
          ],
          "recall_share": 0.04685123212655917,
          "meets_min_rows": true
        },
        {
          "lo": 0.99,
          "hi": 0.995,
          "n": 99,
          "coverage": 0.003102086858432036,
          "mean_score": 0.9928239753810968,
          "n_positive": 98,
          "agreement": 0.98989898989899,
          "agreement_ci95": [
            0.9449846847785951,
            0.9982146927405587
          ],
          "recall_share": 0.029814420444174018,
          "meets_min_rows": true
        },
        {
          "lo": 0.995,
          "hi": 1.0,
          "n": 857,
          "coverage": 0.026853418562386412,
          "mean_score": 0.9992485950132229,
          "n_positive": 854,
          "agreement": 0.9964994165694282,
          "agreement_ci95": [
            0.989758842132123,
            0.9988087869245846
          ],
          "recall_share": 0.2598113781563736,
          "meets_min_rows": true
        }
      ],
      "high_risk_rules": {
        "R101_high_risk_priority": {
          "action": "priority_review",
          "signal": "p_threat",
          "threshold": 0.3,
          "n": 83,
          "coverage": 0.002600739487372313,
          "n_positive_label": 43,
          "agreement_label": 0.5180722891566265,
          "agreement_label_ci95": [
            0.41217757011683503,
            0.622368141558745
          ],
          "n_positive_any_label": 80,
          "agreement_any_label": 0.963855421686747
        },
        "skipped": [
          "R102_hierarchy_violation"
        ]
      },
      "strata": {
        "identity_term_present": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 27617,
              "coverage": 0.9266205878405583,
              "mean_score": 0.032072647783608924,
              "n_positive": 954,
              "agreement": 0.03454394032661042,
              "agreement_ci95": [
                0.03245401920344007,
                0.03676333109804962
              ],
              "recall_share": 0.3281733746130031,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 573,
              "coverage": 0.01922560730103342,
              "mean_score": 0.6510084570120317,
              "n_positive": 396,
              "agreement": 0.6910994764397905,
              "agreement_ci95": [
                0.652100339223797,
                0.7275533708696489
              ],
              "recall_share": 0.13622291021671826,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 245,
              "coverage": 0.008220373104281305,
              "mean_score": 0.8564108791191658,
              "n_positive": 206,
              "agreement": 0.8408163265306122,
              "agreement_ci95": [
                0.7897960294759789,
                0.8813140048035719
              ],
              "recall_share": 0.07086343309253526,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 205,
              "coverage": 0.0068782713729700715,
              "mean_score": 0.9247033128932717,
              "n_positive": 195,
              "agreement": 0.9512195121951219,
              "agreement_ci95": [
                0.9125488078809634,
                0.9732906274064173
              ],
              "recall_share": 0.06707946336429309,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 188,
              "coverage": 0.006307878137162797,
              "mean_score": 0.9671676141597154,
              "n_positive": 183,
              "agreement": 0.973404255319149,
              "agreement_ci95": [
                0.9392619377283395,
                0.9885875533982862
              ],
              "recall_share": 0.0629514963880289,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 139,
              "coverage": 0.004663803516306536,
              "mean_score": 0.9858828965557513,
              "n_positive": 139,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9731068352887076,
                1.0
              ],
              "recall_share": 0.0478156174750602,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 90,
              "coverage": 0.0030197288954502753,
              "mean_score": 0.9927290273500865,
              "n_positive": 89,
              "agreement": 0.9888888888888889,
              "agreement_ci95": [
                0.9397159162551686,
                0.9980359159207876
              ],
              "recall_share": 0.03061575507395941,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 747,
              "coverage": 0.025063749832237282,
              "mean_score": 0.9992607904644556,
              "n_positive": 745,
              "agreement": 0.9973226238286479,
              "agreement_ci95": [
                0.9902909790257544,
                0.9992654595691797
              ],
              "recall_share": 0.2562779497764018,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 1821,
              "coverage": 0.8630331753554502,
              "mean_score": 0.04482586900735119,
              "n_positive": 124,
              "agreement": 0.06809445359692477,
              "agreement_ci95": [
                0.057410090685756456,
                0.08059721819886598
              ],
              "recall_share": 0.3263157894736842,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 84,
              "coverage": 0.03981042654028436,
              "mean_score": 0.6537334345360074,
              "n_positive": 59,
              "agreement": 0.7023809523809523,
              "agreement_ci95": [
                0.5975091486436803,
                0.7895518181595427
              ],
              "recall_share": 0.15526315789473685,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 30,
              "coverage": 0.014218009478672985,
              "mean_score": 0.8588380121790126,
              "n_positive": 27,
              "agreement": 0.9,
              "agreement_ci95": [
                0.7437891726682571,
                0.9654001115225814
              ],
              "recall_share": 0.07105263157894737,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 18,
              "coverage": 0.008530805687203791,
              "mean_score": 0.9278348457812028,
              "n_positive": 17,
              "agreement": 0.9444444444444444,
              "agreement_ci95": [
                0.7424269900391908,
                0.990124809131223
              ],
              "recall_share": 0.04473684210526316,
              "meets_min_rows": false
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 23,
              "coverage": 0.010900473933649289,
              "mean_score": 0.9659672229217062,
              "n_positive": 20,
              "agreement": 0.8695652173913043,
              "agreement_ci95": [
                0.6787251755146014,
                0.9546234094181704
              ],
              "recall_share": 0.05263157894736842,
              "meets_min_rows": false
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 15,
              "coverage": 0.0071090047393364926,
              "mean_score": 0.9857601783689957,
              "n_positive": 15,
              "agreement": 1.0,
              "agreement_ci95": [
                0.7961166964035129,
                1.0
              ],
              "recall_share": 0.039473684210526314,
              "meets_min_rows": false
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 9,
              "coverage": 0.0042654028436018955,
              "mean_score": 0.9937734556911989,
              "n_positive": 9,
              "agreement": 1.0,
              "agreement_ci95": [
                0.700854948272956,
                1.0
              ],
              "recall_share": 0.02368421052631579,
              "meets_min_rows": false
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 110,
              "coverage": 0.052132701421800945,
              "mean_score": 0.9991657768125793,
              "n_positive": 109,
              "agreement": 0.990909090909091,
              "agreement_ci95": [
                0.950294347397897,
                0.9983934247134306
              ],
              "recall_share": 0.2868421052631579,
              "meets_min_rows": true
            }
          ]
        },
        "oov_tercile": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 10143,
              "coverage": 0.9415204678362573,
              "mean_score": 0.030138432429348792,
              "n_positive": 262,
              "agreement": 0.025830622103914028,
              "agreement_ci95": [
                0.022918412925849205,
                0.029101859690048273
              ],
              "recall_share": 0.3214723926380368,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 158,
              "coverage": 0.014666295368049753,
              "mean_score": 0.645649871378234,
              "n_positive": 99,
              "agreement": 0.6265822784810127,
              "agreement_ci95": [
                0.5489942914126521,
                0.6981611672472846
              ],
              "recall_share": 0.12147239263803682,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 72,
              "coverage": 0.006683375104427736,
              "mean_score": 0.8577291325137248,
              "n_positive": 66,
              "agreement": 0.9166666666666666,
              "agreement_ci95": [
                0.8298764469345035,
                0.9612475752696261
              ],
              "recall_share": 0.08098159509202454,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 53,
              "coverage": 0.004919706674092639,
              "mean_score": 0.9282910601049176,
              "n_positive": 47,
              "agreement": 0.8867924528301887,
              "agreement_ci95": [
                0.774234807801829,
                0.9470696908322002
              ],
              "recall_share": 0.05766871165644172,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 60,
              "coverage": 0.00556947925368978,
              "mean_score": 0.9666233738988849,
              "n_positive": 56,
              "agreement": 0.9333333333333333,
              "agreement_ci95": [
                0.8407464259125857,
                0.9737713014577508
              ],
              "recall_share": 0.0687116564417178,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 43,
              "coverage": 0.003991460131811009,
              "mean_score": 0.9863446736282847,
              "n_positive": 43,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9179901955865446,
                1.0
              ],
              "recall_share": 0.05276073619631902,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 26,
              "coverage": 0.002413441009932238,
              "mean_score": 0.9928393850876562,
              "n_positive": 26,
              "agreement": 1.0,
              "agreement_ci95": [
                0.871271076371412,
                1.0
              ],
              "recall_share": 0.03190184049079755,
              "meets_min_rows": false
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 218,
              "coverage": 0.020235774621739533,
              "mean_score": 0.9992655775508068,
              "n_positive": 216,
              "agreement": 0.9908256880733946,
              "agreement_ci95": [
                0.9671724133360691,
                0.9974804566872397
              ],
              "recall_share": 0.2650306748466258,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 9845,
              "coverage": 0.9293873312564901,
              "mean_score": 0.032116270600293054,
              "n_positive": 359,
              "agreement": 0.036465210766886746,
              "agreement_ci95": [
                0.032939663057280584,
                0.04035235427236867
              ],
              "recall_share": 0.3526522593320236,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 223,
              "coverage": 0.02105163787406778,
              "mean_score": 0.643727483987233,
              "n_positive": 158,
              "agreement": 0.7085201793721974,
              "agreement_ci95": [
                0.6457455689938787,
                0.7642323973736629
              ],
              "recall_share": 0.15520628683693516,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 88,
              "coverage": 0.008307372793354102,
              "mean_score": 0.8522433603051657,
              "n_positive": 66,
              "agreement": 0.75,
              "agreement_ci95": [
                0.6503696995985793,
                0.8287167667528479
              ],
              "recall_share": 0.06483300589390963,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 73,
              "coverage": 0.006891343339941471,
              "mean_score": 0.9241968143534026,
              "n_positive": 72,
              "agreement": 0.9863013698630136,
              "agreement_ci95": [
                0.9264026265348793,
                0.997577748560291
              ],
              "recall_share": 0.07072691552062868,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 56,
              "coverage": 0.005286509959407155,
              "mean_score": 0.9673585434126777,
              "n_positive": 55,
              "agreement": 0.9821428571428571,
              "agreement_ci95": [
                0.9055436275572404,
                0.9968407896929579
              ],
              "recall_share": 0.054027504911591355,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 47,
              "coverage": 0.004436892287359577,
              "mean_score": 0.9861914892010092,
              "n_positive": 47,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9244423947340892,
                1.0
              ],
              "recall_share": 0.04616895874263261,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 29,
              "coverage": 0.00273765694326442,
              "mean_score": 0.9923306050635915,
              "n_positive": 29,
              "agreement": 1.0,
              "agreement_ci95": [
                0.8830301998708162,
                0.9999999999999999
              ],
              "recall_share": 0.02848722986247544,
              "meets_min_rows": false
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 232,
              "coverage": 0.02190125554611536,
              "mean_score": 0.9992740206113012,
              "n_positive": 232,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9837116896260826,
                0.9999999999999999
              ],
              "recall_share": 0.22789783889980353,
              "meets_min_rows": true
            }
          ],
          "2": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 9450,
              "coverage": 0.8959044368600683,
              "mean_score": 0.03656078487952851,
              "n_positive": 457,
              "agreement": 0.04835978835978836,
              "agreement_ci95": [
                0.04421504795588572,
                0.05287156629223468
              ],
              "recall_share": 0.3143053645116919,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 276,
              "coverage": 0.026166097838452786,
              "mean_score": 0.6607882092826265,
              "n_positive": 198,
              "agreement": 0.717391304347826,
              "agreement_ci95": [
                0.6615678702622743,
                0.7672463607659521
              ],
              "recall_share": 0.1361760660247593,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 115,
              "coverage": 0.010902540766021995,
              "mean_score": 0.8594077608845498,
              "n_positive": 101,
              "agreement": 0.8782608695652174,
              "agreement_ci95": [
                0.8059875231909075,
                0.9260802317741286
              ],
              "recall_share": 0.06946354883081156,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 97,
              "coverage": 0.00919605612438377,
              "mean_score": 0.9237052859157042,
              "n_positive": 93,
              "agreement": 0.9587628865979382,
              "agreement_ci95": [
                0.8987247433166863,
                0.9838487635260202
              ],
              "recall_share": 0.0639614855570839,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 95,
              "coverage": 0.0090064467197573,
              "mean_score": 0.9671081760440285,
              "n_positive": 92,
              "agreement": 0.968421052631579,
              "agreement_ci95": [
                0.9112290128908604,
                0.9892028606095589
              ],
              "recall_share": 0.06327372764786796,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 64,
              "coverage": 0.006067500948047023,
              "mean_score": 0.9853172552862606,
              "n_positive": 64,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9433759393644894,
                1.0
              ],
              "recall_share": 0.04401650618982118,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 44,
              "coverage": 0.004171406901782328,
              "mean_score": 0.9931400455364855,
              "n_positive": 43,
              "agreement": 0.9772727272727273,
              "agreement_ci95": [
                0.8819229018613075,
                0.9959767479844506
              ],
              "recall_share": 0.029573590096286108,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 407,
              "coverage": 0.038585513841486535,
              "mean_score": 0.9992250054998386,
              "n_positive": 406,
              "agreement": 0.9975429975429976,
              "agreement_ci95": [
                0.9862155730475362,
                0.9995661469930801
              ],
              "recall_share": 0.2792297111416781,
              "meets_min_rows": true
            }
          ]
        }
      }
    },
    "calibration_note": null,
    "threshold_selection": {
      "n_rows": 31914,
      "min_rows": 30,
      "edges": [
        0.5,
        0.8,
        0.9,
        0.95,
        0.98,
        0.99,
        0.995,
        1.0
      ],
      "per_label": {
        "toxic": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 29486,
              "coverage": 0.923920536441687,
              "mean_score": 0.03189358907391137,
              "n_positive": 930,
              "agreement": 0.031540392050464626,
              "agreement_ci95": [
                0.02960574273865421,
                0.03359708802190156
              ],
              "recall_share": 0.30958721704394143,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 686,
              "coverage": 0.021495268534185624,
              "mean_score": 0.6495966429956931,
              "n_positive": 448,
              "agreement": 0.6530612244897959,
              "agreement_ci95": [
                0.6166783454227336,
                0.6877394266557282
              ],
              "recall_share": 0.14913448735019974,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 259,
              "coverage": 0.008115560569029266,
              "mean_score": 0.8528379994475179,
              "n_positive": 193,
              "agreement": 0.7451737451737451,
              "agreement_ci95": [
                0.6887880216484941,
                0.794392982370189
              ],
              "recall_share": 0.06424766977363515,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 207,
              "coverage": 0.0064861816130851666,
              "mean_score": 0.925965790319176,
              "n_positive": 186,
              "agreement": 0.8985507246376812,
              "agreement_ci95": [
                0.8498938114090119,
                0.932684723550259
              ],
              "recall_share": 0.061917443408788284,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 237,
              "coverage": 0.007426207933822147,
              "mean_score": 0.966277813337436,
              "n_positive": 222,
              "agreement": 0.9367088607594937,
              "agreement_ci95": [
                0.8982134566470712,
                0.9612731158123011
              ],
              "recall_share": 0.07390146471371505,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 129,
              "coverage": 0.004042113179169016,
              "mean_score": 0.9854145195195343,
              "n_positive": 124,
              "agreement": 0.9612403100775194,
              "agreement_ci95": [
                0.9124721788450647,
                0.9833324969100818
              ],
              "recall_share": 0.041278295605858856,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 109,
              "coverage": 0.003415428965344363,
              "mean_score": 0.9925884195207701,
              "n_positive": 105,
              "agreement": 0.963302752293578,
              "agreement_ci95": [
                0.9094227987018519,
                0.9856382985341146
              ],
              "recall_share": 0.03495339547270306,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 801,
              "coverage": 0.025098702763677382,
              "mean_score": 0.9992837748679527,
              "n_positive": 796,
              "agreement": 0.9937578027465668,
              "agreement_ci95": [
                0.9854714081014656,
                0.997330846021309
              ],
              "recall_share": 0.26498002663115844,
              "meets_min_rows": true
            }
          ]
        },
        "severe_toxic": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31419,
              "coverage": 0.9844895657078399,
              "mean_score": 0.0030624196957179527,
              "n_positive": 114,
              "agreement": 0.003628377733218753,
              "agreement_ci95": [
                0.0030214937660318895,
                0.004356625083744146
              ],
              "recall_share": 0.36893203883495146,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 139,
              "coverage": 0.004355455286081344,
              "mean_score": 0.14321660215510276,
              "n_positive": 42,
              "agreement": 0.302158273381295,
              "agreement_ci95": [
                0.23198737315921314,
                0.3829703538848236
              ],
              "recall_share": 0.13592233009708737,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 98,
              "coverage": 0.003070752647740803,
              "mean_score": 0.24412219327821466,
              "n_positive": 24,
              "agreement": 0.24489795918367346,
              "agreement_ci95": [
                0.1704497011087401,
                0.33859111046998225
              ],
              "recall_share": 0.07766990291262135,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 114,
              "coverage": 0.0035721000188005266,
              "mean_score": 0.39390733280098345,
              "n_positive": 38,
              "agreement": 0.3333333333333333,
              "agreement_ci95": [
                0.25348080073665114,
                0.4240520441487966
              ],
              "recall_share": 0.12297734627831715,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 86,
              "coverage": 0.002694742119446011,
              "mean_score": 0.6361501504362916,
              "n_positive": 46,
              "agreement": 0.5348837209302325,
              "agreement_ci95": [
                0.43024302641623724,
                0.6365412853559876
              ],
              "recall_share": 0.1488673139158576,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 31,
              "coverage": 0.0009713605314282133,
              "mean_score": 0.8611606133409082,
              "n_positive": 23,
              "agreement": 0.7419354838709677,
              "agreement_ci95": [
                0.5675386404606986,
                0.8629829416321935
              ],
              "recall_share": 0.0744336569579288,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 27,
              "coverage": 0.0008460236886632825,
              "mean_score": 0.9494328322216928,
              "n_positive": 22,
              "agreement": 0.8148148148148148,
              "agreement_ci95": [
                0.6330131622138684,
                0.9181929290877906
              ],
              "recall_share": 0.07119741100323625,
              "meets_min_rows": false
            }
          ]
        },
        "obscene": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 30583,
              "coverage": 0.9582941655699693,
              "mean_score": 0.014127381815005977,
              "n_positive": 487,
              "agreement": 0.015923879279338193,
              "agreement_ci95": [
                0.01458048167983749,
                0.017388868937676716
              ],
              "recall_share": 0.2965895249695493,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 298,
              "coverage": 0.00933759478598734,
              "mean_score": 0.6560930248620364,
              "n_positive": 201,
              "agreement": 0.674496644295302,
              "agreement_ci95": [
                0.6193693475979859,
                0.725182392860238
              ],
              "recall_share": 0.12241169305724726,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 119,
              "coverage": 0.00372877107225669,
              "mean_score": 0.8473188051886439,
              "n_positive": 92,
              "agreement": 0.773109243697479,
              "agreement_ci95": [
                0.6900143224421211,
                0.8391229948117445
              ],
              "recall_share": 0.05602923264311815,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 99,
              "coverage": 0.003102086858432036,
              "mean_score": 0.9240044077716829,
              "n_positive": 81,
              "agreement": 0.8181818181818182,
              "agreement_ci95": [
                0.7308120564761821,
                0.8817813536651241
              ],
              "recall_share": 0.04933008526187576,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 107,
              "coverage": 0.0033527605439618974,
              "mean_score": 0.9666961718462528,
              "n_positive": 90,
              "agreement": 0.8411214953271028,
              "agreement_ci95": [
                0.7602251006709314,
                0.8983732319094638
              ],
              "recall_share": 0.05481120584652863,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 78,
              "coverage": 0.0024440684339161497,
              "mean_score": 0.9858843401621217,
              "n_positive": 72,
              "agreement": 0.9230769230769231,
              "agreement_ci95": [
                0.8421677011320987,
                0.9642695362065872
              ],
              "recall_share": 0.0438489646772229,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 58,
              "coverage": 0.0018173842200914958,
              "mean_score": 0.992453096512027,
              "n_positive": 55,
              "agreement": 0.9482758620689655,
              "agreement_ci95": [
                0.8586059871294194,
                0.9822538654787383
              ],
              "recall_share": 0.03349573690621194,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 572,
              "coverage": 0.0179231685153851,
              "mean_score": 0.9992528133319842,
              "n_positive": 564,
              "agreement": 0.986013986013986,
              "agreement_ci95": [
                0.9726471334039779,
                0.9928964054029665
              ],
              "recall_share": 0.34348355663824603,
              "meets_min_rows": true
            }
          ]
        },
        "threat": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31788,
              "coverage": 0.9960518894529047,
              "mean_score": 0.0012727232269281348,
              "n_positive": 36,
              "agreement": 0.0011325028312570782,
              "agreement_ci95": [
                0.0008181877383146396,
                0.0015673758340058659
              ],
              "recall_share": 0.45569620253164556,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 39,
              "coverage": 0.0012220342169580749,
              "mean_score": 0.13915250531227225,
              "n_positive": 5,
              "agreement": 0.1282051282051282,
              "agreement_ci95": [
                0.05602817501519075,
                0.2670574351095147
              ],
              "recall_share": 0.06329113924050633,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 20,
              "coverage": 0.0006266842138246538,
              "mean_score": 0.25005262452301047,
              "n_positive": 4,
              "agreement": 0.2,
              "agreement_ci95": [
                0.08065766201776073,
                0.41601743409331715
              ],
              "recall_share": 0.05063291139240506,
              "meets_min_rows": false
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 16,
              "coverage": 0.000501347371059723,
              "mean_score": 0.3884042356938948,
              "n_positive": 3,
              "agreement": 0.1875,
              "agreement_ci95": [
                0.06591599019788683,
                0.4300888116754912
              ],
              "recall_share": 0.0379746835443038,
              "meets_min_rows": false
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 21,
              "coverage": 0.0006580184245158864,
              "mean_score": 0.6054920261359209,
              "n_positive": 9,
              "agreement": 0.42857142857142855,
              "agreement_ci95": [
                0.2446996880971642,
                0.6345344577581422
              ],
              "recall_share": 0.11392405063291139,
              "meets_min_rows": false
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 9,
              "coverage": 0.0002820078962210942,
              "mean_score": 0.8518085167583569,
              "n_positive": 6,
              "agreement": 0.6666666666666666,
              "agreement_ci95": [
                0.35420213352505864,
                0.8794161825659264
              ],
              "recall_share": 0.0759493670886076,
              "meets_min_rows": false
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 21,
              "coverage": 0.0006580184245158864,
              "mean_score": 0.9693889974373097,
              "n_positive": 16,
              "agreement": 0.7619047619047619,
              "agreement_ci95": [
                0.5490882614178733,
                0.8937198704460038
              ],
              "recall_share": 0.20253164556962025,
              "meets_min_rows": false
            }
          ]
        },
        "insult": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 30811,
              "coverage": 0.9654383656075703,
              "mean_score": 0.01932144363611822,
              "n_positive": 668,
              "agreement": 0.02168056862808737,
              "agreement_ci95": [
                0.020113017178765076,
                0.02336737718474815
              ],
              "recall_share": 0.43292287751134156,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 391,
              "coverage": 0.01225167638027198,
              "mean_score": 0.6544156847404933,
              "n_positive": 252,
              "agreement": 0.6445012787723785,
              "agreement_ci95": [
                0.5958608261471938,
                0.6903299916057413
              ],
              "recall_share": 0.16331821127673363,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 150,
              "coverage": 0.004700131603684903,
              "mean_score": 0.8563264621173632,
              "n_positive": 118,
              "agreement": 0.7866666666666666,
              "agreement_ci95": [
                0.7143794204732893,
                0.8446376392021363
              ],
              "recall_share": 0.07647440051847051,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 113,
              "coverage": 0.003540765808109294,
              "mean_score": 0.9254093516967694,
              "n_positive": 96,
              "agreement": 0.8495575221238938,
              "agreement_ci95": [
                0.7722307333195981,
                0.9038991332225452
              ],
              "recall_share": 0.06221646143875567,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 121,
              "coverage": 0.0037914394936391553,
              "mean_score": 0.9666898541631355,
              "n_positive": 101,
              "agreement": 0.8347107438016529,
              "agreement_ci95": [
                0.7584457947577176,
                0.8903771261360368
              ],
              "recall_share": 0.06545690213869086,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 62,
              "coverage": 0.0019427210628564266,
              "mean_score": 0.9844975170173075,
              "n_positive": 55,
              "agreement": 0.8870967741935484,
              "agreement_ci95": [
                0.7848025784984924,
                0.944221365010081
              ],
              "recall_share": 0.0356448476992871,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 57,
              "coverage": 0.0017860500094002633,
              "mean_score": 0.99283563373413,
              "n_positive": 52,
              "agreement": 0.9122807017543859,
              "agreement_ci95": [
                0.8105512279941347,
                0.9619483284628987
              ],
              "recall_share": 0.033700583279325985,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 209,
              "coverage": 0.006548850034467632,
              "mean_score": 0.9986764092388233,
              "n_positive": 201,
              "agreement": 0.9617224880382775,
              "agreement_ci95": [
                0.9262994276485942,
                0.9804787959851679
              ],
              "recall_share": 0.13026571613739468,
              "meets_min_rows": true
            }
          ]
        },
        "identity_hate": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 31501,
              "coverage": 0.9870589709845209,
              "mean_score": 0.0034654036766833267,
              "n_positive": 126,
              "agreement": 0.00399987301990413,
              "agreement_ci95": [
                0.0033607653768383174,
                0.004759937570205831
              ],
              "recall_share": 0.4666666666666667,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 155,
              "coverage": 0.004856802657141067,
              "mean_score": 0.14004000810243253,
              "n_positive": 24,
              "agreement": 0.15483870967741936,
              "agreement_ci95": [
                0.10631334822514471,
                0.2200589933940399
              ],
              "recall_share": 0.08888888888888889,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 63,
              "coverage": 0.0019740552735476595,
              "mean_score": 0.2475079586235288,
              "n_positive": 22,
              "agreement": 0.3492063492063492,
              "agreement_ci95": [
                0.24325988942419746,
                0.47248539324920774
              ],
              "recall_share": 0.08148148148148149,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 82,
              "coverage": 0.0025694052766810803,
              "mean_score": 0.39707167383215225,
              "n_positive": 26,
              "agreement": 0.3170731707317073,
              "agreement_ci95": [
                0.22648065607799273,
                0.4240378650309642
              ],
              "recall_share": 0.0962962962962963,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 64,
              "coverage": 0.002005389484238892,
              "mean_score": 0.6442202056271501,
              "n_positive": 35,
              "agreement": 0.546875,
              "agreement_ci95": [
                0.42573597127892454,
                0.6627055230364962
              ],
              "recall_share": 0.12962962962962962,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 21,
              "coverage": 0.0006580184245158864,
              "mean_score": 0.8511369016597493,
              "n_positive": 17,
              "agreement": 0.8095238095238095,
              "agreement_ci95": [
                0.5999943512871267,
                0.9233243500065463
              ],
              "recall_share": 0.06296296296296296,
              "meets_min_rows": false
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 28,
              "coverage": 0.0008773578993545153,
              "mean_score": 0.9691878904091461,
              "n_positive": 20,
              "agreement": 0.7142857142857143,
              "agreement_ci95": [
                0.5294071141762103,
                0.8474600402068736
              ],
              "recall_share": 0.07407407407407407,
              "meets_min_rows": false
            }
          ]
        }
      },
      "pooled_review": [
        {
          "lo": null,
          "hi": 0.5,
          "n": 29473,
          "coverage": 0.923513191702701,
          "mean_score": 0.032805186517590776,
          "n_positive": 1034,
          "agreement": 0.03508295728293693,
          "agreement_ci95": [
            0.03304227845511014,
            0.03724481325284677
          ],
          "recall_share": 0.32231920199501246,
          "meets_min_rows": true
        },
        {
          "lo": 0.5,
          "hi": 0.8,
          "n": 692,
          "coverage": 0.02168327379833302,
          "mean_score": 0.6496861979685267,
          "n_positive": 495,
          "agreement": 0.7153179190751445,
          "agreement_ci95": [
            0.6805790205156581,
            0.7476794515781067
          ],
          "recall_share": 0.15430174563591023,
          "meets_min_rows": true
        },
        {
          "lo": 0.8,
          "hi": 0.9,
          "n": 250,
          "coverage": 0.007833552672808173,
          "mean_score": 0.8522702795757134,
          "n_positive": 208,
          "agreement": 0.832,
          "agreement_ci95": [
            0.7807100417867405,
            0.8732414471921603
          ],
          "recall_share": 0.06483790523690773,
          "meets_min_rows": true
        },
        {
          "lo": 0.9,
          "hi": 0.95,
          "n": 203,
          "coverage": 0.006360844770320236,
          "mean_score": 0.9250460763709286,
          "n_positive": 187,
          "agreement": 0.9211822660098522,
          "agreement_ci95": [
            0.8758152195321754,
            0.9509049212213038
          ],
          "recall_share": 0.05829177057356608,
          "meets_min_rows": true
        },
        {
          "lo": 0.95,
          "hi": 0.98,
          "n": 242,
          "coverage": 0.007582878987278311,
          "mean_score": 0.9660591440771737,
          "n_positive": 233,
          "agreement": 0.9628099173553719,
          "agreement_ci95": [
            0.9308433797538106,
            0.9803129444045077
          ],
          "recall_share": 0.07263092269326683,
          "meets_min_rows": true
        },
        {
          "lo": 0.98,
          "hi": 0.99,
          "n": 134,
          "coverage": 0.00419878423262518,
          "mean_score": 0.9855042172490147,
          "n_positive": 132,
          "agreement": 0.9850746268656716,
          "agreement_ci95": [
            0.9472151433721099,
            0.9958973356676775
          ],
          "recall_share": 0.04114713216957606,
          "meets_min_rows": true
        },
        {
          "lo": 0.99,
          "hi": 0.995,
          "n": 110,
          "coverage": 0.0034467631760355956,
          "mean_score": 0.992563046501903,
          "n_positive": 109,
          "agreement": 0.990909090909091,
          "agreement_ci95": [
            0.950294347397897,
            0.9983934247134306
          ],
          "recall_share": 0.033977556109725686,
          "meets_min_rows": true
        },
        {
          "lo": 0.995,
          "hi": 1.0,
          "n": 810,
          "coverage": 0.025380710659898477,
          "mean_score": 0.9993208182338852,
          "n_positive": 810,
          "agreement": 1.0,
          "agreement_ci95": [
            0.9952798437098835,
            0.9999999999999998
          ],
          "recall_share": 0.2524937655860349,
          "meets_min_rows": true
        }
      ],
      "high_risk_rules": {
        "R101_high_risk_priority": {
          "action": "priority_review",
          "signal": "p_threat",
          "threshold": 0.3,
          "n": 67,
          "coverage": 0.00209939211631259,
          "n_positive_label": 34,
          "agreement_label": 0.5074626865671642,
          "agreement_label_ci95": [
            0.3906377456065883,
            0.6234782821516152
          ],
          "n_positive_any_label": 65,
          "agreement_any_label": 0.9701492537313433
        },
        "skipped": [
          "R102_hierarchy_violation"
        ]
      },
      "priority_band": {
        "n": 1060,
        "table": [
          {
            "lo": null,
            "hi": 0.5,
            "n": 1,
            "coverage": 0.0009433962264150943,
            "mean_score": 0.48391723714328905,
            "n_positive": 0,
            "agreement": 0.0,
            "agreement_ci95": [
              0.0,
              0.7934506882081973
            ],
            "recall_share": 0.0,
            "meets_min_rows": false
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "n": 4,
            "coverage": 0.0037735849056603774,
            "mean_score": 0.6565032297432346,
            "n_positive": 3,
            "agreement": 0.75,
            "agreement_ci95": [
              0.3006418400969081,
              0.954412739704631
            ],
            "recall_share": 0.002843601895734597,
            "meets_min_rows": false
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 4,
            "coverage": 0.0037735849056603774,
            "mean_score": 0.8635907996042613,
            "n_positive": 4,
            "agreement": 1.0,
            "agreement_ci95": [
              0.5101091596030786,
              0.9999999999999999
            ],
            "recall_share": 0.0037914691943127963,
            "meets_min_rows": false
          },
          {
            "lo": 0.9,
            "hi": 0.95,
            "n": 5,
            "coverage": 0.0047169811320754715,
            "mean_score": 0.9153416209358802,
            "n_positive": 5,
            "agreement": 1.0,
            "agreement_ci95": [
              0.5655175313406071,
              1.0
            ],
            "recall_share": 0.004739336492890996,
            "meets_min_rows": false
          },
          {
            "lo": 0.95,
            "hi": 0.98,
            "n": 4,
            "coverage": 0.0037735849056603774,
            "mean_score": 0.9563935445420035,
            "n_positive": 4,
            "agreement": 1.0,
            "agreement_ci95": [
              0.5101091596030786,
              0.9999999999999999
            ],
            "recall_share": 0.0037914691943127963,
            "meets_min_rows": false
          },
          {
            "lo": 0.98,
            "hi": 0.99,
            "n": 124,
            "coverage": 0.1169811320754717,
            "mean_score": 0.9855395405975591,
            "n_positive": 122,
            "agreement": 0.9838709677419355,
            "agreement_ci95": [
              0.9430969942911672,
              0.9955656355793431
            ],
            "recall_share": 0.11563981042654028,
            "meets_min_rows": true
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "n": 108,
            "coverage": 0.1018867924528302,
            "mean_score": 0.9925975636159229,
            "n_positive": 107,
            "agreement": 0.9907407407407407,
            "agreement_ci95": [
              0.9494065442747008,
              0.9983636391641979
            ],
            "recall_share": 0.1014218009478673,
            "meets_min_rows": true
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "n": 810,
            "coverage": 0.7641509433962265,
            "mean_score": 0.9993208182338852,
            "n_positive": 810,
            "agreement": 1.0,
            "agreement_ci95": [
              0.9952798437098835,
              0.9999999999999998
            ],
            "recall_share": 0.7677725118483413,
            "meets_min_rows": true
          }
        ],
        "n_rows_matching_priority_rule": 67,
        "lowest_segment_agreement": 0.0,
        "note": "pooled p_max against any positive label on the final priority band, rule promotions included; the leading bucket holds rows a rule promoted from below the first edge and is included in the lowest segment agreement. The priority-rule match count includes rows already in the model's priority band"
      },
      "strata": {
        "identity_term_present": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 27692,
              "coverage": 0.9268047792764149,
              "mean_score": 0.03161038549197453,
              "n_positive": 910,
              "agreement": 0.03286147623862487,
              "agreement_ci95": [
                0.030825707675047793,
                0.0350268305693333
              ],
              "recall_share": 0.31862745098039214,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 614,
              "coverage": 0.020549549851065965,
              "mean_score": 0.6499594553018077,
              "n_positive": 437,
              "agreement": 0.7117263843648208,
              "agreement_ci95": [
                0.6746692419068512,
                0.7461506889387948
              ],
              "recall_share": 0.15301120448179273,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 221,
              "coverage": 0.007396499213494428,
              "mean_score": 0.8517625712621989,
              "n_positive": 181,
              "agreement": 0.8190045248868778,
              "agreement_ci95": [
                0.7629346386210092,
                0.8641739057797826
              ],
              "recall_share": 0.06337535014005602,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 188,
              "coverage": 0.006292044579805214,
              "mean_score": 0.9249881101913193,
              "n_positive": 175,
              "agreement": 0.9308510638297872,
              "agreement_ci95": [
                0.8853003060159768,
                0.9591469836610647
              ],
              "recall_share": 0.061274509803921566,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 219,
              "coverage": 0.007329562569028414,
              "mean_score": 0.9660467494184594,
              "n_positive": 211,
              "agreement": 0.9634703196347032,
              "agreement_ci95": [
                0.9295858432412984,
                0.9813757032280899
              ],
              "recall_share": 0.07387955182072829,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 126,
              "coverage": 0.004217008601358814,
              "mean_score": 0.9855196768679926,
              "n_positive": 124,
              "agreement": 0.9841269841269841,
              "agreement_ci95": [
                0.9439712357372746,
                0.9956361982144274
              ],
              "recall_share": 0.04341736694677871,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 92,
              "coverage": 0.0030790856454365944,
              "mean_score": 0.992547783429279,
              "n_positive": 91,
              "agreement": 0.9891304347826086,
              "agreement_ci95": [
                0.9409721432543667,
                0.9980786709811413
              ],
              "recall_share": 0.031862745098039214,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 727,
              "coverage": 0.024331470263395695,
              "mean_score": 0.9993478666202082,
              "n_positive": 727,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9947437863101304,
                0.9999999999999999
              ],
              "recall_share": 0.2545518207282913,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 1781,
              "coverage": 0.8751842751842752,
              "mean_score": 0.05138263177383169,
              "n_positive": 124,
              "agreement": 0.06962380685008422,
              "agreement_ci95": [
                0.058706355669458865,
                0.08239382867920889
              ],
              "recall_share": 0.3522727272727273,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 78,
              "coverage": 0.03832923832923833,
              "mean_score": 0.6475351722937254,
              "n_positive": 58,
              "agreement": 0.7435897435897436,
              "agreement_ci95": [
                0.6368667262894674,
                0.8274456224812912
              ],
              "recall_share": 0.16477272727272727,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 29,
              "coverage": 0.01425061425061425,
              "mean_score": 0.8561393670683578,
              "n_positive": 27,
              "agreement": 0.9310344827586207,
              "agreement_ci95": [
                0.7803541969199559,
                0.9808787340031615
              ],
              "recall_share": 0.07670454545454546,
              "meets_min_rows": false
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 15,
              "coverage": 0.007371007371007371,
              "mean_score": 0.9257725858220328,
              "n_positive": 12,
              "agreement": 0.8,
              "agreement_ci95": [
                0.5481455107594471,
                0.9295245070826608
              ],
              "recall_share": 0.03409090909090909,
              "meets_min_rows": false
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 23,
              "coverage": 0.011302211302211302,
              "mean_score": 0.9661771627840633,
              "n_positive": 22,
              "agreement": 0.9565217391304348,
              "agreement_ci95": [
                0.7900884474454931,
                0.9922833339420488
              ],
              "recall_share": 0.0625,
              "meets_min_rows": false
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 8,
              "coverage": 0.003931203931203931,
              "mean_score": 0.9852607282501118,
              "n_positive": 8,
              "agreement": 1.0,
              "agreement_ci95": [
                0.6755924316585924,
                1.0
              ],
              "recall_share": 0.022727272727272728,
              "meets_min_rows": false
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 18,
              "coverage": 0.008845208845208846,
              "mean_score": 0.9926410577619814,
              "n_positive": 18,
              "agreement": 1.0,
              "agreement_ci95": [
                0.8241207740667156,
                1.0
              ],
              "recall_share": 0.05113636363636364,
              "meets_min_rows": false
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 83,
              "coverage": 0.04078624078624079,
              "mean_score": 0.9990839004404286,
              "n_positive": 83,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9557646896910507,
                1.0
              ],
              "recall_share": 0.23579545454545456,
              "meets_min_rows": true
            }
          ]
        },
        "oov_tercile": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 10033,
              "coverage": 0.9431284075954127,
              "mean_score": 0.026919416096703286,
              "n_positive": 203,
              "agreement": 0.0202332303398784,
              "agreement_ci95": [
                0.0176562358200271,
                0.02317747272865578
              ],
              "recall_share": 0.27884615384615385,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 177,
              "coverage": 0.016638465877044557,
              "mean_score": 0.6465515089604685,
              "n_positive": 115,
              "agreement": 0.6497175141242938,
              "agreement_ci95": [
                0.5769347598602169,
                0.7161396287665033
              ],
              "recall_share": 0.15796703296703296,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 63,
              "coverage": 0.005922165820642978,
              "mean_score": 0.8580740831172334,
              "n_positive": 54,
              "agreement": 0.8571428571428571,
              "agreement_ci95": [
                0.7502543398002094,
                0.9229805170259359
              ],
              "recall_share": 0.07417582417582418,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 44,
              "coverage": 0.004136115811242715,
              "mean_score": 0.9253200776505269,
              "n_positive": 39,
              "agreement": 0.8863636363636364,
              "agreement_ci95": [
                0.7602084746202706,
                0.9504721943024861
              ],
              "recall_share": 0.05357142857142857,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 68,
              "coverage": 0.006392178981011468,
              "mean_score": 0.9658083731366789,
              "n_positive": 64,
              "agreement": 0.9411764705882353,
              "agreement_ci95": [
                0.8582831148039058,
                0.9768892346209339
              ],
              "recall_share": 0.08791208791208792,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 30,
              "coverage": 0.0028200789622109417,
              "mean_score": 0.9859301287991713,
              "n_positive": 30,
              "agreement": 1.0,
              "agreement_ci95": [
                0.886486605238548,
                1.0
              ],
              "recall_share": 0.04120879120879121,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 20,
              "coverage": 0.0018800526414739613,
              "mean_score": 0.9923110550094705,
              "n_positive": 20,
              "agreement": 1.0,
              "agreement_ci95": [
                0.8388748398148705,
                1.0
              ],
              "recall_share": 0.027472527472527472,
              "meets_min_rows": false
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 203,
              "coverage": 0.019082534310960707,
              "mean_score": 0.9994167304772066,
              "n_positive": 203,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9814280033506216,
                1.0
              ],
              "recall_share": 0.27884615384615385,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 9820,
              "coverage": 0.9263277049334968,
              "mean_score": 0.03340972776304984,
              "n_positive": 371,
              "agreement": 0.03778004073319756,
              "agreement_ci95": [
                0.03418615556766655,
                0.041735413606988056
              ],
              "recall_share": 0.35,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 245,
              "coverage": 0.023111027261579098,
              "mean_score": 0.652155960926853,
              "n_positive": 173,
              "agreement": 0.7061224489795919,
              "agreement_ci95": [
                0.6462519306344646,
                0.7596289883010722
              ],
              "recall_share": 0.1632075471698113,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 84,
              "coverage": 0.007923780775398548,
              "mean_score": 0.8457849911824658,
              "n_positive": 71,
              "agreement": 0.8452380952380952,
              "agreement_ci95": [
                0.7530135770583347,
                0.9072668957236342
              ],
              "recall_share": 0.0669811320754717,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 72,
              "coverage": 0.006791812093198755,
              "mean_score": 0.9250344482617633,
              "n_positive": 68,
              "agreement": 0.9444444444444444,
              "agreement_ci95": [
                0.8656798395141685,
                0.9781857841702364
              ],
              "recall_share": 0.06415094339622641,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 77,
              "coverage": 0.007263465710782002,
              "mean_score": 0.9648434778399312,
              "n_positive": 75,
              "agreement": 0.974025974025974,
              "agreement_ci95": [
                0.9101541714611711,
                0.9928478402564792
              ],
              "recall_share": 0.07075471698113207,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 35,
              "coverage": 0.003301575323082728,
              "mean_score": 0.9856311940244716,
              "n_positive": 34,
              "agreement": 0.9714285714285714,
              "agreement_ci95": [
                0.8546691262822592,
                0.9949385082005704
              ],
              "recall_share": 0.03207547169811321,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 32,
              "coverage": 0.00301858315253278,
              "mean_score": 0.9923524991754538,
              "n_positive": 32,
              "agreement": 1.0,
              "agreement_ci95": [
                0.8928208002353198,
                1.0
              ],
              "recall_share": 0.03018867924528302,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 236,
              "coverage": 0.022262050749929253,
              "mean_score": 0.9992825256001637,
              "n_positive": 236,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9839833409152283,
                1.0
              ],
              "recall_share": 0.22264150943396227,
              "meets_min_rows": true
            }
          ],
          "2": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 9620,
              "coverage": 0.9011709601873537,
              "mean_score": 0.03832653159060077,
              "n_positive": 460,
              "agreement": 0.04781704781704782,
              "agreement_ci95": [
                0.04373062298385758,
                0.05226445993895338
              ],
              "recall_share": 0.323943661971831,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 270,
              "coverage": 0.02529274004683841,
              "mean_score": 0.6495000795597725,
              "n_positive": 207,
              "agreement": 0.7666666666666667,
              "agreement_ci95": [
                0.7126918624330072,
                0.8131598496790757
              ],
              "recall_share": 0.14577464788732394,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 103,
              "coverage": 0.009648711943793912,
              "mean_score": 0.8540093533807329,
              "n_positive": 83,
              "agreement": 0.8058252427184466,
              "agreement_ci95": [
                0.7190218278091454,
                0.8706369119689668
              ],
              "recall_share": 0.05845070422535211,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 87,
              "coverage": 0.008149882903981265,
              "mean_score": 0.9249171242738893,
              "n_positive": 80,
              "agreement": 0.9195402298850575,
              "agreement_ci95": [
                0.8431181093389192,
                0.9604797291258976
              ],
              "recall_share": 0.056338028169014086,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 97,
              "coverage": 0.00908665105386417,
              "mean_score": 0.9671999556670846,
              "n_positive": 94,
              "agreement": 0.9690721649484536,
              "agreement_ci95": [
                0.9129799642223378,
                0.9894266551336878
              ],
              "recall_share": 0.06619718309859154,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 69,
              "coverage": 0.0064637002341920375,
              "mean_score": 0.9852546298048739,
              "n_positive": 68,
              "agreement": 0.9855072463768116,
              "agreement_ci95": [
                0.9223687745949706,
                0.997437075383743
              ],
              "recall_share": 0.04788732394366197,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 58,
              "coverage": 0.005433255269320843,
              "mean_score": 0.9927661041621616,
              "n_positive": 57,
              "agreement": 0.9827586206896551,
              "agreement_ci95": [
                0.9085914181442059,
                0.9969499615876563
              ],
              "recall_share": 0.04014084507042254,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 371,
              "coverage": 0.03475409836065574,
              "mean_score": 0.9992926966062948,
              "n_positive": 371,
              "agreement": 1.0,
              "agreement_ci95": [
                0.9897517769438825,
                0.9999999999999999
              ],
              "recall_share": 0.2612676056338028,
              "meets_min_rows": true
            }
          ]
        }
      }
    },
    "split_agreement_gap": {
      "pooled_review": [
        {
          "lo": null,
          "hi": 0.5,
          "threshold_selection": 0.03508295728293693,
          "calibration": 0.036619335552687,
          "gap": -0.0015363782697500744
        },
        {
          "lo": 0.5,
          "hi": 0.8,
          "threshold_selection": 0.7153179190751445,
          "calibration": 0.6925418569254186,
          "gap": 0.02277606214972594
        },
        {
          "lo": 0.8,
          "hi": 0.9,
          "threshold_selection": 0.832,
          "calibration": 0.8472727272727273,
          "gap": -0.015272727272727327
        },
        {
          "lo": 0.9,
          "hi": 0.95,
          "threshold_selection": 0.9211822660098522,
          "calibration": 0.9506726457399103,
          "gap": -0.02949037973005808
        },
        {
          "lo": 0.95,
          "hi": 0.98,
          "threshold_selection": 0.9628099173553719,
          "calibration": 0.9620853080568721,
          "gap": 0.0007246092984998231
        },
        {
          "lo": 0.98,
          "hi": 0.99,
          "threshold_selection": 0.9850746268656716,
          "calibration": 1.0,
          "gap": -0.014925373134328401
        },
        {
          "lo": 0.99,
          "hi": 0.995,
          "threshold_selection": 0.990909090909091,
          "calibration": 0.98989898989899,
          "gap": 0.0010101010101010166
        },
        {
          "lo": 0.995,
          "hi": 1.0,
          "threshold_selection": 1.0,
          "calibration": 0.9964994165694282,
          "gap": 0.003500583430571802
        }
      ],
      "per_label": {
        "toxic": [
          {
            "lo": null,
            "hi": 0.5,
            "threshold_selection": 0.031540392050464626,
            "calibration": 0.03320657340757843,
            "gap": -0.0016661813571138065
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.6530612244897959,
            "calibration": 0.6365030674846626,
            "gap": 0.0165581570051333
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.7451737451737451,
            "calibration": 0.8129496402877698,
            "gap": -0.06777589511402471
          },
          {
            "lo": 0.9,
            "hi": 0.95,
            "threshold_selection": 0.8985507246376812,
            "calibration": 0.9282511210762332,
            "gap": -0.029700396438552046
          },
          {
            "lo": 0.95,
            "hi": 0.98,
            "threshold_selection": 0.9367088607594937,
            "calibration": 0.926605504587156,
            "gap": 0.010103356172337707
          },
          {
            "lo": 0.98,
            "hi": 0.99,
            "threshold_selection": 0.9612403100775194,
            "calibration": 0.9865771812080537,
            "gap": -0.02533687113053429
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "threshold_selection": 0.963302752293578,
            "calibration": 0.9603960396039604,
            "gap": 0.0029067126896176188
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "threshold_selection": 0.9937578027465668,
            "calibration": 0.9940546967895363,
            "gap": -0.00029689404296950883
          }
        ],
        "severe_toxic": [
          {
            "lo": null,
            "hi": 0.1,
            "threshold_selection": 0.003628377733218753,
            "calibration": 0.003378270707843325,
            "gap": 0.00025010702537542817
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "threshold_selection": 0.302158273381295,
            "calibration": 0.21875,
            "gap": 0.08340827338129497
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "threshold_selection": 0.24489795918367346,
            "calibration": 0.2948717948717949,
            "gap": -0.049973835688121415
          },
          {
            "lo": 0.3,
            "hi": 0.5,
            "threshold_selection": 0.3333333333333333,
            "calibration": 0.3870967741935484,
            "gap": -0.05376344086021506
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.5348837209302325,
            "calibration": 0.5169491525423728,
            "gap": 0.017934568387859673
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.7419354838709677,
            "calibration": 0.5806451612903226,
            "gap": 0.16129032258064513
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "threshold_selection": 0.8148148148148148,
            "calibration": 0.68,
            "gap": 0.13481481481481472
          }
        ],
        "obscene": [
          {
            "lo": null,
            "hi": 0.5,
            "threshold_selection": 0.015923879279338193,
            "calibration": 0.015802388352690988,
            "gap": 0.00012149092664720529
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.674496644295302,
            "calibration": 0.6466165413533834,
            "gap": 0.02788010294191856
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.773109243697479,
            "calibration": 0.8359375,
            "gap": -0.06282825630252098
          },
          {
            "lo": 0.9,
            "hi": 0.95,
            "threshold_selection": 0.8181818181818182,
            "calibration": 0.8660714285714286,
            "gap": -0.04788961038961037
          },
          {
            "lo": 0.95,
            "hi": 0.98,
            "threshold_selection": 0.8411214953271028,
            "calibration": 0.8925619834710744,
            "gap": -0.05144048814397162
          },
          {
            "lo": 0.98,
            "hi": 0.99,
            "threshold_selection": 0.9230769230769231,
            "calibration": 0.9,
            "gap": 0.023076923076923106
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "threshold_selection": 0.9482758620689655,
            "calibration": 0.9333333333333333,
            "gap": 0.014942528735632177
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "threshold_selection": 0.986013986013986,
            "calibration": 0.9828178694158075,
            "gap": 0.0031961165981785333
          }
        ],
        "threat": [
          {
            "lo": null,
            "hi": 0.1,
            "threshold_selection": 0.0011325028312570782,
            "calibration": 0.0014787779630620144,
            "gap": -0.00034627513180493616
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "threshold_selection": 0.1282051282051282,
            "calibration": 0.2702702702702703,
            "gap": -0.1420651420651421
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "threshold_selection": 0.2,
            "calibration": 0.5454545454545454,
            "gap": -0.3454545454545454
          },
          {
            "lo": 0.3,
            "hi": 0.5,
            "threshold_selection": 0.1875,
            "calibration": 0.36,
            "gap": -0.1725
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.42857142857142855,
            "calibration": 0.5,
            "gap": -0.07142857142857145
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.6666666666666666,
            "calibration": 0.5,
            "gap": 0.16666666666666663
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "threshold_selection": 0.7619047619047619,
            "calibration": 0.7272727272727273,
            "gap": 0.03463203463203457
          }
        ],
        "insult": [
          {
            "lo": null,
            "hi": 0.5,
            "threshold_selection": 0.02168056862808737,
            "calibration": 0.020985608939999352,
            "gap": 0.0006949596880880191
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.6445012787723785,
            "calibration": 0.6787564766839378,
            "gap": -0.03425519791155929
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.7866666666666666,
            "calibration": 0.774390243902439,
            "gap": 0.01227642276422758
          },
          {
            "lo": 0.9,
            "hi": 0.95,
            "threshold_selection": 0.8495575221238938,
            "calibration": 0.8540145985401459,
            "gap": -0.004457076416252148
          },
          {
            "lo": 0.95,
            "hi": 0.98,
            "threshold_selection": 0.8347107438016529,
            "calibration": 0.889763779527559,
            "gap": -0.05505303572590614
          },
          {
            "lo": 0.98,
            "hi": 0.99,
            "threshold_selection": 0.8870967741935484,
            "calibration": 0.9245283018867925,
            "gap": -0.03743152769324409
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "threshold_selection": 0.9122807017543859,
            "calibration": 0.96875,
            "gap": -0.056469298245614086
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "threshold_selection": 0.9617224880382775,
            "calibration": 0.91,
            "gap": 0.0517224880382775
          }
        ],
        "identity_hate": [
          {
            "lo": null,
            "hi": 0.1,
            "threshold_selection": 0.00399987301990413,
            "calibration": 0.003975574072896127,
            "gap": 2.429894700800312e-05
          },
          {
            "lo": 0.1,
            "hi": 0.2,
            "threshold_selection": 0.15483870967741936,
            "calibration": 0.1813186813186813,
            "gap": -0.02647997164126195
          },
          {
            "lo": 0.2,
            "hi": 0.3,
            "threshold_selection": 0.3492063492063492,
            "calibration": 0.27380952380952384,
            "gap": 0.07539682539682535
          },
          {
            "lo": 0.3,
            "hi": 0.5,
            "threshold_selection": 0.3170731707317073,
            "calibration": 0.32941176470588235,
            "gap": -0.012338593974175027
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "threshold_selection": 0.546875,
            "calibration": 0.5079365079365079,
            "gap": 0.03893849206349209
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "threshold_selection": 0.8095238095238095,
            "calibration": 0.6060606060606061,
            "gap": 0.20346320346320346
          },
          {
            "lo": 0.9,
            "hi": 1.0,
            "threshold_selection": 0.7142857142857143,
            "calibration": 0.8,
            "gap": -0.08571428571428574
          }
        ]
      },
      "note": "threshold-selection minus calibration agreement per segment. Calibration rows fitted the Platt calibrator, so this is a descriptive split comparison, not an independent estimate or bound on threshold-selection optimism. It does not measure train-to-test shift"
    },
    "strata_definition": {
      "identity_term_present": "whole-word match of review_router.signals.IDENTITY_TERMS",
      "oov_tercile": {
        "cut_points": [
          0.13157501270894997,
          0.20512820512820512
        ],
        "degenerate": false,
        "rows_per_stratum_threshold_selection": {
          "0": 10638,
          "1": 10601,
          "2": 10675
        },
        "definition": "share of word-analyzer tokens absent from the training vocabulary; terciles cut at the 1/3 and 2/3 quantiles of the threshold-selection split. Rows tied with a cut point go to the upper stratum, so equal cut points (degenerate) collapse the terciles into one stratum"
      }
    },
    "note": "calib is in-sample for the Platt fit and out-of-sample for thresholds; thresh is the reverse. Agreement = share of rows in the segment with the label positive; development splits only, not test results",
    "test": {
      "n_rows": 63978,
      "min_rows": 30,
      "edges": [
        0.5,
        0.8,
        0.9,
        0.95,
        0.98,
        0.99,
        0.995,
        1.0
      ],
      "per_label": {
        "toxic": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 56465,
              "coverage": 0.8825690080965332,
              "mean_score": 0.04053466178971401,
              "n_positive": 1554,
              "agreement": 0.027521473479146373,
              "agreement_ci95": [
                0.026203898324387187,
                0.028903332114318267
              ],
              "recall_share": 0.25517241379310346,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 2464,
              "coverage": 0.03851323892588077,
              "mean_score": 0.6513659859326011,
              "n_positive": 855,
              "agreement": 0.3469967532467532,
              "agreement_ci95": [
                0.32845279265036864,
                0.36601704564232385
              ],
              "recall_share": 0.14039408866995073,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 968,
              "coverage": 0.015130201006596018,
              "mean_score": 0.8522646547130224,
              "n_positive": 474,
              "agreement": 0.4896694214876033,
              "agreement_ci95": [
                0.45828143854781384,
                0.5211390730827329
              ],
              "recall_share": 0.07783251231527094,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 714,
              "coverage": 0.011160086279658633,
              "mean_score": 0.9260097090765138,
              "n_positive": 393,
              "agreement": 0.5504201680672269,
              "agreement_ci95": [
                0.5137591522560959,
                0.5865415466088134
              ],
              "recall_share": 0.0645320197044335,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 645,
              "coverage": 0.010081590546750445,
              "mean_score": 0.9665876949854184,
              "n_positive": 418,
              "agreement": 0.648062015503876,
              "agreement_ci95": [
                0.6104281086366259,
                0.6839427232293185
              ],
              "recall_share": 0.06863711001642037,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 399,
              "coverage": 0.006236518803338647,
              "mean_score": 0.985437327554943,
              "n_positive": 310,
              "agreement": 0.7769423558897243,
              "agreement_ci95": [
                0.7335635768811443,
                0.8150393415317824
              ],
              "recall_share": 0.05090311986863711,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 359,
              "coverage": 0.005611303885710713,
              "mean_score": 0.9926011549150535,
              "n_positive": 280,
              "agreement": 0.7799442896935933,
              "agreement_ci95": [
                0.7342502723323692,
                0.8197106797035925
              ],
              "recall_share": 0.04597701149425287,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 1964,
              "coverage": 0.030698052455531588,
              "mean_score": 0.9991612796517575,
              "n_positive": 1806,
              "agreement": 0.9195519348268839,
              "agreement_ci95": [
                0.9066879619700083,
                0.930777877852716
              ],
              "recall_share": 0.296551724137931,
              "meets_min_rows": true
            }
          ]
        },
        "severe_toxic": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 62780,
              "coverage": 0.9812748132170434,
              "mean_score": 0.003881242902649418,
              "n_positive": 114,
              "agreement": 0.0018158649251353934,
              "agreement_ci95": [
                0.0015119337438678749,
                0.0021807593678324455
              ],
              "recall_share": 0.3106267029972752,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 430,
              "coverage": 0.006721060364500297,
              "mean_score": 0.141861714995717,
              "n_positive": 41,
              "agreement": 0.09534883720930233,
              "agreement_ci95": [
                0.07106417427588962,
                0.12679948564874474
              ],
              "recall_share": 0.11171662125340599,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 202,
              "coverage": 0.0031573353340210697,
              "mean_score": 0.24589948840740586,
              "n_positive": 29,
              "agreement": 0.14356435643564355,
              "agreement_ci95": [
                0.10185472967462753,
                0.1985777449949119
              ],
              "recall_share": 0.07901907356948229,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 217,
              "coverage": 0.003391790928131545,
              "mean_score": 0.3870854331994601,
              "n_positive": 58,
              "agreement": 0.2672811059907834,
              "agreement_ci95": [
                0.21282286544808549,
                0.3298354718422497
              ],
              "recall_share": 0.15803814713896458,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 203,
              "coverage": 0.003172965706961768,
              "mean_score": 0.6511077216048682,
              "n_positive": 60,
              "agreement": 0.2955665024630542,
              "agreement_ci95": [
                0.2370636908750162,
                0.3616627905136461
              ],
              "recall_share": 0.16348773841961853,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 70,
              "coverage": 0.0010941261058488856,
              "mean_score": 0.8595735805034571,
              "n_positive": 28,
              "agreement": 0.4,
              "agreement_ci95": [
                0.29334257649517426,
                0.5170620361424806
              ],
              "recall_share": 0.07629427792915532,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 76,
              "coverage": 0.0011879083434930757,
              "mean_score": 0.9507601929184719,
              "n_positive": 37,
              "agreement": 0.4868421052631579,
              "agreement_ci95": [
                0.37783728697779023,
                0.5971130705429373
              ],
              "recall_share": 0.1008174386920981,
              "meets_min_rows": true
            }
          ]
        },
        "obscene": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 60500,
              "coverage": 0.9456375629122511,
              "mean_score": 0.021793012077167967,
              "n_positive": 1203,
              "agreement": 0.01988429752066116,
              "agreement_ci95": [
                0.018801990075343132,
                0.02102757116861583
              ],
              "recall_share": 0.3259279328095367,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 989,
              "coverage": 0.015458438838350684,
              "mean_score": 0.6478874472185574,
              "n_positive": 437,
              "agreement": 0.4418604651162791,
              "agreement_ci95": [
                0.41119429828063075,
                0.47297653385849187
              ],
              "recall_share": 0.11839609861826063,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 383,
              "coverage": 0.005986432836287474,
              "mean_score": 0.852759369840135,
              "n_positive": 232,
              "agreement": 0.6057441253263708,
              "agreement_ci95": [
                0.5559842666659828,
                0.6534038383066417
              ],
              "recall_share": 0.06285559468978597,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 247,
              "coverage": 0.0038607021163524964,
              "mean_score": 0.9271711146789424,
              "n_positive": 159,
              "agreement": 0.6437246963562753,
              "agreement_ci95": [
                0.5822187385835471,
                0.700828570727466
              ],
              "recall_share": 0.04307775670549986,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 289,
              "coverage": 0.004517177779861828,
              "mean_score": 0.9664031961206225,
              "n_positive": 206,
              "agreement": 0.71280276816609,
              "agreement_ci95": [
                0.6581150142408597,
                0.761907480080405
              ],
              "recall_share": 0.05581143321593064,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 156,
              "coverage": 0.002438338178748945,
              "mean_score": 0.9853403463769366,
              "n_positive": 138,
              "agreement": 0.8846153846153846,
              "agreement_ci95": [
                0.8249883545435954,
                0.9257555440660656
              ],
              "recall_share": 0.03738824166892441,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 167,
              "coverage": 0.002610272281096627,
              "mean_score": 0.9928881954429473,
              "n_positive": 144,
              "agreement": 0.8622754491017964,
              "agreement_ci95": [
                0.801816420361676,
                0.9064425748045568
              ],
              "recall_share": 0.039013817393660256,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 1247,
              "coverage": 0.01949107505705086,
              "mean_score": 0.9992954069236885,
              "n_positive": 1172,
              "agreement": 0.9398556535685646,
              "agreement_ci95": [
                0.9252600088054684,
                0.9517496171635286
              ],
              "recall_share": 0.3175291248984015,
              "meets_min_rows": true
            }
          ]
        },
        "threat": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 63546,
              "coverage": 0.9932476788896183,
              "mean_score": 0.00150408620598396,
              "n_positive": 66,
              "agreement": 0.0010386176942687188,
              "agreement_ci95": [
                0.0008165353198888823,
                0.0013210224735351911
              ],
              "recall_share": 0.3127962085308057,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 118,
              "coverage": 0.001844384007002407,
              "mean_score": 0.13648300000000818,
              "n_positive": 18,
              "agreement": 0.15254237288135594,
              "agreement_ci95": [
                0.0987223244826063,
                0.22827194534234324
              ],
              "recall_share": 0.08530805687203792,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 47,
              "coverage": 0.0007346275282128232,
              "mean_score": 0.23884957055459882,
              "n_positive": 10,
              "agreement": 0.2127659574468085,
              "agreement_ci95": [
                0.11989707943203741,
                0.3490402682739114
              ],
              "recall_share": 0.04739336492890995,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 79,
              "coverage": 0.001234799462315171,
              "mean_score": 0.4009311258850537,
              "n_positive": 21,
              "agreement": 0.26582278481012656,
              "agreement_ci95": [
                0.18093337637511664,
                0.37243035599206065
              ],
              "recall_share": 0.0995260663507109,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 78,
              "coverage": 0.0012191690893744724,
              "mean_score": 0.6443456711708819,
              "n_positive": 32,
              "agreement": 0.41025641025641024,
              "agreement_ci95": [
                0.30781905373553214,
                0.5211185019805041
              ],
              "recall_share": 0.15165876777251186,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 34,
              "coverage": 0.0005314326799837444,
              "mean_score": 0.8495645541443221,
              "n_positive": 15,
              "agreement": 0.4411764705882353,
              "agreement_ci95": [
                0.28883482142347944,
                0.6054610086461933
              ],
              "recall_share": 0.07109004739336493,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 76,
              "coverage": 0.0011879083434930757,
              "mean_score": 0.9724900115897231,
              "n_positive": 49,
              "agreement": 0.6447368421052632,
              "agreement_ci95": [
                0.5325638542435106,
                0.7429822130284868
              ],
              "recall_share": 0.23222748815165878,
              "meets_min_rows": true
            }
          ]
        },
        "insult": {
          "edges": [
            0.5,
            0.8,
            0.9,
            0.95,
            0.98,
            0.99,
            0.995,
            1.0
          ],
          "high_risk_edges": false,
          "table": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 61320,
              "coverage": 0.9584544687236237,
              "mean_score": 0.025957927519467336,
              "n_positive": 1541,
              "agreement": 0.025130463144161775,
              "agreement_ci95": [
                0.02392103660156481,
                0.026399383409148422
              ],
              "recall_share": 0.44966442953020136,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 1074,
              "coverage": 0.016787020538310046,
              "mean_score": 0.6417171473346815,
              "n_positive": 579,
              "agreement": 0.5391061452513967,
              "agreement_ci95": [
                0.5092081341935321,
                0.5687254053895384
              ],
              "recall_share": 0.16895243653341116,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 375,
              "coverage": 0.005861389852761887,
              "mean_score": 0.8530253744405697,
              "n_positive": 259,
              "agreement": 0.6906666666666667,
              "agreement_ci95": [
                0.6421488183673687,
                0.7353177884350477
              ],
              "recall_share": 0.07557630580682813,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 253,
              "coverage": 0.003954484353996686,
              "mean_score": 0.926373019838763,
              "n_positive": 192,
              "agreement": 0.758893280632411,
              "agreement_ci95": [
                0.70256491356404,
                0.8074773536407419
              ],
              "recall_share": 0.05602567843594981,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 270,
              "coverage": 0.004220200693988559,
              "mean_score": 0.966778222232305,
              "n_positive": 228,
              "agreement": 0.8444444444444444,
              "agreement_ci95": [
                0.7964148481024922,
                0.8828102800422812
              ],
              "recall_share": 0.0665304931426904,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 151,
              "coverage": 0.002360186314045453,
              "mean_score": 0.9849312986668622,
              "n_positive": 129,
              "agreement": 0.8543046357615894,
              "agreement_ci95": [
                0.7892546516810622,
                0.9017747455685979
              ],
              "recall_share": 0.03764225269915378,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 98,
              "coverage": 0.0015317765481884398,
              "mean_score": 0.9923207116031638,
              "n_positive": 87,
              "agreement": 0.8877551020408163,
              "agreement_ci95": [
                0.8101006655770652,
                0.9361573008232768
              ],
              "recall_share": 0.025386635541289757,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 437,
              "coverage": 0.006830472975085186,
              "mean_score": 0.9987182929325762,
              "n_positive": 412,
              "agreement": 0.9427917620137299,
              "agreement_ci95": [
                0.9169133829507851,
                0.9609532337283105
              ],
              "recall_share": 0.12022176831047564,
              "meets_min_rows": true
            }
          ]
        },
        "identity_hate": {
          "edges": [
            0.1,
            0.2,
            0.3,
            0.5,
            0.8,
            0.9,
            1.0
          ],
          "high_risk_edges": true,
          "table": [
            {
              "lo": null,
              "hi": 0.1,
              "n": 62727,
              "coverage": 0.9804464034511864,
              "mean_score": 0.004404707050484219,
              "n_positive": 250,
              "agreement": 0.003985524574744528,
              "agreement_ci95": [
                0.003521923264591372,
                0.004509874925323762
              ],
              "recall_share": 0.351123595505618,
              "meets_min_rows": true
            },
            {
              "lo": 0.1,
              "hi": 0.2,
              "n": 458,
              "coverage": 0.007158710806839851,
              "mean_score": 0.13896710083533978,
              "n_positive": 86,
              "agreement": 0.18777292576419213,
              "agreement_ci95": [
                0.15465842153952206,
                0.226081452501304
              ],
              "recall_share": 0.12078651685393259,
              "meets_min_rows": true
            },
            {
              "lo": 0.2,
              "hi": 0.3,
              "n": 189,
              "coverage": 0.002954140485791991,
              "mean_score": 0.24879104643119473,
              "n_positive": 41,
              "agreement": 0.21693121693121692,
              "agreement_ci95": [
                0.16412605790656112,
                0.28101400367187723
              ],
              "recall_share": 0.05758426966292135,
              "meets_min_rows": true
            },
            {
              "lo": 0.3,
              "hi": 0.5,
              "n": 204,
              "coverage": 0.0031885960799024663,
              "mean_score": 0.39360421050503924,
              "n_positive": 81,
              "agreement": 0.39705882352941174,
              "agreement_ci95": [
                0.33241506158195466,
                0.46550783490816583
              ],
              "recall_share": 0.11376404494382023,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 217,
              "coverage": 0.003391790928131545,
              "mean_score": 0.6434146108395037,
              "n_positive": 124,
              "agreement": 0.5714285714285714,
              "agreement_ci95": [
                0.5049062612861193,
                0.6354659322188274
              ],
              "recall_share": 0.17415730337078653,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 60,
              "coverage": 0.0009378223764419019,
              "mean_score": 0.8525362432003407,
              "n_positive": 40,
              "agreement": 0.6666666666666666,
              "agreement_ci95": [
                0.5405686635056332,
                0.7727073854829577
              ],
              "recall_share": 0.056179775280898875,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 1.0,
              "n": 123,
              "coverage": 0.0019225358717058988,
              "mean_score": 0.9631064869101681,
              "n_positive": 90,
              "agreement": 0.7317073170731707,
              "agreement_ci95": [
                0.6472648486223403,
                0.8021150347590326
              ],
              "recall_share": 0.12640449438202248,
              "meets_min_rows": true
            }
          ]
        }
      },
      "pooled_review": [
        {
          "lo": null,
          "hi": 0.5,
          "n": 56393,
          "coverage": 0.8814436212448029,
          "mean_score": 0.04134388199492032,
          "n_positive": 1597,
          "agreement": 0.028319117620981327,
          "agreement_ci95": [
            0.02698180941377358,
            0.029720682714492665
          ],
          "recall_share": 0.2558065032836777,
          "meets_min_rows": true
        },
        {
          "lo": 0.5,
          "hi": 0.8,
          "n": 2476,
          "coverage": 0.03870080340116915,
          "mean_score": 0.6505795872077165,
          "n_positive": 888,
          "agreement": 0.35864297253634897,
          "agreement_ci95": [
            0.33998435953476464,
            0.37773953063852767
          ],
          "recall_share": 0.142239308024988,
          "meets_min_rows": true
        },
        {
          "lo": 0.8,
          "hi": 0.9,
          "n": 976,
          "coverage": 0.015255243990121604,
          "mean_score": 0.8521842579818862,
          "n_positive": 487,
          "agreement": 0.4989754098360656,
          "agreement_ci95": [
            0.4676725830432401,
            0.5302862704203769
          ],
          "recall_share": 0.07800736825244274,
          "meets_min_rows": true
        },
        {
          "lo": 0.9,
          "hi": 0.95,
          "n": 720,
          "coverage": 0.011253868517302822,
          "mean_score": 0.9262222385817678,
          "n_positive": 397,
          "agreement": 0.5513888888888889,
          "agreement_ci95": [
            0.5148832997289581,
            0.5873490318017169
          ],
          "recall_share": 0.06359122216882909,
          "meets_min_rows": true
        },
        {
          "lo": 0.95,
          "hi": 0.98,
          "n": 653,
          "coverage": 0.010206633530276033,
          "mean_score": 0.966552774818308,
          "n_positive": 431,
          "agreement": 0.6600306278713629,
          "agreement_ci95": [
            0.62285667202149,
            0.6953327436648459
          ],
          "recall_share": 0.06903732180041647,
          "meets_min_rows": true
        },
        {
          "lo": 0.98,
          "hi": 0.99,
          "n": 397,
          "coverage": 0.006205258057457251,
          "mean_score": 0.9853709349460512,
          "n_positive": 309,
          "agreement": 0.7783375314861462,
          "agreement_ci95": [
            0.7349203980026731,
            0.8164197767806103
          ],
          "recall_share": 0.049495434887073524,
          "meets_min_rows": true
        },
        {
          "lo": 0.99,
          "hi": 0.995,
          "n": 355,
          "coverage": 0.00554878239394792,
          "mean_score": 0.9926513221995683,
          "n_positive": 281,
          "agreement": 0.7915492957746478,
          "agreement_ci95": [
            0.7462846037200882,
            0.830571818180914
          ],
          "recall_share": 0.04501041166106039,
          "meets_min_rows": true
        },
        {
          "lo": 0.995,
          "hi": 1.0,
          "n": 2008,
          "coverage": 0.03138578886492232,
          "mean_score": 0.9991735308656957,
          "n_positive": 1853,
          "agreement": 0.922808764940239,
          "agreement_ci95": [
            0.9103110499175051,
            0.9336918373399179
          ],
          "recall_share": 0.2968124299215121,
          "meets_min_rows": true
        }
      ],
      "high_risk_rules": {
        "R101_high_risk_priority": {
          "action": "priority_review",
          "signal": "p_threat",
          "threshold": 0.3,
          "n": 267,
          "coverage": 0.004173309575166463,
          "n_positive_label": 117,
          "agreement_label": 0.43820224719101125,
          "agreement_label_ci95": [
            0.3799817292174409,
            0.4981757724578726
          ],
          "n_positive_any_label": 210,
          "agreement_any_label": 0.7865168539325843
        },
        "skipped": [
          "R102_hierarchy_violation"
        ]
      },
      "priority_band": {
        "n": 2823,
        "table": [
          {
            "lo": null,
            "hi": 0.5,
            "n": 4,
            "coverage": 0.0014169323414806943,
            "mean_score": 0.4404690862289899,
            "n_positive": 1,
            "agreement": 0.25,
            "agreement_ci95": [
              0.04558726029536886,
              0.6993581599030918
            ],
            "recall_share": 0.0004051863857374392,
            "meets_min_rows": false
          },
          {
            "lo": 0.5,
            "hi": 0.8,
            "n": 30,
            "coverage": 0.010626992561105207,
            "mean_score": 0.6604918511027568,
            "n_positive": 13,
            "agreement": 0.43333333333333335,
            "agreement_ci95": [
              0.27377485469947366,
              0.6080269312687199
            ],
            "recall_share": 0.00526742301458671,
            "meets_min_rows": true
          },
          {
            "lo": 0.8,
            "hi": 0.9,
            "n": 22,
            "coverage": 0.0077931278781438185,
            "mean_score": 0.8632741847073114,
            "n_positive": 11,
            "agreement": 0.5,
            "agreement_ci95": [
              0.30722106144268146,
              0.6927789385573185
            ],
            "recall_share": 0.004457050243111832,
            "meets_min_rows": false
          },
          {
            "lo": 0.9,
            "hi": 0.95,
            "n": 26,
            "coverage": 0.009210060219624512,
            "mean_score": 0.9277845674764055,
            "n_positive": 16,
            "agreement": 0.6153846153846154,
            "agreement_ci95": [
              0.42534849943440245,
              0.775714056651308
            ],
            "recall_share": 0.006482982171799027,
            "meets_min_rows": false
          },
          {
            "lo": 0.95,
            "hi": 0.98,
            "n": 12,
            "coverage": 0.004250797024442083,
            "mean_score": 0.9660481413859136,
            "n_positive": 8,
            "agreement": 0.6666666666666666,
            "agreement_ci95": [
              0.3906220869404939,
              0.86187990987514
            ],
            "recall_share": 0.0032414910858995136,
            "meets_min_rows": false
          },
          {
            "lo": 0.98,
            "hi": 0.99,
            "n": 377,
            "coverage": 0.13354587318455544,
            "mean_score": 0.9854139013271068,
            "n_positive": 294,
            "agreement": 0.7798408488063661,
            "agreement_ci95": [
              0.7353077838589435,
              0.8187285351379501
            ],
            "recall_share": 0.11912479740680713,
            "meets_min_rows": true
          },
          {
            "lo": 0.99,
            "hi": 0.995,
            "n": 344,
            "coverage": 0.12185618136733971,
            "mean_score": 0.9926411376984944,
            "n_positive": 272,
            "agreement": 0.7906976744186046,
            "agreement_ci95": [
              0.7446155887101814,
              0.8303590006996789
            ],
            "recall_share": 0.11021069692058347,
            "meets_min_rows": true
          },
          {
            "lo": 0.995,
            "hi": 1.0,
            "n": 2008,
            "coverage": 0.7113000354233086,
            "mean_score": 0.9991735308656957,
            "n_positive": 1853,
            "agreement": 0.922808764940239,
            "agreement_ci95": [
              0.9103110499175051,
              0.9336918373399179
            ],
            "recall_share": 0.7508103727714749,
            "meets_min_rows": true
          }
        ],
        "n_rows_matching_priority_rule": 267,
        "lowest_segment_agreement": 0.25,
        "note": "pooled p_max against any positive label on the final priority band, rule promotions included; the leading bucket holds rows a rule promoted from below the first edge and is included in the lowest segment agreement. The priority-rule match count includes rows already in the model's priority band"
      },
      "strata": {
        "identity_term_present": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 52421,
              "coverage": 0.8876490111081008,
              "mean_score": 0.03980395448258722,
              "n_positive": 1395,
              "agreement": 0.026611472501478415,
              "agreement_ci95": [
                0.025268015956617044,
                0.028024304651974728
              ],
              "recall_share": 0.2552607502287283,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 2169,
              "coverage": 0.03672785153075048,
              "mean_score": 0.6490352545721153,
              "n_positive": 783,
              "agreement": 0.36099585062240663,
              "agreement_ci95": [
                0.34104542473459365,
                0.3814377792123437
              ],
              "recall_share": 0.14327538883806037,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 859,
              "coverage": 0.014545516120292604,
              "mean_score": 0.8519198634755297,
              "n_positive": 432,
              "agreement": 0.5029103608847497,
              "agreement_ci95": [
                0.46953592648087955,
                0.5362588808368896
              ],
              "recall_share": 0.07904849039341262,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 649,
              "coverage": 0.010989569222432945,
              "mean_score": 0.9265118708315971,
              "n_positive": 363,
              "agreement": 0.559322033898305,
              "agreement_ci95": [
                0.5208879575913449,
                0.5970579831152188
              ],
              "recall_share": 0.06642268984446477,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 561,
              "coverage": 0.009499458141425087,
              "mean_score": 0.9664284423113757,
              "n_positive": 367,
              "agreement": 0.6541889483065954,
              "agreement_ci95": [
                0.6139019439391717,
                0.6923786898206754
              ],
              "recall_share": 0.06715462031107045,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 353,
              "coverage": 0.005977377404497426,
              "mean_score": 0.9854083082528476,
              "n_positive": 278,
              "agreement": 0.7875354107648725,
              "agreement_ci95": [
                0.7418860073562883,
                0.8269940791572252
              ],
              "recall_share": 0.05086916742909424,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 295,
              "coverage": 0.00499525873746952,
              "mean_score": 0.9926463657983645,
              "n_positive": 233,
              "agreement": 0.7898305084745763,
              "agreement_ci95": [
                0.7397615330116155,
                0.83244822878025
              ],
              "recall_share": 0.04263494967978042,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 1749,
              "coverage": 0.029615957735031156,
              "mean_score": 0.9991995656210371,
              "n_positive": 1614,
              "agreement": 0.9228130360205832,
              "agreement_ci95": [
                0.9093579830461812,
                0.93441484792276
              ],
              "recall_share": 0.2953339432753888,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 3972,
              "coverage": 0.806989028850061,
              "mean_score": 0.061667280817682975,
              "n_positive": 202,
              "agreement": 0.05085599194360524,
              "agreement_ci95": [
                0.04444696733579488,
                0.05813294262773082
              ],
              "recall_share": 0.2596401028277635,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 307,
              "coverage": 0.062373019097927675,
              "mean_score": 0.6614905236462145,
              "n_positive": 105,
              "agreement": 0.34201954397394135,
              "agreement_ci95": [
                0.29119936752530445,
                0.3967444468259133
              ],
              "recall_share": 0.13496143958868895,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 117,
              "coverage": 0.023770824867939862,
              "mean_score": 0.854125410810605,
              "n_positive": 55,
              "agreement": 0.4700854700854701,
              "agreement_ci95": [
                0.3820433501998678,
                0.5600295106807991
              ],
              "recall_share": 0.07069408740359898,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 71,
              "coverage": 0.014425030475416498,
              "mean_score": 0.923574755058679,
              "n_positive": 34,
              "agreement": 0.4788732394366197,
              "agreement_ci95": [
                0.366775238281018,
                0.5931400273372243
              ],
              "recall_share": 0.043701799485861184,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 92,
              "coverage": 0.018691588785046728,
              "mean_score": 0.9673109328225382,
              "n_positive": 64,
              "agreement": 0.6956521739130435,
              "agreement_ci95": [
                0.5953572662034347,
                0.7802630594907686
              ],
              "recall_share": 0.08226221079691516,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 44,
              "coverage": 0.008939455505891914,
              "mean_score": 0.985071099098343,
              "n_positive": 31,
              "agreement": 0.7045454545454546,
              "agreement_ci95": [
                0.5577979975396521,
                0.8184447095371015
              ],
              "recall_share": 0.03984575835475578,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 60,
              "coverage": 0.012190166598943519,
              "mean_score": 0.9926756911721534,
              "n_positive": 48,
              "agreement": 0.8,
              "agreement_ci95": [
                0.6821819409122731,
                0.8817149472671906
              ],
              "recall_share": 0.061696658097686374,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 259,
              "coverage": 0.052620885818772854,
              "mean_score": 0.9989977208769225,
              "n_positive": 239,
              "agreement": 0.9227799227799228,
              "agreement_ci95": [
                0.8837435065503159,
                0.949458382269303
              ],
              "recall_share": 0.30719794344473006,
              "meets_min_rows": true
            }
          ]
        },
        "oov_tercile": {
          "0": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 17615,
              "coverage": 0.9168271482850153,
              "mean_score": 0.030468744594746994,
              "n_positive": 318,
              "agreement": 0.01805279591257451,
              "agreement_ci95": [
                0.016189104525221154,
                0.020126646449356404
              ],
              "recall_share": 0.24499229583975346,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 510,
              "coverage": 0.026544527143080206,
              "mean_score": 0.6477934265758823,
              "n_positive": 181,
              "agreement": 0.35490196078431374,
              "agreement_ci95": [
                0.3146010547948158,
                0.397372361437077
              ],
              "recall_share": 0.13944530046224962,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 206,
              "coverage": 0.010721907042106908,
              "mean_score": 0.8532770219515315,
              "n_positive": 99,
              "agreement": 0.48058252427184467,
              "agreement_ci95": [
                0.41333736420692874,
                0.5485386156749806
              ],
              "recall_share": 0.07627118644067797,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 153,
              "coverage": 0.007963358142924062,
              "mean_score": 0.9255365324340786,
              "n_positive": 90,
              "agreement": 0.5882352941176471,
              "agreement_ci95": [
                0.5090213418948868,
                0.6631270185742333
              ],
              "recall_share": 0.06933744221879815,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 136,
              "coverage": 0.007078540571488055,
              "mean_score": 0.9665343625665666,
              "n_positive": 95,
              "agreement": 0.6985294117647058,
              "agreement_ci95": [
                0.6168225102610356,
                0.7693290676185811
              ],
              "recall_share": 0.07318952234206472,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 102,
              "coverage": 0.005308905428616041,
              "mean_score": 0.9856569317136239,
              "n_positive": 80,
              "agreement": 0.7843137254901961,
              "agreement_ci95": [
                0.6949614246999715,
                0.8530279984716095
              ],
              "recall_share": 0.061633281972265024,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 82,
              "coverage": 0.004267943579867798,
              "mean_score": 0.9926284432711707,
              "n_positive": 63,
              "agreement": 0.7682926829268293,
              "agreement_ci95": [
                0.6662274736909519,
                0.8463453620159114
              ],
              "recall_share": 0.0485362095531587,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 409,
              "coverage": 0.021287669806901575,
              "mean_score": 0.9991486692655436,
              "n_positive": 372,
              "agreement": 0.9095354523227384,
              "agreement_ci95": [
                0.8777938177275945,
                0.9336556934425574
              ],
              "recall_share": 0.28659476117103233,
              "meets_min_rows": true
            }
          ],
          "1": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 17557,
              "coverage": 0.9036026762738034,
              "mean_score": 0.03802505866114086,
              "n_positive": 477,
              "agreement": 0.0271686506806402,
              "agreement_ci95": [
                0.024865340426179494,
                0.029678825986579338
              ],
              "recall_share": 0.2970112079701121,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 670,
              "coverage": 0.034482758620689655,
              "mean_score": 0.6476231684718435,
              "n_positive": 240,
              "agreement": 0.3582089552238806,
              "agreement_ci95": [
                0.3228061167065791,
                0.39522844840397986
              ],
              "recall_share": 0.149439601494396,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 266,
              "coverage": 0.013690169840452907,
              "mean_score": 0.8523793813478617,
              "n_positive": 150,
              "agreement": 0.5639097744360902,
              "agreement_ci95": [
                0.5038249810332706,
                0.6221749306962191
              ],
              "recall_share": 0.09339975093399751,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 168,
              "coverage": 0.008646423057128153,
              "mean_score": 0.926856299656962,
              "n_positive": 90,
              "agreement": 0.5357142857142857,
              "agreement_ci95": [
                0.46034519342900027,
                0.6094866159499536
              ],
              "recall_share": 0.05603985056039851,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 174,
              "coverage": 0.008955223880597015,
              "mean_score": 0.9667874606854099,
              "n_positive": 112,
              "agreement": 0.6436781609195402,
              "agreement_ci95": [
                0.5701201096833415,
                0.711029180895598
              ],
              "recall_share": 0.06973848069738481,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 83,
              "coverage": 0.004271744724652599,
              "mean_score": 0.9854841738425137,
              "n_positive": 65,
              "agreement": 0.7831325301204819,
              "agreement_ci95": [
                0.6830317718728761,
                0.8581843777112127
              ],
              "recall_share": 0.040473225404732256,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 81,
              "coverage": 0.004168811116829645,
              "mean_score": 0.9927412101904926,
              "n_positive": 66,
              "agreement": 0.8148148148148148,
              "agreement_ci95": [
                0.7166841046629637,
                0.8844371003761371
              ],
              "recall_share": 0.0410958904109589,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 431,
              "coverage": 0.02218219248584663,
              "mean_score": 0.9991250969677763,
              "n_positive": 406,
              "agreement": 0.9419953596287703,
              "agreement_ci95": [
                0.9157760042621619,
                0.9604053998297746
              ],
              "recall_share": 0.25280199252801994,
              "meets_min_rows": true
            }
          ],
          "2": [
            {
              "lo": null,
              "hi": 0.5,
              "n": 21221,
              "coverage": 0.8376159463193211,
              "mean_score": 0.05311684870597159,
              "n_positive": 802,
              "agreement": 0.037792752462183686,
              "agreement_ci95": [
                0.035309588034938645,
                0.0404432255724635
              ],
              "recall_share": 0.2401916741539383,
              "meets_min_rows": true
            },
            {
              "lo": 0.5,
              "hi": 0.8,
              "n": 1296,
              "coverage": 0.05115452930728242,
              "mean_score": 0.6532043885003632,
              "n_positive": 467,
              "agreement": 0.3603395061728395,
              "agreement_ci95": [
                0.33464940021404704,
                0.3868550974871155
              ],
              "recall_share": 0.13986223420185684,
              "meets_min_rows": true
            },
            {
              "lo": 0.8,
              "hi": 0.9,
              "n": 504,
              "coverage": 0.01989342806394316,
              "mean_score": 0.8516346306146313,
              "n_positive": 238,
              "agreement": 0.4722222222222222,
              "agreement_ci95": [
                0.4290125368144621,
                0.5158521458333861
              ],
              "recall_share": 0.07127882599580712,
              "meets_min_rows": true
            },
            {
              "lo": 0.9,
              "hi": 0.95,
              "n": 399,
              "coverage": 0.015748963883955004,
              "mean_score": 0.9262182054488449,
              "n_positive": 217,
              "agreement": 0.543859649122807,
              "agreement_ci95": [
                0.4948018365541293,
                0.5920809785791397
              ],
              "recall_share": 0.0649895178197065,
              "meets_min_rows": true
            },
            {
              "lo": 0.95,
              "hi": 0.98,
              "n": 343,
              "coverage": 0.013538582987961319,
              "mean_score": 0.9664410218310231,
              "n_positive": 224,
              "agreement": 0.6530612244897959,
              "agreement_ci95": [
                0.6012432826606438,
                0.7014886927458357
              ],
              "recall_share": 0.06708595387840671,
              "meets_min_rows": true
            },
            {
              "lo": 0.98,
              "hi": 0.99,
              "n": 212,
              "coverage": 0.008367870534833234,
              "mean_score": 0.9851889986314342,
              "n_positive": 164,
              "agreement": 0.7735849056603774,
              "agreement_ci95": [
                0.7126713007019043,
                0.8247602038905257
              ],
              "recall_share": 0.04911650194669063,
              "meets_min_rows": true
            },
            {
              "lo": 0.99,
              "hi": 0.995,
              "n": 192,
              "coverage": 0.0075784487862640615,
              "mean_score": 0.9926231719124002,
              "n_positive": 152,
              "agreement": 0.7916666666666666,
              "agreement_ci95": [
                0.728780314931051,
                0.8431108496863744
              ],
              "recall_share": 0.04552261156034741,
              "meets_min_rows": true
            },
            {
              "lo": 0.995,
              "hi": 1.0,
              "n": 1168,
              "coverage": 0.046102230116439706,
              "mean_score": 0.9992001091229434,
              "n_positive": 1075,
              "agreement": 0.9203767123287672,
              "agreement_ci95": [
                0.9034380599369605,
                0.9345592583499955
              ],
              "recall_share": 0.3219526804432465,
              "meets_min_rows": true
            }
          ]
        }
      }
    }
  },
  "ranking": {
    "n_pool": 6312,
    "n_high_risk": 916,
    "fractions": [
      0.5,
      0.67,
      1.0
    ],
    "by_key": {
      "fifo": {
        "ap_vs_high_risk": 0.14512040557667935,
        "at_fraction": [
          {
            "fraction": 0.5,
            "k": 3156,
            "harm_share_at_k": 0.49596299411269973,
            "high_risk_share_at_k": 0.47925764192139736
          },
          {
            "fraction": 0.67,
            "k": 4230,
            "harm_share_at_k": 0.6647603027754415,
            "high_risk_share_at_k": 0.6561135371179039
          },
          {
            "fraction": 1.0,
            "k": 6312,
            "harm_share_at_k": 1.0,
            "high_risk_share_at_k": 1.0
          }
        ]
      },
      "prob": {
        "ap_vs_high_risk": 0.32369536060165044,
        "at_fraction": [
          {
            "fraction": 0.5,
            "k": 3156,
            "harm_share_at_k": 0.7111858704793944,
            "high_risk_share_at_k": 0.8056768558951966
          },
          {
            "fraction": 0.67,
            "k": 4230,
            "harm_share_at_k": 0.8302775441547519,
            "high_risk_share_at_k": 0.8908296943231441
          },
          {
            "fraction": 1.0,
            "k": 6312,
            "harm_share_at_k": 1.0,
            "high_risk_share_at_k": 1.0
          }
        ]
      },
      "severity": {
        "ap_vs_high_risk": 0.5030484061774891,
        "at_fraction": [
          {
            "fraction": 0.5,
            "k": 3156,
            "harm_share_at_k": 0.7363330529857023,
            "high_risk_share_at_k": 0.8580786026200873
          },
          {
            "fraction": 0.67,
            "k": 4230,
            "harm_share_at_k": 0.8576114381833474,
            "high_risk_share_at_k": 0.9224890829694323
          },
          {
            "fraction": 1.0,
            "k": 6312,
            "harm_share_at_k": 1.0,
            "high_risk_share_at_k": 1.0
          }
        ]
      },
      "priority": {
        "ap_vs_high_risk": 0.48362185056731544,
        "at_fraction": [
          {
            "fraction": 0.5,
            "k": 3156,
            "harm_share_at_k": 0.7311185870479394,
            "high_risk_share_at_k": 0.8493449781659389
          },
          {
            "fraction": 0.67,
            "k": 4230,
            "harm_share_at_k": 0.8555088309503784,
            "high_risk_share_at_k": 0.9192139737991266
          },
          {
            "fraction": 1.0,
            "k": 6312,
            "harm_share_at_k": 1.0,
            "high_risk_share_at_k": 1.0
          }
        ]
      },
      "oracle_true_harm": {
        "ap_vs_high_risk": 1.0,
        "at_fraction": [
          {
            "fraction": 0.5,
            "k": 3156,
            "harm_share_at_k": 0.8849453322119428,
            "high_risk_share_at_k": 1.0
          },
          {
            "fraction": 0.67,
            "k": 4230,
            "harm_share_at_k": 0.9995794785534062,
            "high_risk_share_at_k": 1.0
          },
          {
            "fraction": 1.0,
            "k": 6312,
            "harm_share_at_k": 1.0,
            "high_risk_share_at_k": 1.0
          }
        ]
      }
    },
    "note": "oracle_true_harm ranks by the true-label harm proxy: it uses true labels and is not shippable; it is the ceiling for these shares. Shares are top-k over the whole review pool (both bands), k = ceil(fraction * n_pool); high risk = harm proxy >= 5; fifo scores are constant, so its order is pool row order."
  },
  "ci": {
    "conclusion": "success",
    "headSha": "210d9857220b23d175d4aff5a54ab221ad86280b",
    "jobs": [
      {
        "completedAt": "2026-09-23T11:59:36Z",
        "conclusion": "success",
        "databaseId": 107169498452,
        "name": "evaluate",
        "startedAt": "2026-09-23T11:57:24Z",
        "status": "completed",
        "steps": [
          {
            "completedAt": "2026-09-23T11:57:26Z",
            "conclusion": "success",
            "name": "Set up job",
            "number": 1,
            "startedAt": "2026-09-23T11:57:25Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:57:27Z",
            "conclusion": "success",
            "name": "Run actions/checkout@v4",
            "number": 2,
            "startedAt": "2026-09-23T11:57:26Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:57:28Z",
            "conclusion": "success",
            "name": "Run actions/setup-python@v5",
            "number": 3,
            "startedAt": "2026-09-23T11:57:27Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:03Z",
            "conclusion": "success",
            "name": "Install",
            "number": 4,
            "startedAt": "2026-09-23T11:57:28Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:09Z",
            "conclusion": "success",
            "name": "Restore the corpus cache",
            "number": 5,
            "startedAt": "2026-09-23T11:58:03Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:09Z",
            "conclusion": "skipped",
            "name": "Fetch the corpus",
            "number": 6,
            "startedAt": "2026-09-23T11:58:09Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:09Z",
            "conclusion": "success",
            "name": "Verify the corpus against the pinned hashes",
            "number": 7,
            "startedAt": "2026-09-23T11:58:09Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:24Z",
            "conclusion": "success",
            "name": "Run the evaluation with the code under test",
            "number": 8,
            "startedAt": "2026-09-23T11:58:09Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:27Z",
            "conclusion": "success",
            "name": "Archive the run (report, manifest, config, policy, predictions, per-job records)",
            "number": 9,
            "startedAt": "2026-09-23T11:59:24Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:32Z",
            "conclusion": "success",
            "name": "Regression gates against the report just produced",
            "number": 10,
            "startedAt": "2026-09-23T11:59:27Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:32Z",
            "conclusion": "success",
            "name": "Post Restore the corpus cache",
            "number": 18,
            "startedAt": "2026-09-23T11:59:32Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:32Z",
            "conclusion": "success",
            "name": "Post Run actions/setup-python@v5",
            "number": 19,
            "startedAt": "2026-09-23T11:59:32Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:33Z",
            "conclusion": "success",
            "name": "Post Run actions/checkout@v4",
            "number": 20,
            "startedAt": "2026-09-23T11:59:32Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:59:33Z",
            "conclusion": "success",
            "name": "Complete job",
            "number": 21,
            "startedAt": "2026-09-23T11:59:33Z",
            "status": "completed"
          }
        ],
        "url": "https://github.com/XinyangWuEthz/review-router/actions/runs/35857528118/job/107169498452"
      },
      {
        "completedAt": "2026-09-23T11:58:12Z",
        "conclusion": "success",
        "databaseId": 107169498727,
        "name": "negative-check",
        "startedAt": "2026-09-23T11:57:24Z",
        "status": "completed",
        "steps": [
          {
            "completedAt": "2026-09-23T11:57:26Z",
            "conclusion": "success",
            "name": "Set up job",
            "number": 1,
            "startedAt": "2026-09-23T11:57:25Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:57:27Z",
            "conclusion": "success",
            "name": "Run actions/checkout@v4",
            "number": 2,
            "startedAt": "2026-09-23T11:57:26Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:57:27Z",
            "conclusion": "success",
            "name": "Run actions/setup-python@v5",
            "number": 3,
            "startedAt": "2026-09-23T11:57:27Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:57:51Z",
            "conclusion": "success",
            "name": "Install",
            "number": 4,
            "startedAt": "2026-09-23T11:57:27Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:10Z",
            "conclusion": "success",
            "name": "Controlled failures must be caught",
            "number": 5,
            "startedAt": "2026-09-23T11:57:51Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:10Z",
            "conclusion": "success",
            "name": "Post Run actions/setup-python@v5",
            "number": 9,
            "startedAt": "2026-09-23T11:58:10Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:11Z",
            "conclusion": "success",
            "name": "Post Run actions/checkout@v4",
            "number": 10,
            "startedAt": "2026-09-23T11:58:10Z",
            "status": "completed"
          },
          {
            "completedAt": "2026-09-23T11:58:11Z",
            "conclusion": "success",
            "name": "Complete job",
            "number": 11,
            "startedAt": "2026-09-23T11:58:11Z",
            "status": "completed"
          }
        ],
        "url": "https://github.com/XinyangWuEthz/review-router/actions/runs/35857528118/job/107169498727"
      }
    ],
    "status": "completed",
    "url": "https://github.com/XinyangWuEthz/review-router/actions/runs/35857528118"
  },
  "artifact_sha256": {
    "manifest.json": "d468260855d625fbda8048bffaa59c892394e2fe38b96b4d1009c7e6eb8dfc63",
    "report.json": "82d529e1f20efa2dee8426175e13191f6d7b2f2d80a97e31d821d935be190393",
    "policy.yaml": "3df259614b78297f94c1465c767c8386ef9ca13ea43242e2f72c7c47ded27b2e",
    "config.yaml": "801efb58457b838a7316359cc89b91670d762379269720fe6d548611f137162d",
    "predictions.csv": "438029e7d44f3b333240b3c536e97d5ecdb1349009851abf85e7c18a3e111f09",
    "simulation_jobs.csv": "4516e2fab3f04ec0366dc496771246ed0163532fcca5560989c08094b1ba6349"
  },
  "test_cumulative": {
    "thresholds": {
      "priority_review": {
        "toxic": 0.8473530547862892,
        "severe_toxic": null,
        "obscene": 0.9258484945343994,
        "threat": null,
        "insult": 0.990352467757599,
        "identity_hate": null
      },
      "human_review": {
        "toxic": 0.6575178598345869,
        "severe_toxic": null,
        "obscene": 0.6575248417421482,
        "threat": null,
        "insult": 0.917856881186332,
        "identity_hate": null
      },
      "subgroup": {
        "priority_review": {
          "identity_term_present": {
            "toxic": 0.8950082386695898,
            "severe_toxic": null,
            "obscene": 0.9447978435575853,
            "threat": null,
            "insult": null,
            "identity_hate": null
          }
        }
      }
    },
    "tiers": {
      "priority_review": {
        "n_predicted_positive": 4624,
        "coverage": 0.07227484447778924,
        "precision": 0.7629757785467128,
        "precision_ci95": [
          0.7505034424642372,
          0.775011535006036
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1688,
        "coverage": 0.02638406952389884,
        "precision": 0.4188388625592417,
        "precision_ci95": [
          0.39551310500329784,
          0.4425331855863781
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57666,
        "coverage": 0.901341085998312,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "review_workload": {
      "n_total": 63978,
      "n_requires_human_review": 6312,
      "n_priority_review": 4624,
      "n_human_review": 1688,
      "review_fraction": 0.09865891400168808,
      "n_truly_positive": 4235,
      "precision": 0.6709442332065906,
      "precision_ci95": [
        0.6592517416596075,
        0.6824287793049322
      ],
      "high_risk_by_band": {
        "priority_review": {
          "n": 4624,
          "n_high_risk": 837,
          "high_risk_share": 0.1810121107266436
        },
        "human_review": {
          "n": 1688,
          "n_high_risk": 79,
          "high_risk_share": 0.046800947867298576
        },
        "allow": {
          "n": 57666,
          "n_high_risk": 164,
          "high_risk_share": 0.002843963514029064
        }
      },
      "n_high_risk_total": 1080,
      "high_risk_recall_into_queue": 0.8481481481481481,
      "high_risk_recall_into_priority": 0.775
    },
    "rules": {
      "matched_per_rule": {
        "R101_high_risk_priority": 267,
        "R102_hierarchy_violation": 0
      },
      "n_rows_with_any_rule": 267,
      "n_added_to_queue_from_allow": 18,
      "n_added_to_queue_from_allow_true_positive": 7,
      "n_promoted_to_priority_review": 44,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "allow_false_negatives": {
      "toxic": 1913,
      "severe_toxic": 6,
      "obscene": 721,
      "threat": 24,
      "insult": 694,
      "identity_hate": 140
    },
    "same_admitted_rows": true,
    "n_changed_band": 1801,
    "note": "Descriptive rerouting of the previously inspected test set, not fresh validation. Uses the same saved probabilities and labels, with cumulative thresholds copied from threshold_selection.alternatives.cumulative_precision. Thresholds were selected on the selection split; no model or threshold is fitted on test rows. All admitted rows still require human confirmation."
  }
}