{
  "with_r103": {
    "run_id": "20260922T100649877069Z-baseline",
    "git_commit": "e750939f82c711983546be5678a0788bb96e9d63",
    "git_dirty": true,
    "finished_utc": "2026-09-22T10:07:19+00:00",
    "data_sha256": {
      "train": "bd4084611bd27c939ba98e5e63bc3e5a2c1a4e99477dcba46c829e4c986c429d",
      "test": "c2513ce4abb98c4d1d216e3ca0d4377d57589a0989aa8c06a840509a16c786e8",
      "test_labels": "2a56dcbeba5c05f965a636f56cb5ae972bad60c3b952c239b49be18d7ab70f49"
    },
    "split_label_counts": {
      "train": {
        "toxic": 9182,
        "severe_toxic": 983,
        "obscene": 5140,
        "threat": 293,
        "insult": 4776,
        "identity_hate": 854,
        "rows": 95743
      },
      "calib": {
        "toxic": 3108,
        "severe_toxic": 303,
        "obscene": 1667,
        "threat": 106,
        "insult": 1558,
        "identity_hate": 281,
        "rows": 31914
      },
      "thresh": {
        "toxic": 3004,
        "severe_toxic": 309,
        "obscene": 1642,
        "threat": 79,
        "insult": 1543,
        "identity_hate": 270,
        "rows": 31914
      },
      "test_scored": {
        "toxic": 6090,
        "severe_toxic": 367,
        "obscene": 3691,
        "threat": 211,
        "insult": 3427,
        "identity_hate": 712,
        "rows": 63978
      }
    },
    "dependencies": {
      "python": "3.13.7",
      "review_router": "0.1.0",
      "numpy": "2.5.3",
      "pandas": "3.0.6",
      "scikit-learn": "1.9.1",
      "scipy": "1.18.1",
      "pyyaml": "6.0.3"
    },
    "policy_snapshot": {
      "version": 1,
      "tiers": {
        "allow": 1,
        "human_review": 2,
        "auto_action": 3
      },
      "severity_weights": {
        "threat": 10,
        "identity_hate": 6,
        "severe_toxic": 5,
        "insult": 2,
        "obscene": 2,
        "toxic": 1
      },
      "tier_precision_floors": {
        "human_review": 0.9,
        "auto_action": 0.99
      },
      "rules": [
        {
          "id": "R101_rare_high_harm_never_auto",
          "description": "Auto-action requires precision >= 0.99. Arithmetic on the scored test set says that is unreachable for threat: 211 positives against 63,767 negatives means a 1% FPR yields ~638 false positives against ~106 true ones. Route to a human instead of relaxing the precision bar.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_threat",
              "op": ">=",
              "threshold": 0.3
            }
          ]
        },
        {
          "id": "R102_hierarchy_violation",
          "description": "severe_toxic is an exact subset of toxic in the training labels \u2014 no row has severe_toxic=1 with toxic=0. A prediction asserting severe without toxic is logically impossible, so the model is out of distribution and neither head should be trusted. Send it to a human.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_severe_toxic",
              "op": ">=",
              "threshold": 0.5
            },
            {
              "signal": "p_toxic",
              "op": "<",
              "threshold": 0.5
            }
          ]
        },
        {
          "id": "R103_identity_term_low_confidence",
          "description": "The documented failure mode on this corpus is false positives on benign identity mentions. Suppress auto-action in the borderline band so over-enforcement does not concentrate on one group; a human decides.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "identity_term_present",
              "op": "==",
              "threshold": 1
            },
            {
              "signal": "p_max",
              "op": "<",
              "threshold": 0.95
            }
          ]
        }
      ],
      "gates": {
        "measured_on": null,
        "eval_slice": null,
        "per_label_average_precision": {
          "toxic": {
            "floor": null,
            "measured": null
          },
          "obscene": {
            "floor": null,
            "measured": null
          },
          "insult": {
            "floor": null,
            "measured": null
          },
          "identity_hate": {
            "floor": null,
            "measured": null
          },
          "severe_toxic": {
            "floor": null,
            "measured": null
          },
          "threat": {
            "floor": null,
            "measured": null
          }
        },
        "routing": {
          "auto_action_precision_floor": 0.99,
          "human_review_precision_floor": null,
          "harm_per_reviewer_hour_vs_fifo_min": null,
          "high_risk_wait_p90_vs_fifo_max": null,
          "queue_depth_p95_max": null,
          "reviewer_utilization_min": 0.6
        },
        "consistency": {
          "hierarchy_violation_rate_max": 0.005
        },
        "min_predicted_positives_for_precision": 30
      }
    },
    "tiers": {
      "auto_action": {
        "n_predicted_positive": 2221,
        "coverage": 0.03471505830129107,
        "precision": 0.8982440342188204,
        "precision_ci95": [
          0.8849751247046426,
          0.9101377103867604
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 8200,
        "coverage": 0.12816905811372659,
        "precision": 0.30109756097560975,
        "precision_ci95": [
          0.2912636411778855,
          0.3111177533981113
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 53557,
        "coverage": 0.8371158835849823,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "tiers_model_only": {
      "auto_action": {
        "n_predicted_positive": 2367,
        "coverage": 0.03699709275063303,
        "precision": 0.899873257287706,
        "precision_ci95": [
          0.8871253261551716,
          0.9113253643828748
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 3925,
        "coverage": 0.061349213792241085,
        "precision": 0.53171974522293,
        "precision_ci95": [
          0.5160856264588024,
          0.5472918354739326
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57686,
        "coverage": 0.9016536934571259,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "threshold_selection_tiers": {
      "auto_action": {
        "n_predicted_positive": 876,
        "coverage": 0.027448768565519834,
        "precision": 0.9897260273972602,
        "precision_ci95": [
          0.9805901327343466,
          0.9945855552707775
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 3031,
        "coverage": 0.09497399260512628,
        "precision": 0.40415704387990764,
        "precision_ci95": [
          0.38681887044057334,
          0.42173785060337593
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 28007,
        "coverage": 0.8775772388293539,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "per_label": {
      "toxic": {
        "positives": 6090,
        "negatives": 57888,
        "average_precision": 0.752184951704409,
        "roc_auc": 0.9569541029799007,
        "at_auto_action": {
          "threshold": 0.9892913517285165,
          "n_predicted_positive": 2365,
          "precision": 0.8976744186046511,
          "recall": 0.34860426929392446
        },
        "at_human_review": {
          "threshold": 0.6576420429168361,
          "n_predicted_positive": 6248,
          "precision": 0.6630921895006402,
          "recall": 0.680295566502463
        }
      },
      "severe_toxic": {
        "positives": 367,
        "negatives": 63611,
        "average_precision": 0.31126081415093715,
        "roc_auc": 0.981912884414067,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 3691,
        "negatives": 60287,
        "average_precision": 0.7733891183021634,
        "roc_auc": 0.971524362084933,
        "at_auto_action": {
          "threshold": 0.9983767822578123,
          "n_predicted_positive": 1033,
          "precision": 0.9515972894482091,
          "recall": 0.2663234895692224
        },
        "at_human_review": {
          "threshold": 0.6573428758151054,
          "n_predicted_positive": 2939,
          "precision": 0.7730520585233073,
          "recall": 0.6155513410999729
        }
      },
      "threat": {
        "positives": 211,
        "negatives": 63767,
        "average_precision": 0.45797903621474007,
        "roc_auc": 0.9917297028570469,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 3427,
        "negatives": 60551,
        "average_precision": 0.6960639271574238,
        "roc_auc": 0.9654037029086796,
        "at_auto_action": {
          "threshold": 0.9998379195789239,
          "n_predicted_positive": 131,
          "precision": 0.9847328244274809,
          "recall": 0.03764225269915378
        },
        "at_human_review": {
          "threshold": 0.9175983962802222,
          "n_predicted_positive": 1130,
          "precision": 0.8787610619469026,
          "recall": 0.2897578056609279
        }
      },
      "identity_hate": {
        "positives": 712,
        "negatives": 63266,
        "average_precision": 0.480036045041327,
        "roc_auc": 0.9739788700251516,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "threshold_selection_per_label": null,
    "rules": {
      "matched_per_rule": {
        "R101_rare_high_harm_never_auto": 267,
        "R102_hierarchy_violation": 0,
        "R103_identity_term_low_confidence": 4467
      },
      "n_rows_with_any_rule": 4728,
      "n_added_to_queue_from_allow": 4129,
      "n_added_to_queue_from_allow_true_positive": 247,
      "n_downgraded_from_auto_action": 146,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "consistency": {
      "hierarchy_violation_rate": 0.0,
      "n_violations": 0,
      "n_rows": 63978
    },
    "allow_false_negatives": {
      "toxic": 1684,
      "severe_toxic": 4,
      "obscene": 677,
      "threat": 21,
      "insult": 634,
      "identity_hate": 70
    },
    "simulation": {
      "assumptions": {
        "reviewers": 4,
        "handle_minutes": 2.0,
        "horizon_hours": 8.0,
        "capacity_per_hour": 120.0,
        "arrivals": "Poisson; jobs sampled with replacement from the human_review set",
        "handle_time": "deterministic",
        "harm_proxy": "max severity weight over TRUE labels (0 if clean); not real-world harm",
        "high_risk": "harm proxy >= 5.0",
        "loads_per_hour": [
          60.0,
          108.0,
          180.0
        ],
        "seeds": [
          1,
          2,
          3,
          4,
          5
        ],
        "queued_jobs": 8200,
        "queued_high_risk": 452
      },
      "summary": {
        "fifo@60": {
          "strategy": "fifo",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_handled": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 353.2,
            "std": 27.315929418564547
          },
          "harm_handled": {
            "mean": 352.8,
            "std": 27.095387061269307
          },
          "harm_per_reviewer_hour": {
            "mean": 11.025,
            "std": 0.8467308456646658
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.2973983211717983,
            "std": 0.1573110257581183
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.055035732434471255,
            "std": 0.052772610370365874
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "prob@60": {
          "strategy": "prob",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_handled": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 353.2,
            "std": 27.315929418564547
          },
          "harm_handled": {
            "mean": 352.8,
            "std": 27.095387061269307
          },
          "harm_per_reviewer_hour": {
            "mean": 11.025,
            "std": 0.8467308456646658
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.24862100215565158,
            "std": 0.14301118690681794
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.055035732434471255,
            "std": 0.052772610370365874
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "severity@60": {
          "strategy": "severity",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_handled": {
            "mean": 23.2,
            "std": 1.9390719429665315
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 353.2,
            "std": 27.315929418564547
          },
          "harm_handled": {
            "mean": 352.8,
            "std": 27.095387061269307
          },
          "harm_per_reviewer_hour": {
            "mean": 11.025,
            "std": 0.8467308456646658
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.24862100215565158,
            "std": 0.14301118690681794
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.055035732434471255,
            "std": 0.052772610370365874
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "fifo@108": {
          "strategy": "fifo",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_handled": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 711.2,
            "std": 48.73356133097601
          },
          "harm_handled": {
            "mean": 709.2,
            "std": 48.55677089757926
          },
          "harm_per_reviewer_hour": {
            "mean": 22.1625,
            "std": 1.517399090549352
          },
          "wait_p50": {
            "mean": 1.6465033420211455,
            "std": 0.978802764200094
          },
          "wait_p90": {
            "mean": 5.989271534385042,
            "std": 2.8671244892364833
          },
          "high_risk_wait_p50": {
            "mean": 1.569689633886553,
            "std": 0.6804685388738722
          },
          "high_risk_wait_p90": {
            "mean": 5.641146872124078,
            "std": 3.124574975458196
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "prob@108": {
          "strategy": "prob",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_handled": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 711.2,
            "std": 48.73356133097601
          },
          "harm_handled": {
            "mean": 709.6,
            "std": 47.78116783838587
          },
          "harm_per_reviewer_hour": {
            "mean": 22.175,
            "std": 1.4931614949495584
          },
          "wait_p50": {
            "mean": 0.4156372953695893,
            "std": 0.07210275785561279
          },
          "wait_p90": {
            "mean": 3.8006119393212408,
            "std": 1.0128441618743553
          },
          "high_risk_wait_p50": {
            "mean": 0.26383050581432493,
            "std": 0.08796026655621252
          },
          "high_risk_wait_p90": {
            "mean": 1.2279856404312295,
            "std": 0.33715290558998723
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "severity@108": {
          "strategy": "severity",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_handled": {
            "mean": 50.4,
            "std": 6.499230723708768
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 711.2,
            "std": 48.73356133097601
          },
          "harm_handled": {
            "mean": 709.6,
            "std": 47.78116783838587
          },
          "harm_per_reviewer_hour": {
            "mean": 22.175,
            "std": 1.4931614949495584
          },
          "wait_p50": {
            "mean": 0.4130328683347841,
            "std": 0.07337061087498785
          },
          "wait_p90": {
            "mean": 3.837007283272667,
            "std": 1.1077917403832036
          },
          "high_risk_wait_p50": {
            "mean": 0.2629487191961662,
            "std": 0.11130507314459351
          },
          "high_risk_wait_p90": {
            "mean": 1.1479152026615709,
            "std": 0.23553718416380232
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "fifo@180": {
          "strategy": "fifo",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 87.6,
            "std": 8.309031231136395
          },
          "high_risk_handled": {
            "mean": 59.4,
            "std": 5.678027826631356
          },
          "high_risk_unhandled": {
            "mean": 28.2,
            "std": 4.308131845707603
          },
          "harm_arrived": {
            "mean": 1203.6,
            "std": 95.5522893498633
          },
          "harm_handled": {
            "mean": 818.0,
            "std": 60.8177605638353
          },
          "harm_per_reviewer_hour": {
            "mean": 25.5625,
            "std": 1.9005550176198531
          },
          "wait_p50": {
            "mean": 78.75870506496214,
            "std": 2.4070498005840677
          },
          "wait_p90": {
            "mean": 139.86341084347154,
            "std": 7.777914356926516
          },
          "high_risk_wait_p50": {
            "mean": 77.70531912625,
            "std": 8.082034341601553
          },
          "high_risk_wait_p90": {
            "mean": 140.93024986798008,
            "std": 7.8704834239620824
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "prob@180": {
          "strategy": "prob",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 87.6,
            "std": 8.309031231136395
          },
          "high_risk_handled": {
            "mean": 84.8,
            "std": 6.6452990903344595
          },
          "high_risk_unhandled": {
            "mean": 2.8,
            "std": 2.7856776554368237
          },
          "harm_arrived": {
            "mean": 1203.6,
            "std": 95.5522893498633
          },
          "harm_handled": {
            "mean": 1174.8,
            "std": 77.51748189924645
          },
          "harm_per_reviewer_hour": {
            "mean": 36.7125,
            "std": 2.4224213093514515
          },
          "wait_p50": {
            "mean": 0.7331710942489466,
            "std": 0.2536214677694178
          },
          "wait_p90": {
            "mean": 11.355868520818822,
            "std": 1.770871789405372
          },
          "high_risk_wait_p50": {
            "mean": 0.5500507748890868,
            "std": 0.2752723434029232
          },
          "high_risk_wait_p90": {
            "mean": 4.5389607117453,
            "std": 3.40651729185038
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "severity@180": {
          "strategy": "severity",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 87.6,
            "std": 8.309031231136395
          },
          "high_risk_handled": {
            "mean": 85.2,
            "std": 7.0823724838503095
          },
          "high_risk_unhandled": {
            "mean": 2.4,
            "std": 1.624807680927192
          },
          "harm_arrived": {
            "mean": 1203.6,
            "std": 95.5522893498633
          },
          "harm_handled": {
            "mean": 1178.0,
            "std": 82.92647345691242
          },
          "harm_per_reviewer_hour": {
            "mean": 36.8125,
            "std": 2.591452295528513
          },
          "wait_p50": {
            "mean": 0.7251064419580494,
            "std": 0.25919863006219057
          },
          "wait_p90": {
            "mean": 11.839910071345184,
            "std": 1.6563670915039248
          },
          "high_risk_wait_p50": {
            "mean": 0.5262206462015115,
            "std": 0.2837362246727671
          },
          "high_risk_wait_p90": {
            "mean": 3.9050747802744183,
            "std": 3.2229348344838353
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        }
      },
      "paired": null,
      "harm_per_reviewer_hour": {
        "router": 36.8125,
        "fifo": 25.5625,
        "load_per_hour": 180.0
      },
      "high_risk_wait_p90": {
        "router": 1.1479152026615709,
        "fifo": 5.641146872124078,
        "load_per_hour": 108.0
      },
      "queue_depth_p95": 14.219999999999981,
      "reviewer_utilization": 0.9054738607875965,
      "primary": {
        "load_per_hour": 108.0,
        "strategy": "severity",
        "note": "queue-clearing gates read this load; the router strategy is compared against fifo at the same load for time-to-action"
      },
      "thesis": {
        "load_per_hour": 180.0,
        "strategy": "severity",
        "note": "harm handled per reviewer-hour only differs between orderings when the queue does not clear, so the FIFO comparison reads this load"
      }
    },
    "identity_false_positives": {
      "threshold_selection": {
        "definition": "predicted positive = routed to auto_action or human_review; false positive = no label is truly positive",
        "model_tier": {
          "with_identity_term": {
            "n_rows": 2035,
            "n_predicted_positive": 210,
            "n_false_positive": 13,
            "false_positive_rate": 0.06190476190476191,
            "false_positive_rate_ci95": [
              0.03653085811360575,
              0.10301859612885285
            ]
          },
          "without_identity_term": {
            "n_rows": 29879,
            "n_predicted_positive": 1869,
            "n_false_positive": 122,
            "false_positive_rate": 0.06527554842161584,
            "false_positive_rate_ci95": [
              0.05494472253976211,
              0.0773897351934296
            ]
          },
          "false_positive_rate_ratio": 0.9483606557377049
        },
        "final_tier": {
          "with_identity_term": {
            "n_rows": 2035,
            "n_predicted_positive": 2035,
            "n_false_positive": 1683,
            "false_positive_rate": 0.827027027027027,
            "false_positive_rate_ci95": [
              0.8099818679626472,
              0.8428398579258944
            ]
          },
          "without_identity_term": {
            "n_rows": 29879,
            "n_predicted_positive": 1872,
            "n_false_positive": 124,
            "false_positive_rate": 0.06623931623931624,
            "false_positive_rate_ci95": [
              0.05583810883933352,
              0.07841708511052611
            ]
          },
          "false_positive_rate_ratio": 12.485440278988666
        }
      },
      "test": {
        "definition": "predicted positive = routed to auto_action or human_review; false positive = no label is truly positive",
        "model_tier": {
          "with_identity_term": {
            "n_rows": 4922,
            "n_predicted_positive": 810,
            "n_false_positive": 272,
            "false_positive_rate": 0.3358024691358025,
            "false_positive_rate_ci95": [
              0.30412168441237586,
              0.36903332987548954
            ]
          },
          "without_identity_term": {
            "n_rows": 59056,
            "n_predicted_positive": 5482,
            "n_false_positive": 1794,
            "false_positive_rate": 0.3272528274352426,
            "false_positive_rate_ci95": [
              0.3149568495943547,
              0.33979073759052253
            ]
          },
          "false_positive_rate_ratio": 1.0261254937583442
        },
        "final_tier": {
          "with_identity_term": {
            "n_rows": 4922,
            "n_predicted_positive": 4921,
            "n_false_positive": 4143,
            "false_positive_rate": 0.8419020524283682,
            "false_positive_rate_ci95": [
              0.831442537993384,
              0.8518281882005875
            ]
          },
          "without_identity_term": {
            "n_rows": 59056,
            "n_predicted_positive": 5500,
            "n_false_positive": 1805,
            "false_positive_rate": 0.3281818181818182,
            "false_positive_rate_ci95": [
              0.3158961177494481,
              0.3407073629067409
            ]
          },
          "false_positive_rate_ratio": 2.5653525143246676
        }
      }
    },
    "subgroup_thresholds": null,
    "thresholds": {
      "auto_action": {
        "toxic": 0.9892913517285165,
        "severe_toxic": null,
        "obscene": 0.9983767822578123,
        "threat": null,
        "insult": 0.9998379195789239,
        "identity_hate": null
      },
      "human_review": {
        "toxic": 0.6576420429168361,
        "severe_toxic": null,
        "obscene": 0.6573428758151054,
        "threat": null,
        "insult": 0.9175983962802222,
        "identity_hate": null
      }
    },
    "identity_summary": {
      "selection_identity_predicted_positive_model": 210,
      "selection_identity_predicted_positive_final": 2035,
      "selection_identity_moved_to_positive_by_rules": 1825,
      "test_model_tier_predicted_positive": {
        "ratio": 1.0261254937583442,
        "ci95": null,
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 272,
          "false_positive_rate": 0.3358024691358025,
          "false_positive_rate_ci95": [
            0.30412168441237586,
            0.36903332987548954
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5482,
          "n_false_positive": 1794,
          "false_positive_rate": 0.3272528274352426,
          "false_positive_rate_ci95": [
            0.3149568495943547,
            0.33979073759052253
          ]
        }
      },
      "test_final_tier_predicted_positive": {
        "ratio": 2.5653525143246676,
        "ci95": null,
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 4921,
          "n_false_positive": 4143,
          "false_positive_rate": 0.8419020524283682,
          "false_positive_rate_ci95": [
            0.831442537993384,
            0.8518281882005875
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5500,
          "n_false_positive": 1805,
          "false_positive_rate": 0.3281818181818182,
          "false_positive_rate_ci95": [
            0.3158961177494481,
            0.3407073629067409
          ]
        }
      },
      "test_by_term": null,
      "identity_terms": null
    }
  },
  "without_r103": {
    "run_id": "20260922T101035330306Z-baseline",
    "git_commit": "e750939f82c711983546be5678a0788bb96e9d63",
    "git_dirty": true,
    "finished_utc": "2026-09-22T10:11:03+00:00",
    "data_sha256": {
      "train": "bd4084611bd27c939ba98e5e63bc3e5a2c1a4e99477dcba46c829e4c986c429d",
      "test": "c2513ce4abb98c4d1d216e3ca0d4377d57589a0989aa8c06a840509a16c786e8",
      "test_labels": "2a56dcbeba5c05f965a636f56cb5ae972bad60c3b952c239b49be18d7ab70f49"
    },
    "split_label_counts": {
      "train": {
        "toxic": 9182,
        "severe_toxic": 983,
        "obscene": 5140,
        "threat": 293,
        "insult": 4776,
        "identity_hate": 854,
        "rows": 95743
      },
      "calib": {
        "toxic": 3108,
        "severe_toxic": 303,
        "obscene": 1667,
        "threat": 106,
        "insult": 1558,
        "identity_hate": 281,
        "rows": 31914
      },
      "thresh": {
        "toxic": 3004,
        "severe_toxic": 309,
        "obscene": 1642,
        "threat": 79,
        "insult": 1543,
        "identity_hate": 270,
        "rows": 31914
      },
      "test_scored": {
        "toxic": 6090,
        "severe_toxic": 367,
        "obscene": 3691,
        "threat": 211,
        "insult": 3427,
        "identity_hate": 712,
        "rows": 63978
      }
    },
    "dependencies": {
      "python": "3.13.7",
      "review_router": "0.1.0",
      "numpy": "2.5.3",
      "pandas": "3.0.6",
      "scikit-learn": "1.9.1",
      "scipy": "1.18.1",
      "pyyaml": "6.0.3"
    },
    "policy_snapshot": {
      "version": 1,
      "tiers": {
        "allow": 1,
        "human_review": 2,
        "auto_action": 3
      },
      "severity_weights": {
        "threat": 10,
        "identity_hate": 6,
        "severe_toxic": 5,
        "insult": 2,
        "obscene": 2,
        "toxic": 1
      },
      "tier_precision_floors": {
        "human_review": 0.9,
        "auto_action": 0.99
      },
      "rules": [
        {
          "id": "R101_rare_high_harm_never_auto",
          "description": "Auto-action requires precision >= 0.99. Arithmetic on the scored test set says that is unreachable for threat: 211 positives against 63,767 negatives means a 1% FPR yields ~638 false positives against ~106 true ones. Route to a human instead of relaxing the precision bar.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_threat",
              "op": ">=",
              "threshold": 0.3
            }
          ]
        },
        {
          "id": "R102_hierarchy_violation",
          "description": "severe_toxic is an exact subset of toxic in the training labels \u2014 no row has severe_toxic=1 with toxic=0. A prediction asserting severe without toxic is logically impossible, so the model is out of distribution and neither head should be trusted. Send it to a human.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_severe_toxic",
              "op": ">=",
              "threshold": 0.5
            },
            {
              "signal": "p_toxic",
              "op": "<",
              "threshold": 0.5
            }
          ]
        }
      ],
      "retired_rules": [
        {
          "id": "R103_identity_term_low_confidence",
          "retired_on": "2026-09-22",
          "was": "identity_term_present == 1 AND p_max < 0.95 -> human_review, meant to suppress auto-action on borderline identity mentions.\n",
          "why": "Measured on the threshold-selection split (run 20260922T100649877069Z, 31,914 rows): the auto-action thresholds chosen from the 0.99 precision floor are 0.989 (toxic), 0.998 (obscene) and 0.9998 (insult), all above the rule's 0.95 band, so the rule can never intercept an auto-action. Its only effect was to move 1,825 allow-tier comments with an identity term into the human queue, of which 1,683 (92%) had no positive label, cutting queue precision from 0.887 to 0.404. Its premise also did not hold for this model: the false-positive rate among predicted positives was 0.062 with an identity term and 0.065 without (test split: 0.336 vs 0.327). Rules cannot fix a concentration that is not there; a gate can catch one that appears.\n",
          "replaced_by": "gates.fairness.identity_false_positive_rate_ratio_max"
        }
      ],
      "gates": {
        "measured_on": null,
        "eval_slice": null,
        "per_label_average_precision": {
          "toxic": {
            "floor": null,
            "measured": null
          },
          "obscene": {
            "floor": null,
            "measured": null
          },
          "insult": {
            "floor": null,
            "measured": null
          },
          "identity_hate": {
            "floor": null,
            "measured": null
          },
          "severe_toxic": {
            "floor": null,
            "measured": null
          },
          "threat": {
            "floor": null,
            "measured": null
          }
        },
        "routing": {
          "auto_action_precision_floor": 0.99,
          "human_review_precision_floor": null,
          "harm_per_reviewer_hour_vs_fifo_min": null,
          "high_risk_wait_p90_vs_fifo_max": null,
          "queue_depth_p95_max": null,
          "reviewer_utilization_min": 0.6
        },
        "consistency": {
          "hierarchy_violation_rate_max": 0.005
        },
        "fairness": {
          "identity_false_positive_rate_ratio_max": null,
          "identity_false_positive_rate_ratio_measured": null
        },
        "min_predicted_positives_for_precision": 30
      }
    },
    "tiers": {
      "auto_action": {
        "n_predicted_positive": 2221,
        "coverage": 0.03471505830129107,
        "precision": 0.8982440342188204,
        "precision_ci95": [
          0.8849751247046426,
          0.9101377103867604
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 4089,
        "coverage": 0.06391259495451561,
        "precision": 0.5451210564930301,
        "precision_ci95": [
          0.5298230221333109,
          0.5603343914159693
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57668,
        "coverage": 0.9013723467441933,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "tiers_model_only": {
      "auto_action": {
        "n_predicted_positive": 2367,
        "coverage": 0.03699709275063303,
        "precision": 0.899873257287706,
        "precision_ci95": [
          0.8871253261551716,
          0.9113253643828748
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 3925,
        "coverage": 0.061349213792241085,
        "precision": 0.53171974522293,
        "precision_ci95": [
          0.5160856264588024,
          0.5472918354739326
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57686,
        "coverage": 0.9016536934571259,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "threshold_selection_tiers": {
      "auto_action": {
        "n_predicted_positive": 876,
        "coverage": 0.027448768565519834,
        "precision": 0.9897260273972602,
        "precision_ci95": [
          0.9805901327343466,
          0.9945855552707775
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1206,
        "coverage": 0.03778905809362662,
        "precision": 0.8872305140961857,
        "precision_ci95": [
          0.8681349302997525,
          0.903867048260018
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 29832,
        "coverage": 0.9347621733408535,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "per_label": {
      "toxic": {
        "positives": 6090,
        "negatives": 57888,
        "average_precision": 0.752184951704409,
        "roc_auc": 0.9569541029799007,
        "at_auto_action": {
          "threshold": 0.9892913517285165,
          "n_predicted_positive": 2365,
          "precision": 0.8976744186046511,
          "recall": 0.34860426929392446
        },
        "at_human_review": {
          "threshold": 0.6576420429168361,
          "n_predicted_positive": 6248,
          "precision": 0.6630921895006402,
          "recall": 0.680295566502463
        }
      },
      "severe_toxic": {
        "positives": 367,
        "negatives": 63611,
        "average_precision": 0.31126081415093715,
        "roc_auc": 0.981912884414067,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 3691,
        "negatives": 60287,
        "average_precision": 0.7733891183021634,
        "roc_auc": 0.971524362084933,
        "at_auto_action": {
          "threshold": 0.9983767822578123,
          "n_predicted_positive": 1033,
          "precision": 0.9515972894482091,
          "recall": 0.2663234895692224
        },
        "at_human_review": {
          "threshold": 0.6573428758151054,
          "n_predicted_positive": 2939,
          "precision": 0.7730520585233073,
          "recall": 0.6155513410999729
        }
      },
      "threat": {
        "positives": 211,
        "negatives": 63767,
        "average_precision": 0.45797903621474007,
        "roc_auc": 0.9917297028570469,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 3427,
        "negatives": 60551,
        "average_precision": 0.6960639271574238,
        "roc_auc": 0.9654037029086796,
        "at_auto_action": {
          "threshold": 0.9998379195789239,
          "n_predicted_positive": 131,
          "precision": 0.9847328244274809,
          "recall": 0.03764225269915378
        },
        "at_human_review": {
          "threshold": 0.9175983962802222,
          "n_predicted_positive": 1130,
          "precision": 0.8787610619469026,
          "recall": 0.2897578056609279
        }
      },
      "identity_hate": {
        "positives": 712,
        "negatives": 63266,
        "average_precision": 0.480036045041327,
        "roc_auc": 0.9739788700251516,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "threshold_selection_per_label": null,
    "rules": {
      "matched_per_rule": {
        "R101_rare_high_harm_never_auto": 267,
        "R102_hierarchy_violation": 0
      },
      "n_rows_with_any_rule": 267,
      "n_added_to_queue_from_allow": 18,
      "n_added_to_queue_from_allow_true_positive": 7,
      "n_downgraded_from_auto_action": 146,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "consistency": {
      "hierarchy_violation_rate": 0.0,
      "n_violations": 0,
      "n_rows": 63978
    },
    "allow_false_negatives": {
      "toxic": 1914,
      "severe_toxic": 6,
      "obscene": 722,
      "threat": 24,
      "insult": 695,
      "identity_hate": 140
    },
    "simulation": {
      "assumptions": {
        "reviewers": 4,
        "handle_minutes": 2.0,
        "horizon_hours": 8.0,
        "capacity_per_hour": 120.0,
        "arrivals": "Poisson; jobs sampled with replacement from the human_review set",
        "handle_time": "deterministic",
        "harm_proxy": "max severity weight over TRUE labels (0 if clean); not real-world harm",
        "high_risk": "harm proxy >= 5.0",
        "loads_per_hour": [
          60.0,
          108.0,
          180.0
        ],
        "seeds": [
          1,
          2,
          3,
          4,
          5
        ],
        "queued_jobs": 4089,
        "queued_high_risk": 381
      },
      "summary": {
        "fifo@60": {
          "strategy": "fifo",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_handled": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 684.2,
            "std": 24.9110417285187
          },
          "harm_handled": {
            "mean": 682.8,
            "std": 23.852882425400917
          },
          "harm_per_reviewer_hour": {
            "mean": 21.3375,
            "std": 0.7454025757937787
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.2973983211717983,
            "std": 0.1573110257581183
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.3406552598467364,
            "std": 0.26732449177244605
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "prob@60": {
          "strategy": "prob",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_handled": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 684.2,
            "std": 24.9110417285187
          },
          "harm_handled": {
            "mean": 682.8,
            "std": 23.852882425400917
          },
          "harm_per_reviewer_hour": {
            "mean": 21.3375,
            "std": 0.7454025757937787
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.2513060135882726,
            "std": 0.13738932521313937
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.29990490989177376,
            "std": 0.28705260083057577
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "severity@60": {
          "strategy": "severity",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_handled": {
            "mean": 45.8,
            "std": 5.192301994298868
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 684.2,
            "std": 24.9110417285187
          },
          "harm_handled": {
            "mean": 682.8,
            "std": 23.852882425400917
          },
          "harm_per_reviewer_hour": {
            "mean": 21.3375,
            "std": 0.7454025757937787
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.24293703304220576,
            "std": 0.13277485279739343
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.3128026708692476,
            "std": 0.26712718105092664
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "fifo@108": {
          "strategy": "fifo",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 81.6,
            "std": 7.863841300534999
          },
          "high_risk_handled": {
            "mean": 81.0,
            "std": 8.024961059095553
          },
          "high_risk_unhandled": {
            "mean": 0.6,
            "std": 0.7999999999999999
          },
          "harm_arrived": {
            "mean": 1243.2,
            "std": 41.24269632310671
          },
          "harm_handled": {
            "mean": 1235.4,
            "std": 45.53064901799666
          },
          "harm_per_reviewer_hour": {
            "mean": 38.60625,
            "std": 1.4228327818123956
          },
          "wait_p50": {
            "mean": 1.6465033420211455,
            "std": 0.978802764200094
          },
          "wait_p90": {
            "mean": 5.989271534385042,
            "std": 2.8671244892364833
          },
          "high_risk_wait_p50": {
            "mean": 1.5355311407620547,
            "std": 0.8485135035410789
          },
          "high_risk_wait_p90": {
            "mean": 6.052307426562871,
            "std": 2.924242912965674
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "prob@108": {
          "strategy": "prob",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 81.6,
            "std": 7.863841300534999
          },
          "high_risk_handled": {
            "mean": 80.8,
            "std": 8.182909018191513
          },
          "high_risk_unhandled": {
            "mean": 0.8,
            "std": 1.1661903789690604
          },
          "harm_arrived": {
            "mean": 1243.2,
            "std": 41.24269632310671
          },
          "harm_handled": {
            "mean": 1234.2,
            "std": 47.38522976624678
          },
          "harm_per_reviewer_hour": {
            "mean": 38.56875,
            "std": 1.480788430195212
          },
          "wait_p50": {
            "mean": 0.4131482341743123,
            "std": 0.07306922292737829
          },
          "wait_p90": {
            "mean": 4.18481706100559,
            "std": 1.2826188155406917
          },
          "high_risk_wait_p50": {
            "mean": 0.29893491501111313,
            "std": 0.048345229297486324
          },
          "high_risk_wait_p90": {
            "mean": 2.3616953196861745,
            "std": 0.5834796090804639
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "severity@108": {
          "strategy": "severity",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 81.6,
            "std": 7.863841300534999
          },
          "high_risk_handled": {
            "mean": 81.0,
            "std": 8.024961059095553
          },
          "high_risk_unhandled": {
            "mean": 0.6,
            "std": 0.7999999999999999
          },
          "harm_arrived": {
            "mean": 1243.2,
            "std": 41.24269632310671
          },
          "harm_handled": {
            "mean": 1235.4,
            "std": 45.53064901799666
          },
          "harm_per_reviewer_hour": {
            "mean": 38.60625,
            "std": 1.4228327818123956
          },
          "wait_p50": {
            "mean": 0.40901241687014434,
            "std": 0.07020258477324523
          },
          "wait_p90": {
            "mean": 3.9215364941665634,
            "std": 0.7806542751531598
          },
          "high_risk_wait_p50": {
            "mean": 0.2614753040115779,
            "std": 0.05039368738003757
          },
          "high_risk_wait_p90": {
            "mean": 1.2821030463009353,
            "std": 0.13573284064945945
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "fifo@180": {
          "strategy": "fifo",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 130.0,
            "std": 7.3484692283495345
          },
          "high_risk_handled": {
            "mean": 85.6,
            "std": 2.6532998322843198
          },
          "high_risk_unhandled": {
            "mean": 44.4,
            "std": 7.059745037889116
          },
          "harm_arrived": {
            "mean": 2013.0,
            "std": 110.26332119068425
          },
          "harm_handled": {
            "mean": 1353.6,
            "std": 32.33326460473795
          },
          "harm_per_reviewer_hour": {
            "mean": 42.3,
            "std": 1.0104145188980609
          },
          "wait_p50": {
            "mean": 78.75870506496214,
            "std": 2.4070498005840677
          },
          "wait_p90": {
            "mean": 139.86341084347154,
            "std": 7.777914356926516
          },
          "high_risk_wait_p50": {
            "mean": 78.93153278410024,
            "std": 5.132280792962084
          },
          "high_risk_wait_p90": {
            "mean": 143.76466643065183,
            "std": 8.738523115731237
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "prob@180": {
          "strategy": "prob",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 130.0,
            "std": 7.3484692283495345
          },
          "high_risk_handled": {
            "mean": 110.0,
            "std": 6.957010852370434
          },
          "high_risk_unhandled": {
            "mean": 20.0,
            "std": 3.63318042491699
          },
          "harm_arrived": {
            "mean": 2013.0,
            "std": 110.26332119068425
          },
          "harm_handled": {
            "mean": 1617.6,
            "std": 84.45969452940261
          },
          "harm_per_reviewer_hour": {
            "mean": 50.55,
            "std": 2.6393654540438316
          },
          "wait_p50": {
            "mean": 0.7349132673123855,
            "std": 0.2518782236191279
          },
          "wait_p90": {
            "mean": 11.984519044759839,
            "std": 3.8729903504072336
          },
          "high_risk_wait_p50": {
            "mean": 0.5249606587028758,
            "std": 0.2233777409944969
          },
          "high_risk_wait_p90": {
            "mean": 2.7349861854170787,
            "std": 0.6328780264041517
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "severity@180": {
          "strategy": "severity",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 130.0,
            "std": 7.3484692283495345
          },
          "high_risk_handled": {
            "mean": 118.2,
            "std": 7.467261881037787
          },
          "high_risk_unhandled": {
            "mean": 11.8,
            "std": 1.8330302779823358
          },
          "harm_arrived": {
            "mean": 2013.0,
            "std": 110.26332119068425
          },
          "harm_handled": {
            "mean": 1686.2,
            "std": 83.13218390009973
          },
          "harm_per_reviewer_hour": {
            "mean": 52.69375,
            "std": 2.5978807468781167
          },
          "wait_p50": {
            "mean": 0.7162131972670821,
            "std": 0.24671238540110807
          },
          "wait_p90": {
            "mean": 11.915087953247786,
            "std": 2.7562330366642174
          },
          "high_risk_wait_p50": {
            "mean": 0.45931001644847386,
            "std": 0.17064873113489948
          },
          "high_risk_wait_p90": {
            "mean": 1.498615961327203,
            "std": 0.18406233225201857
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        }
      },
      "paired": null,
      "harm_per_reviewer_hour": {
        "router": 52.69375,
        "fifo": 42.3,
        "load_per_hour": 180.0
      },
      "high_risk_wait_p90": {
        "router": 1.2821030463009353,
        "fifo": 6.052307426562871,
        "load_per_hour": 108.0
      },
      "queue_depth_p95": 14.219999999999981,
      "reviewer_utilization": 0.9054738607875965,
      "primary": {
        "load_per_hour": 108.0,
        "strategy": "severity",
        "note": "queue-clearing gates read this load; the router strategy is compared against fifo at the same load for time-to-action"
      },
      "thesis": {
        "load_per_hour": 180.0,
        "strategy": "severity",
        "note": "harm handled per reviewer-hour only differs between orderings when the queue does not clear, so the FIFO comparison reads this load"
      }
    },
    "identity_false_positives": {
      "threshold_selection": {
        "definition": "predicted positive = routed to auto_action or human_review; false positive = no label is truly positive",
        "model_tier": {
          "with_identity_term": {
            "n_rows": 2035,
            "n_predicted_positive": 210,
            "n_false_positive": 13,
            "false_positive_rate": 0.06190476190476191,
            "false_positive_rate_ci95": [
              0.03653085811360575,
              0.10301859612885285
            ]
          },
          "without_identity_term": {
            "n_rows": 29879,
            "n_predicted_positive": 1869,
            "n_false_positive": 122,
            "false_positive_rate": 0.06527554842161584,
            "false_positive_rate_ci95": [
              0.05494472253976211,
              0.0773897351934296
            ]
          },
          "false_positive_rate_ratio": 0.9483606557377049,
          "allow_miss_rate": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "miss_rate": 0.08493150684931507,
              "miss_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28010,
              "n_truly_positive": 1109,
              "miss_rate": 0.03959300249910746,
              "miss_rate_ci95": [
                0.03737177670557739,
                0.04194049691253792
              ]
            }
          }
        },
        "final_tier": {
          "with_identity_term": {
            "n_rows": 2035,
            "n_predicted_positive": 210,
            "n_false_positive": 13,
            "false_positive_rate": 0.06190476190476191,
            "false_positive_rate_ci95": [
              0.03653085811360575,
              0.10301859612885285
            ]
          },
          "without_identity_term": {
            "n_rows": 29879,
            "n_predicted_positive": 1872,
            "n_false_positive": 124,
            "false_positive_rate": 0.06623931623931624,
            "false_positive_rate_ci95": [
              0.05583810883933352,
              0.07841708511052611
            ]
          },
          "false_positive_rate_ratio": 0.9345622119815669,
          "allow_miss_rate": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "miss_rate": 0.08493150684931507,
              "miss_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28007,
              "n_truly_positive": 1108,
              "miss_rate": 0.03956153818688185,
              "miss_rate_ci95": [
                0.03734107077265871,
                0.0419082963746984
              ]
            }
          }
        }
      },
      "test": {
        "definition": "predicted positive = routed to auto_action or human_review; false positive = no label is truly positive",
        "model_tier": {
          "with_identity_term": {
            "n_rows": 4922,
            "n_predicted_positive": 810,
            "n_false_positive": 272,
            "false_positive_rate": 0.3358024691358025,
            "false_positive_rate_ci95": [
              0.30412168441237586,
              0.36903332987548954
            ]
          },
          "without_identity_term": {
            "n_rows": 59056,
            "n_predicted_positive": 5482,
            "n_false_positive": 1794,
            "false_positive_rate": 0.3272528274352426,
            "false_positive_rate_ci95": [
              0.3149568495943547,
              0.33979073759052253
            ]
          },
          "false_positive_rate_ratio": 1.0261254937583442,
          "allow_miss_rate": {
            "with_identity_term": {
              "n_allowed": 4112,
              "n_truly_positive": 240,
              "miss_rate": 0.058365758754863814,
              "miss_rate_ci95": [
                0.051604023318199826,
                0.06595187957918881
              ]
            },
            "without_identity_term": {
              "n_allowed": 53574,
              "n_truly_positive": 1777,
              "miss_rate": 0.033169074551088217,
              "miss_rate_ci95": [
                0.03168583339404747,
                0.03471925800202505
              ]
            }
          }
        },
        "final_tier": {
          "with_identity_term": {
            "n_rows": 4922,
            "n_predicted_positive": 810,
            "n_false_positive": 272,
            "false_positive_rate": 0.3358024691358025,
            "false_positive_rate_ci95": [
              0.30412168441237586,
              0.36903332987548954
            ]
          },
          "without_identity_term": {
            "n_rows": 59056,
            "n_predicted_positive": 5500,
            "n_false_positive": 1805,
            "false_positive_rate": 0.3281818181818182,
            "false_positive_rate_ci95": [
              0.3158961177494481,
              0.3407073629067409
            ]
          },
          "false_positive_rate_ratio": 1.0232208200813926,
          "allow_miss_rate": {
            "with_identity_term": {
              "n_allowed": 4112,
              "n_truly_positive": 240,
              "miss_rate": 0.058365758754863814,
              "miss_rate_ci95": [
                0.051604023318199826,
                0.06595187957918881
              ]
            },
            "without_identity_term": {
              "n_allowed": 53556,
              "n_truly_positive": 1770,
              "miss_rate": 0.03304951826125924,
              "miss_rate_ci95": [
                0.031568683248493615,
                0.03459733521520847
              ]
            }
          }
        }
      }
    },
    "subgroup_thresholds": null,
    "thresholds": {
      "auto_action": {
        "toxic": 0.9892913517285165,
        "severe_toxic": null,
        "obscene": 0.9983767822578123,
        "threat": null,
        "insult": 0.9998379195789239,
        "identity_hate": null
      },
      "human_review": {
        "toxic": 0.6576420429168361,
        "severe_toxic": null,
        "obscene": 0.6573428758151054,
        "threat": null,
        "insult": 0.9175983962802222,
        "identity_hate": null
      }
    },
    "identity_summary": {
      "selection_identity_predicted_positive_model": 210,
      "selection_identity_predicted_positive_final": 210,
      "selection_identity_moved_to_positive_by_rules": 0,
      "selection_identity_allowed": 1825,
      "selection_identity_allowed_truly_positive": 155,
      "test_model_tier_predicted_positive": {
        "ratio": 1.0261254937583442,
        "ci95": null,
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 272,
          "false_positive_rate": 0.3358024691358025,
          "false_positive_rate_ci95": [
            0.30412168441237586,
            0.36903332987548954
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5482,
          "n_false_positive": 1794,
          "false_positive_rate": 0.3272528274352426,
          "false_positive_rate_ci95": [
            0.3149568495943547,
            0.33979073759052253
          ]
        }
      },
      "test_final_tier_predicted_positive": {
        "ratio": 1.0232208200813926,
        "ci95": null,
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 272,
          "false_positive_rate": 0.3358024691358025,
          "false_positive_rate_ci95": [
            0.30412168441237586,
            0.36903332987548954
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5500,
          "n_false_positive": 1805,
          "false_positive_rate": 0.3281818181818182,
          "false_positive_rate_ci95": [
            0.3158961177494481,
            0.3407073629067409
          ]
        }
      },
      "test_by_term": null,
      "identity_terms": null
    }
  },
  "final": {
    "run_id": "20260922T112845423138Z-baseline",
    "git_commit": "3429f1fe41701d31de68ac936530d6aebfee1132",
    "git_dirty": false,
    "finished_utc": "2026-09-22T11:29:14+00:00",
    "data_sha256": {
      "train": "bd4084611bd27c939ba98e5e63bc3e5a2c1a4e99477dcba46c829e4c986c429d",
      "test": "c2513ce4abb98c4d1d216e3ca0d4377d57589a0989aa8c06a840509a16c786e8",
      "test_labels": "2a56dcbeba5c05f965a636f56cb5ae972bad60c3b952c239b49be18d7ab70f49"
    },
    "split_label_counts": {
      "train": {
        "toxic": 9182,
        "severe_toxic": 983,
        "obscene": 5140,
        "threat": 293,
        "insult": 4776,
        "identity_hate": 854,
        "rows": 95743
      },
      "calib": {
        "toxic": 3108,
        "severe_toxic": 303,
        "obscene": 1667,
        "threat": 106,
        "insult": 1558,
        "identity_hate": 281,
        "rows": 31914
      },
      "thresh": {
        "toxic": 3004,
        "severe_toxic": 309,
        "obscene": 1642,
        "threat": 79,
        "insult": 1543,
        "identity_hate": 270,
        "rows": 31914
      },
      "test_scored": {
        "toxic": 6090,
        "severe_toxic": 367,
        "obscene": 3691,
        "threat": 211,
        "insult": 3427,
        "identity_hate": 712,
        "rows": 63978
      }
    },
    "dependencies": {
      "python": "3.13.7",
      "review_router": "0.1.0",
      "numpy": "2.5.3",
      "pandas": "3.0.6",
      "scikit-learn": "1.9.1",
      "scipy": "1.18.1",
      "pyyaml": "6.0.3"
    },
    "policy_snapshot": {
      "version": 1,
      "tiers": {
        "allow": 1,
        "human_review": 2,
        "auto_action": 3
      },
      "severity_weights": {
        "threat": 10,
        "identity_hate": 6,
        "severe_toxic": 5,
        "insult": 2,
        "obscene": 2,
        "toxic": 1
      },
      "tier_precision_floors": {
        "human_review": 0.9,
        "auto_action": 0.99
      },
      "subgroup_thresholds": {
        "auto_action": [
          {
            "signal": "identity_term_present",
            "rationale": "auto_action is the only tier that enforces without appeal, and the documented failure mode on this corpus is false positives on benign identity mentions. The 0.99 floor is chosen on the whole selection split and says nothing about this slice: on the selection split of the run named in gates.eval_slice (population tier, auto_action, with an identity term) the slice held 102 of 921 auto-action rows with 2 wrong on the trigger label, a false discovery rate of 0.020 with a 95% interval of 0.005 to 0.069, undetermined against the 0.01 the floor allows. So the slice must clear the floor on its own, at or above the population threshold, before a comment in it is auto-actioned. This replaces rule R103 (see retired_rules): same intent, applied through the threshold machinery instead of a rule.\n"
          }
        ]
      },
      "rules": [
        {
          "id": "R101_rare_high_harm_never_auto",
          "description": "On the scored test set, 211 threat positives against 63,767 negatives would yield about 638 false positives and 106 true positives at an assumed 1% FPR and 50% recall, about 14% precision. This calculation does not rule out 99% precision at a different operating point. As a policy precaution, predictions with p_threat >= 0.30 require a human.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_threat",
              "op": ">=",
              "threshold": 0.3
            }
          ]
        },
        {
          "id": "R102_hierarchy_violation",
          "description": "severe_toxic is a subset of toxic in the observed training labels: no row has severe_toxic=1 with toxic=0. Independently fitted heads can violate this relation. Send such predictions to a human for review; the inconsistency alone does not establish distribution shift.\n",
          "action": "human_review",
          "conditions": [
            {
              "signal": "p_severe_toxic",
              "op": ">=",
              "threshold": 0.5
            },
            {
              "signal": "p_toxic",
              "op": "<",
              "threshold": 0.5
            }
          ]
        }
      ],
      "retired_rules": [
        {
          "id": "R103_identity_term_low_confidence",
          "retired_on": "2026-09-22",
          "was": "identity_term_present == 1 AND p_max < 0.95 -> human_review, meant to suppress auto-action on borderline identity mentions.\n",
          "why": "Measured on the threshold-selection split (31,914 rows). From run 20260922T100649877069Z, made with the rule still active: the auto-action thresholds chosen from the 0.99 floor were 0.989 (toxic), 0.998 (obscene) and 0.9998 (insult), all above the rule's 0.95 band, so the rule could never intercept an auto-action, and because rules override the model tier in both directions its only effect was to move 1,825 allow-tier comments with an identity term into the human queue, of which 1,670 (92%) had no positive label, giving a queue precision of 0.404 and concentrating human scrutiny on the speech it meant to protect. From the run named in gates.eval_slice (same split, same population thresholds, rule removed): the premise check on the population tier (classifier alone, false positive per each tier's own definition), where the false discovery rate among predicted positives was 0.071 with an identity term (15 of 210) and 0.068 without (128 of 1,869), ratio 1.04, 95% interval 0.62 to 1.75: neither confirmed nor excluded for that FDR comparison. It does not establish parity in FPR among truly negative comments. Two replacements were considered. An unconditional cap (identity mentions never auto-actioned) would have turned all 102 identity-term auto-action rows on that split, every one truly positive by some label, into human reviews to prevent about two wrong-trigger enforcements; rejected as a blunt override of the floor-defined tiers. Instead the floor is required on the slice itself (subgroup_thresholds above) and the concentration is measured and gated on every run.\n",
          "replaced_by": "subgroup_thresholds.auto_action[identity_term_present] and gates.fairness"
        }
      ],
      "gates": {
        "measured_on": "2026-09-22",
        "eval_slice": "Jigsaw scored test rows (63,978), run 20260922T110655042496Z-baseline, made on the working tree on top of commit e750939f82c7 (manifest git_dirty: true); re-run after committing to refresh.\n",
        "per_label_average_precision": {
          "toxic": {
            "floor": 0.732,
            "measured": 0.7522
          },
          "obscene": {
            "floor": 0.753,
            "measured": 0.7734
          },
          "insult": {
            "floor": 0.676,
            "measured": 0.6961
          },
          "identity_hate": {
            "floor": 0.46,
            "measured": 0.48
          },
          "severe_toxic": {
            "floor": 0.291,
            "measured": 0.3113
          },
          "threat": {
            "floor": 0.437,
            "measured": 0.458
          }
        },
        "routing": {
          "auto_action_precision_floor": 0.99,
          "human_review_precision_floor": 0.52,
          "harm_per_reviewer_hour_vs_fifo_min": 1.12,
          "high_risk_wait_p90_vs_fifo_max": 0.55,
          "queue_depth_p95_max": 39,
          "reviewer_utilization_min": 0.6
        },
        "consistency": {
          "hierarchy_violation_rate_max": 0.005
        },
        "fairness": {
          "identity_false_discovery_rate_ratio_max": 1.25
        },
        "min_predicted_positives_for_precision": 30
      }
    },
    "tiers": {
      "auto_action": {
        "n_predicted_positive": 2125,
        "coverage": 0.03321454249898403,
        "precision": 0.904,
        "precision_ci95": [
          0.8907357688824067,
          0.9158062086034517
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 4185,
        "coverage": 0.06541311075682266,
        "precision": 0.5507765830346476,
        "precision_ci95": [
          0.5356666476782052,
          0.5657933870764892
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57668,
        "coverage": 0.9013723467441933,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "tiers_model_only": {
      "auto_action": {
        "n_predicted_positive": 2270,
        "coverage": 0.03548094657538529,
        "precision": 0.9052863436123348,
        "precision_ci95": [
          0.89254659972632,
          0.9166566949067452
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 4022,
        "coverage": 0.06286535996748882,
        "precision": 0.538040775733466,
        "precision_ci95": [
          0.5226041345732986,
          0.553404819859058
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 57686,
        "coverage": 0.9016536934571259,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "threshold_selection_tiers": {
      "auto_action": {
        "n_predicted_positive": 845,
        "coverage": 0.026477408034091623,
        "precision": 0.991715976331361,
        "precision_ci95": [
          0.9829998680778467,
          0.9959815321652452
        ],
        "precision_note": null
      },
      "human_review": {
        "n_predicted_positive": 1237,
        "coverage": 0.03876041862505483,
        "precision": 0.8900565885206144,
        "precision_ci95": [
          0.8714018189629503,
          0.9062962447929256
        ],
        "precision_note": null
      },
      "allow": {
        "n_predicted_positive": 29832,
        "coverage": 0.9347621733408535,
        "precision": null,
        "precision_ci95": null,
        "precision_note": "allow is the fall-through; see false-negative counts"
      }
    },
    "per_label": {
      "toxic": {
        "positives": 6090,
        "negatives": 57888,
        "average_precision": 0.752184951704409,
        "roc_auc": 0.9569541029799007,
        "at_auto_action": {
          "threshold": 0.9892913517285165,
          "n_predicted_positive": 2365,
          "precision": 0.8976744186046511,
          "recall": 0.34860426929392446
        },
        "at_human_review": {
          "threshold": 0.6576420429168361,
          "n_predicted_positive": 6248,
          "precision": 0.6630921895006402,
          "recall": 0.680295566502463
        }
      },
      "severe_toxic": {
        "positives": 367,
        "negatives": 63611,
        "average_precision": 0.31126081415093715,
        "roc_auc": 0.981912884414067,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 3691,
        "negatives": 60287,
        "average_precision": 0.7733891183021634,
        "roc_auc": 0.971524362084933,
        "at_auto_action": {
          "threshold": 0.9983767822578123,
          "n_predicted_positive": 1033,
          "precision": 0.9515972894482091,
          "recall": 0.2663234895692224
        },
        "at_human_review": {
          "threshold": 0.6573428758151054,
          "n_predicted_positive": 2939,
          "precision": 0.7730520585233073,
          "recall": 0.6155513410999729
        }
      },
      "threat": {
        "positives": 211,
        "negatives": 63767,
        "average_precision": 0.45797903621474007,
        "roc_auc": 0.9917297028570469,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 3427,
        "negatives": 60551,
        "average_precision": 0.6960639271574238,
        "roc_auc": 0.9654037029086796,
        "at_auto_action": {
          "threshold": 0.9998379195789239,
          "n_predicted_positive": 131,
          "precision": 0.9847328244274809,
          "recall": 0.03764225269915378
        },
        "at_human_review": {
          "threshold": 0.9175983962802222,
          "n_predicted_positive": 1130,
          "precision": 0.8787610619469026,
          "recall": 0.2897578056609279
        }
      },
      "identity_hate": {
        "positives": 712,
        "negatives": 63266,
        "average_precision": 0.480036045041327,
        "roc_auc": 0.9739788700251516,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "threshold_selection_per_label": {
      "toxic": {
        "positives": 3004,
        "negatives": 28910,
        "average_precision": 0.8591610234939934,
        "roc_auc": 0.9711124588407661,
        "at_auto_action": {
          "threshold": 0.9892913517285165,
          "n_predicted_positive": 921,
          "precision": 0.990228013029316,
          "recall": 0.303595206391478
        },
        "at_human_review": {
          "threshold": 0.6576420429168361,
          "n_predicted_positive": 2073,
          "precision": 0.9001447178002895,
          "recall": 0.6211717709720372
        }
      },
      "severe_toxic": {
        "positives": 309,
        "negatives": 31605,
        "average_precision": 0.49211366123463823,
        "roc_auc": 0.9875885538982657,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "obscene": {
        "positives": 1642,
        "negatives": 30272,
        "average_precision": 0.8739067908950364,
        "roc_auc": 0.9852926342372399,
        "at_auto_action": {
          "threshold": 0.9983767822578123,
          "n_predicted_positive": 467,
          "precision": 0.9914346895074947,
          "recall": 0.28197320341047505
        },
        "at_human_review": {
          "threshold": 0.6573428758151054,
          "n_predicted_positive": 1177,
          "precision": 0.9005947323704333,
          "recall": 0.6455542021924482
        }
      },
      "threat": {
        "positives": 79,
        "negatives": 31835,
        "average_precision": 0.4586037397547445,
        "roc_auc": 0.9861349959144561,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      },
      "insult": {
        "positives": 1543,
        "negatives": 30371,
        "average_precision": 0.7743203146898984,
        "roc_auc": 0.9778902312262654,
        "at_auto_action": {
          "threshold": 0.9998379195789239,
          "n_predicted_positive": 51,
          "precision": 1.0,
          "recall": 0.03305249513933895
        },
        "at_human_review": {
          "threshold": 0.9175983962802222,
          "n_predicted_positive": 526,
          "precision": 0.9011406844106464,
          "recall": 0.30719377835385614
        }
      },
      "identity_hate": {
        "positives": 270,
        "negatives": 31644,
        "average_precision": 0.41474278947852444,
        "roc_auc": 0.9700926277054454,
        "at_auto_action": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        },
        "at_human_review": {
          "threshold": null,
          "n_predicted_positive": 0,
          "precision": null,
          "recall": null
        }
      }
    },
    "rules": {
      "matched_per_rule": {
        "R101_rare_high_harm_never_auto": 267,
        "R102_hierarchy_violation": 0
      },
      "n_rows_with_any_rule": 267,
      "n_added_to_queue_from_allow": 18,
      "n_added_to_queue_from_allow_true_positive": 7,
      "n_downgraded_from_auto_action": 145,
      "note": "rule-forced queue entries are included in every tier and simulation metric"
    },
    "consistency": {
      "hierarchy_violation_rate": 0.0,
      "n_violations": 0,
      "n_rows": 63978
    },
    "allow_false_negatives": {
      "toxic": 1914,
      "severe_toxic": 6,
      "obscene": 722,
      "threat": 24,
      "insult": 695,
      "identity_hate": 140
    },
    "simulation": {
      "assumptions": {
        "reviewers": 4,
        "handle_minutes": 2.0,
        "horizon_hours": 8.0,
        "capacity_per_hour": 120.0,
        "arrivals": "Poisson; jobs sampled with replacement from the human_review set",
        "handle_time": "deterministic",
        "harm_proxy": "max severity weight over TRUE labels (0 if clean); not real-world harm",
        "high_risk": "harm proxy >= 5.0",
        "loads_per_hour": [
          60.0,
          108.0,
          180.0
        ],
        "seeds": [
          1,
          2,
          3,
          4,
          5
        ],
        "queued_jobs": 4185,
        "queued_high_risk": 429
      },
      "summary": {
        "fifo@60": {
          "strategy": "fifo",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_handled": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 678.0,
            "std": 42.322570810384384
          },
          "harm_handled": {
            "mean": 676.4,
            "std": 41.93137250317476
          },
          "harm_per_reviewer_hour": {
            "mean": 21.1375,
            "std": 1.3103553907242111
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.2973983211717983,
            "std": 0.1573110257581183
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.38719307304028,
            "std": 0.24334294086925154
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "prob@60": {
          "strategy": "prob",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_handled": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 678.0,
            "std": 42.322570810384384
          },
          "harm_handled": {
            "mean": 676.4,
            "std": 41.93137250317476
          },
          "harm_per_reviewer_hour": {
            "mean": 21.1375,
            "std": 1.3103553907242111
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.25109017542825013,
            "std": 0.13941110208738772
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.32348396160840487,
            "std": 0.2178949429396602
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "severity@60": {
          "strategy": "severity",
          "load_per_hour": 60.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 469.8,
            "std": 27.981422408448076
          },
          "n_handled": {
            "mean": 467.8,
            "std": 26.49830183238164
          },
          "high_risk_arrived": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_handled": {
            "mean": 42.6,
            "std": 5.851495535331118
          },
          "high_risk_unhandled": {
            "mean": 0.0,
            "std": 0.0
          },
          "harm_arrived": {
            "mean": 678.0,
            "std": 42.322570810384384
          },
          "harm_handled": {
            "mean": 676.4,
            "std": 41.93137250317476
          },
          "harm_per_reviewer_hour": {
            "mean": 21.1375,
            "std": 1.3103553907242111
          },
          "wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "wait_p90": {
            "mean": 0.25382884715320103,
            "std": 0.13578485033201643
          },
          "high_risk_wait_p50": {
            "mean": 0.0,
            "std": 0.0
          },
          "high_risk_wait_p90": {
            "mean": 0.32062875646416805,
            "std": 0.2130514416845895
          },
          "backlog_end": {
            "mean": 0.4,
            "std": 0.8000000000000002
          },
          "queue_depth_p95": {
            "mean": 0.6099999999999909,
            "std": 0.4779121258139515
          },
          "reviewer_utilization": {
            "mean": 0.48816964253898903,
            "std": 0.028244633139592905
          }
        },
        "fifo@108": {
          "strategy": "fifo",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 89.0,
            "std": 4.09878030638384
          },
          "high_risk_handled": {
            "mean": 88.2,
            "std": 4.578209256903839
          },
          "high_risk_unhandled": {
            "mean": 0.8,
            "std": 0.7483314773547883
          },
          "harm_arrived": {
            "mean": 1278.0,
            "std": 39.67366884975475
          },
          "harm_handled": {
            "mean": 1269.0,
            "std": 41.4584128977461
          },
          "harm_per_reviewer_hour": {
            "mean": 39.65625,
            "std": 1.2955754030545656
          },
          "wait_p50": {
            "mean": 1.6465033420211455,
            "std": 0.978802764200094
          },
          "wait_p90": {
            "mean": 5.989271534385042,
            "std": 2.8671244892364833
          },
          "high_risk_wait_p50": {
            "mean": 1.5044494397173822,
            "std": 0.9216398193343448
          },
          "high_risk_wait_p90": {
            "mean": 6.178469239278163,
            "std": 2.7615107730049506
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "prob@108": {
          "strategy": "prob",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 89.0,
            "std": 4.09878030638384
          },
          "high_risk_handled": {
            "mean": 88.2,
            "std": 4.578209256903839
          },
          "high_risk_unhandled": {
            "mean": 0.8,
            "std": 0.7483314773547883
          },
          "harm_arrived": {
            "mean": 1278.0,
            "std": 39.67366884975475
          },
          "harm_handled": {
            "mean": 1269.2,
            "std": 41.40241538847703
          },
          "harm_per_reviewer_hour": {
            "mean": 39.6625,
            "std": 1.2938254808899072
          },
          "wait_p50": {
            "mean": 0.40344718180251676,
            "std": 0.07215084118027483
          },
          "wait_p90": {
            "mean": 4.230141362794436,
            "std": 1.1790872336019191
          },
          "high_risk_wait_p50": {
            "mean": 0.3121040843701849,
            "std": 0.050383584223618216
          },
          "high_risk_wait_p90": {
            "mean": 1.3749401504841048,
            "std": 0.20229435005999408
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "severity@108": {
          "strategy": "severity",
          "load_per_hour": 108.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 872.4,
            "std": 20.37253052519495
          },
          "n_handled": {
            "mean": 867.4,
            "std": 20.254382241875458
          },
          "high_risk_arrived": {
            "mean": 89.0,
            "std": 4.09878030638384
          },
          "high_risk_handled": {
            "mean": 88.0,
            "std": 4.69041575982343
          },
          "high_risk_unhandled": {
            "mean": 1.0,
            "std": 1.0954451150103321
          },
          "harm_arrived": {
            "mean": 1278.0,
            "std": 39.67366884975475
          },
          "harm_handled": {
            "mean": 1267.8,
            "std": 41.873141749813804
          },
          "harm_per_reviewer_hour": {
            "mean": 39.61875,
            "std": 1.3085356796816814
          },
          "wait_p50": {
            "mean": 0.40958137093132335,
            "std": 0.0751885813087163
          },
          "wait_p90": {
            "mean": 4.357974726656176,
            "std": 1.588899677988441
          },
          "high_risk_wait_p50": {
            "mean": 0.2952104594656049,
            "std": 0.07051466045318901
          },
          "high_risk_wait_p90": {
            "mean": 1.0724504222628188,
            "std": 0.2702835868145261
          },
          "backlog_end": {
            "mean": 1.2,
            "std": 1.5999999999999999
          },
          "queue_depth_p95": {
            "mean": 14.219999999999981,
            "std": 6.199403197082773
          },
          "reviewer_utilization": {
            "mean": 0.9054738607875965,
            "std": 0.02138288856943648
          }
        },
        "fifo@180": {
          "strategy": "fifo",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 147.6,
            "std": 6.974238309665077
          },
          "high_risk_handled": {
            "mean": 100.0,
            "std": 3.794733192202055
          },
          "high_risk_unhandled": {
            "mean": 47.6,
            "std": 6.374950980203691
          },
          "harm_arrived": {
            "mean": 2135.4,
            "std": 74.71171260250965
          },
          "harm_handled": {
            "mean": 1428.6,
            "std": 29.05580836941213
          },
          "harm_per_reviewer_hour": {
            "mean": 44.64375,
            "std": 0.9079940115441291
          },
          "wait_p50": {
            "mean": 78.75870506496214,
            "std": 2.4070498005840677
          },
          "wait_p90": {
            "mean": 139.86341084347154,
            "std": 7.777914356926516
          },
          "high_risk_wait_p50": {
            "mean": 81.84985993728016,
            "std": 5.411182909833444
          },
          "high_risk_wait_p90": {
            "mean": 143.27504698240756,
            "std": 9.272825979169033
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "prob@180": {
          "strategy": "prob",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 147.6,
            "std": 6.974238309665077
          },
          "high_risk_handled": {
            "mean": 124.6,
            "std": 3.1368774282716245
          },
          "high_risk_unhandled": {
            "mean": 23.0,
            "std": 5.89915248150105
          },
          "harm_arrived": {
            "mean": 2135.4,
            "std": 74.71171260250965
          },
          "harm_handled": {
            "mean": 1699.4,
            "std": 33.499850745936165
          },
          "harm_per_reviewer_hour": {
            "mean": 53.10625,
            "std": 1.0468703358105051
          },
          "wait_p50": {
            "mean": 0.7264318424958752,
            "std": 0.2607890434830117
          },
          "wait_p90": {
            "mean": 11.015903736606868,
            "std": 2.481762611975994
          },
          "high_risk_wait_p50": {
            "mean": 0.4865574802309965,
            "std": 0.23382240605562732
          },
          "high_risk_wait_p90": {
            "mean": 3.0966863302941245,
            "std": 1.4395859044901338
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        },
        "severity@180": {
          "strategy": "severity",
          "load_per_hour": 180.0,
          "n_seeds": 5,
          "n_arrivals": {
            "mean": 1422.0,
            "std": 44.04997162314637
          },
          "n_handled": {
            "mean": 953.2,
            "std": 3.9191835884530852
          },
          "high_risk_arrived": {
            "mean": 147.6,
            "std": 6.974238309665077
          },
          "high_risk_handled": {
            "mean": 133.8,
            "std": 5.344155686354955
          },
          "high_risk_unhandled": {
            "mean": 13.8,
            "std": 2.3151673805580453
          },
          "harm_arrived": {
            "mean": 2135.4,
            "std": 74.71171260250965
          },
          "harm_handled": {
            "mean": 1793.4,
            "std": 40.47517757836276
          },
          "harm_per_reviewer_hour": {
            "mean": 56.04375,
            "std": 1.2648492993238363
          },
          "wait_p50": {
            "mean": 0.7220379852418344,
            "std": 0.2523576372916612
          },
          "wait_p90": {
            "mean": 11.493689466168354,
            "std": 1.3506245485526764
          },
          "high_risk_wait_p50": {
            "mean": 0.47031773619605133,
            "std": 0.22779038663578435
          },
          "high_risk_wait_p90": {
            "mean": 2.1594126451790934,
            "std": 1.0528467145478622
          },
          "backlog_end": {
            "mean": 464.8,
            "std": 41.498915648484115
          },
          "queue_depth_p95": {
            "mean": 441.5299999999999,
            "std": 32.43828910408187
          },
          "reviewer_utilization": {
            "mean": 0.9955291947317481,
            "std": 0.004382166265363362
          }
        }
      },
      "paired": {
        "severity_vs_fifo@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.0665643165761119,
            "min_diff": -0.1417200425648838,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 3,
            "n_seeds_tied": 2
          },
          "wait_p90": {
            "mean_diff": -0.0435694740185973,
            "min_diff": -0.09729269536403606,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          }
        },
        "prob_vs_fifo@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.06370911143187512,
            "min_diff": -0.10622039468485867,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          },
          "wait_p90": {
            "mean_diff": -0.046308145743548176,
            "min_diff": -0.09729269536403606,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 1
          }
        },
        "severity_vs_prob@60": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.002855205144236783,
            "min_diff": -0.11824517316192296,
            "max_diff": 0.10396914744073904,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 3
          },
          "wait_p90": {
            "mean_diff": 0.0027386717249508865,
            "min_diff": -0.0004195643103005464,
            "max_diff": 0.011767301226240007,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 2
          }
        },
        "severity_vs_fifo@108": {
          "high_risk_handled": {
            "mean_diff": -0.2,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "harm_per_reviewer_hour": {
            "mean_diff": -0.0375,
            "min_diff": -0.1875,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": 0.2,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_wait_p90": {
            "mean_diff": -5.106018817015344,
            "min_diff": -10.661270828149638,
            "max_diff": -3.157660582581379,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -1.631296807728864,
            "min_diff": -4.160141901500318,
            "max_diff": -0.040877117854664746,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "prob_vs_fifo@108": {
          "high_risk_handled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 0.00625,
            "min_diff": 0.0,
            "max_diff": 0.03125,
            "n_seeds": 5,
            "n_seeds_first_better": 1,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": 0.0,
            "min_diff": 0.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 5
          },
          "high_risk_wait_p90": {
            "mean_diff": -4.8035290887940585,
            "min_diff": -10.162569110908812,
            "max_diff": -2.6558222893054477,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -1.7591301715906051,
            "min_diff": -5.109926177670329,
            "max_diff": 0.021149172336258903,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_prob@108": {
          "high_risk_handled": {
            "mean_diff": -0.2,
            "min_diff": -1.0,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "harm_per_reviewer_hour": {
            "mean_diff": -0.04375,
            "min_diff": -0.21875,
            "max_diff": 0.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_unhandled": {
            "mean_diff": 0.2,
            "min_diff": 0.0,
            "max_diff": 1.0,
            "n_seeds": 5,
            "n_seeds_first_better": 0,
            "n_seeds_tied": 4
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.3024897282212861,
            "min_diff": -0.5303603873138942,
            "max_diff": -0.08345127637205652,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": 0.12783336386174088,
            "min_diff": -0.18027836704729516,
            "max_diff": 0.9497842761700106,
            "n_seeds": 5,
            "n_seeds_first_better": 4,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_fifo@180": {
          "high_risk_handled": {
            "mean_diff": 33.8,
            "min_diff": 28.0,
            "max_diff": 40.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 11.4,
            "min_diff": 10.28125,
            "max_diff": 13.125,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -33.8,
            "min_diff": -40.0,
            "max_diff": -28.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -141.11563433722844,
            "min_diff": -156.63387665219773,
            "max_diff": -132.677908463992,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -128.3697213773032,
            "min_diff": -143.19857519732355,
            "max_diff": -123.26967499659091,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "prob_vs_fifo@180": {
          "high_risk_handled": {
            "mean_diff": 24.6,
            "min_diff": 21.0,
            "max_diff": 28.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 8.4625,
            "min_diff": 7.28125,
            "max_diff": 10.03125,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -24.6,
            "min_diff": -28.0,
            "max_diff": -21.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -140.1783606521134,
            "min_diff": -158.33848386719262,
            "max_diff": -131.48145414142428,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": -128.8475071068647,
            "min_diff": -147.29121960550512,
            "max_diff": -123.21500676401463,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          }
        },
        "severity_vs_prob@180": {
          "high_risk_handled": {
            "mean_diff": 9.2,
            "min_diff": 5.0,
            "max_diff": 15.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "harm_per_reviewer_hour": {
            "mean_diff": 2.9375,
            "min_diff": 2.15625,
            "max_diff": 3.875,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_unhandled": {
            "mean_diff": -9.2,
            "min_diff": -15.0,
            "max_diff": -5.0,
            "n_seeds": 5,
            "n_seeds_first_better": 5,
            "n_seeds_tied": 0
          },
          "high_risk_wait_p90": {
            "mean_diff": -0.9372736851150313,
            "min_diff": -4.469671299997692,
            "max_diff": 1.7046072149948968,
            "n_seeds": 5,
            "n_seeds_first_better": 3,
            "n_seeds_tied": 0
          },
          "wait_p90": {
            "mean_diff": 0.4777857295614879,
            "min_diff": -1.8204841135101741,
            "max_diff": 4.09264440818157,
            "n_seeds": 5,
            "n_seeds_first_better": 2,
            "n_seeds_tied": 0
          }
        }
      },
      "harm_per_reviewer_hour": {
        "router": 56.04375,
        "fifo": 44.64375,
        "load_per_hour": 180.0
      },
      "high_risk_wait_p90": {
        "router": 1.0724504222628188,
        "fifo": 6.178469239278163,
        "load_per_hour": 108.0
      },
      "queue_depth_p95": 14.219999999999981,
      "reviewer_utilization": 0.9054738607875965,
      "primary": {
        "load_per_hour": 108.0,
        "strategy": "severity",
        "note": "queue-clearing gates read this load; the router strategy is compared against fifo at the same load for time-to-action"
      },
      "thesis": {
        "load_per_hour": 180.0,
        "strategy": "severity",
        "note": "harm handled per reviewer-hour only differs between orderings when the queue does not clear, so the FIFO comparison reads this load"
      }
    },
    "identity_false_positives": {
      "threshold_selection": {
        "definition": "identity term = any whole-word match of review_router.signals.IDENTITY_TERMS; predicted positive = routed to auto_action or human_review; false positive = the tier's own definition (triggering label not true for auto_action, no label true for human_review); FDR = false positives / predicted positives; FDR ratio = with-term FDR / without-term FDR with a 95% log-ratio interval. clean_negative_false_positives separately reports FPR = clean comments routed to the tier / all clean comments in that subgroup. allow_false_omission = truly positive allowed comments / all allowed comments.",
        "identity_terms": [
          "african",
          "africans",
          "arab",
          "arabs",
          "asian",
          "asians",
          "atheist",
          "atheists",
          "autistic",
          "bisexual",
          "black",
          "blacks",
          "buddhist",
          "buddhists",
          "catholic",
          "catholics",
          "chinese",
          "christian",
          "christians",
          "disability",
          "disabled",
          "elderly",
          "female",
          "females",
          "feminist",
          "feminists",
          "gay",
          "gays",
          "heterosexual",
          "hindu",
          "hindus",
          "hispanic",
          "homosexual",
          "homosexuals",
          "immigrant",
          "immigrants",
          "indian",
          "indians",
          "islam",
          "islamic",
          "jew",
          "jewish",
          "jews",
          "latina",
          "latino",
          "lesbian",
          "lesbians",
          "lgbt",
          "male",
          "males",
          "mexican",
          "mexicans",
          "muslim",
          "muslims",
          "queer",
          "refugee",
          "refugees",
          "trans",
          "transgender",
          "white",
          "whites",
          "woman",
          "women"
        ],
        "population_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 15,
              "false_discovery_rate": 0.07142857142857142,
              "false_discovery_rate_ci95": [
                0.043761281311323466,
                0.11449361957803818
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1869,
              "n_false_positive": 128,
              "false_discovery_rate": 0.06848582129481005,
              "false_discovery_rate_ci95": [
                0.05789760516266364,
                0.0808442288826522
              ]
            },
            "false_discovery_rate_ratio": 1.04296875,
            "false_discovery_rate_ratio_ci95": [
              0.6228506211741813,
              1.7464601888425535
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 102,
              "n_false_positive": 2,
              "false_discovery_rate": 0.0196078431372549,
              "false_discovery_rate_ci95": [
                0.005393752121060413,
                0.06869308458902693
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 819,
              "n_false_positive": 7,
              "false_discovery_rate": 0.008547008547008548,
              "false_discovery_rate_ci95": [
                0.004146222644830832,
                0.01753651913820058
              ]
            },
            "false_discovery_rate_ratio": 2.2941176470588234,
            "false_discovery_rate_ratio_ci95": [
              0.4830740021539949,
              10.894760958112945
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 108,
              "n_false_positive": 13,
              "false_discovery_rate": 0.12037037037037036,
              "false_discovery_rate_ci95": [
                0.0717106396469066,
                0.19510865240979408
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1050,
              "n_false_positive": 121,
              "false_discovery_rate": 0.11523809523809524,
              "false_discovery_rate_ci95": [
                0.09731124556433196,
                0.13597001017812918
              ]
            },
            "false_discovery_rate_ratio": 1.044536271808999,
            "false_discovery_rate_ratio_ci95": [
              0.610730821033624,
              1.7864761127956477
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 122,
                "false_positive_rate": 0.004514672686230248,
                "false_positive_rate_ci95": [
                  0.0037827546821183845,
                  0.005387442259257935
                ]
              },
              "false_positive_rate_ratio": 1.7109328579916816,
              "false_positive_rate_ratio_ci95": [
                0.9678651372274237,
                3.024482577129694
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 1,
                "false_positive_rate": 3.700551382155941e-05,
                "false_positive_rate_ci95": [
                  6.53241165272662e-06,
                  0.00020960305094593518
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.21799392956802496,
                131.25872477694202
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 121,
                "false_positive_rate": 0.004477667172408689,
                "false_positive_rate_ci95": [
                  0.0037490088495444516,
                  0.005347187583742797
                ]
              },
              "false_positive_rate_ratio": 1.7250727989668193,
              "false_positive_rate_ratio_ci95": [
                0.9756412168557583,
                3.050174705949491
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28010,
              "n_truly_positive": 1109,
              "false_omission_rate": 0.03959300249910746,
              "false_omission_rate_ci95": [
                0.03737177670557739,
                0.04194049691253792
              ]
            }
          }
        },
        "model_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 13,
              "false_discovery_rate": 0.06190476190476191,
              "false_discovery_rate_ci95": [
                0.03653085811360575,
                0.10301859612885285
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1869,
              "n_false_positive": 128,
              "false_discovery_rate": 0.06848582129481005,
              "false_discovery_rate_ci95": [
                0.05789760516266364,
                0.0808442288826522
              ]
            },
            "false_discovery_rate_ratio": 0.9039062500000001,
            "false_discovery_rate_ratio_ci95": [
              0.5202516115413566,
              1.5704833789334889
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 71,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0,
              "false_discovery_rate_ci95": [
                0.0,
                0.051327952965064906
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 819,
              "n_false_positive": 7,
              "false_discovery_rate": 0.008547008547008548,
              "false_discovery_rate_ci95": [
                0.004146222644830832,
                0.01753651913820058
              ]
            },
            "false_discovery_rate_ratio": 0.0,
            "false_discovery_rate_ratio_ci95": [
              0.043806736488549325,
              13.159497122585343
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 139,
              "n_false_positive": 13,
              "false_discovery_rate": 0.09352517985611511,
              "false_discovery_rate_ci95": [
                0.05547239270536431,
                0.15344075558511086
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1050,
              "n_false_positive": 121,
              "false_discovery_rate": 0.11523809523809524,
              "false_discovery_rate_ci95": [
                0.09731124556433196,
                0.13597001017812918
              ]
            },
            "false_discovery_rate_ratio": 0.8115821392472798,
            "false_discovery_rate_ratio_ci95": [
              0.4710542129784767,
              1.3982797533652176
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 122,
                "false_positive_rate": 0.004514672686230248,
                "false_positive_rate_ci95": [
                  0.0037827546821183845,
                  0.005387442259257935
                ]
              },
              "false_positive_rate_ratio": 1.7109328579916816,
              "false_positive_rate_ratio_ci95": [
                0.9678651372274237,
                3.024482577129694
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 1,
                "false_positive_rate": 3.700551382155941e-05,
                "false_positive_rate_ci95": [
                  6.53241165272662e-06,
                  0.00020960305094593518
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.21799392956802496,
                131.25872477694202
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 121,
                "false_positive_rate": 0.004477667172408689,
                "false_positive_rate_ci95": [
                  0.0037490088495444516,
                  0.005347187583742797
                ]
              },
              "false_positive_rate_ratio": 1.7250727989668193,
              "false_positive_rate_ratio_ci95": [
                0.9756412168557583,
                3.050174705949491
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28010,
              "n_truly_positive": 1109,
              "false_omission_rate": 0.03959300249910746,
              "false_omission_rate_ci95": [
                0.03737177670557739,
                0.04194049691253792
              ]
            }
          }
        },
        "final_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_truly_positive_any_label": 352,
              "rate": 0.17297297297297298
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_truly_positive_any_label": 2856,
              "rate": 0.09558552829746644
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 210,
              "n_false_positive": 13,
              "false_discovery_rate": 0.06190476190476191,
              "false_discovery_rate_ci95": [
                0.03653085811360575,
                0.10301859612885285
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1872,
              "n_false_positive": 130,
              "false_discovery_rate": 0.06944444444444445,
              "false_discovery_rate_ci95": [
                0.058788657698158976,
                0.08186366536044554
              ]
            },
            "false_discovery_rate_ratio": 0.8914285714285715,
            "false_discovery_rate_ratio_ci95": [
              0.5132829778657895,
              1.5481614084754693
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 64,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0,
              "false_discovery_rate_ci95": [
                0.0,
                0.05662406063551054
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 781,
              "n_false_positive": 7,
              "false_discovery_rate": 0.008962868117797696,
              "false_discovery_rate_ci95": [
                0.004348265335958092,
                0.01838429890284628
              ]
            },
            "false_discovery_rate_ratio": 0.0,
            "false_discovery_rate_ratio_ci95": [
              0.04632418029486149,
              13.886619362619607
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 2035,
              "n_predicted_positive": 146,
              "n_false_positive": 13,
              "false_discovery_rate": 0.08904109589041095,
              "false_discovery_rate_ci95": [
                0.05277427127462007,
                0.14637928144763634
              ]
            },
            "without_identity_term": {
              "n_rows": 29879,
              "n_predicted_positive": 1091,
              "n_false_positive": 123,
              "false_discovery_rate": 0.11274060494958753,
              "false_discovery_rate_ci95": [
                0.09531584653839371,
                0.13288290941352549
              ]
            },
            "false_discovery_rate_ratio": 0.7897872814344582,
            "false_discovery_rate_ratio_ci95": [
              0.458005955163958,
              1.361912313328624
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 124,
                "false_positive_rate": 0.004588683713873367,
                "false_positive_rate_ci95": [
                  0.003850286352596687,
                  0.005467911604957776
                ]
              },
              "false_positive_rate_ratio": 1.6833371667337511,
              "false_positive_rate_ratio_ci95": [
                0.9526791141311439,
                2.974374030957439
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 0,
                "false_positive_rate": 0.0,
                "false_positive_rate_ci95": [
                  0.0,
                  0.002277308789792038
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 1,
                "false_positive_rate": 3.700551382155941e-05,
                "false_positive_rate_ci95": [
                  6.53241165272662e-06,
                  0.00020960305094593518
                ]
              },
              "false_positive_rate_ratio": 0.0,
              "false_positive_rate_ratio_ci95": [
                0.21799392956802496,
                131.25872477694202
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 1683,
                "n_false_positive": 13,
                "false_positive_rate": 0.007724301841948901,
                "false_positive_rate_ci95": [
                  0.004519683103302885,
                  0.013171048129427603
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 27023,
                "n_false_positive": 123,
                "false_positive_rate": 0.0045516782000518076,
                "false_positive_rate_ci95": [
                  0.0038165139034532286,
                  0.0054276835460121626
                ]
              },
              "false_positive_rate_ratio": 1.6970228347559768,
              "false_positive_rate_ratio_ci95": [
                0.9602120685908566,
                2.9992192307159202
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 1825,
              "n_truly_positive": 155,
              "false_omission_rate": 0.08493150684931507,
              "false_omission_rate_ci95": [
                0.07299686563215325,
                0.09860984054634357
              ]
            },
            "without_identity_term": {
              "n_allowed": 28007,
              "n_truly_positive": 1108,
              "false_omission_rate": 0.03956153818688185,
              "false_omission_rate_ci95": [
                0.03734107077265871,
                0.0419082963746984
              ]
            }
          },
          "by_term": [
            {
              "term": "gay",
              "n_predicted_positive": 90,
              "n_false_positive": 6,
              "false_discovery_rate": 0.06666666666666667
            },
            {
              "term": "black",
              "n_predicted_positive": 16,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "jew",
              "n_predicted_positive": 16,
              "n_false_positive": 1,
              "false_discovery_rate": 0.0625
            },
            {
              "term": "white",
              "n_predicted_positive": 16,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "homosexual",
              "n_predicted_positive": 13,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "jews",
              "n_predicted_positive": 13,
              "n_false_positive": 3,
              "false_discovery_rate": 0.23076923076923078
            },
            {
              "term": "women",
              "n_predicted_positive": 10,
              "n_false_positive": 1,
              "false_discovery_rate": 0.1
            },
            {
              "term": "indian",
              "n_predicted_positive": 9,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "muslim",
              "n_predicted_positive": 7,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "woman",
              "n_predicted_positive": 6,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            },
            {
              "term": "arab",
              "n_predicted_positive": 5,
              "n_false_positive": 1,
              "false_discovery_rate": 0.2
            },
            {
              "term": "asian",
              "n_predicted_positive": 5,
              "n_false_positive": 0,
              "false_discovery_rate": 0.0
            }
          ]
        }
      },
      "test": {
        "definition": "identity term = any whole-word match of review_router.signals.IDENTITY_TERMS; predicted positive = routed to auto_action or human_review; false positive = the tier's own definition (triggering label not true for auto_action, no label true for human_review); FDR = false positives / predicted positives; FDR ratio = with-term FDR / without-term FDR with a 95% log-ratio interval. clean_negative_false_positives separately reports FPR = clean comments routed to the tier / all clean comments in that subgroup. allow_false_omission = truly positive allowed comments / all allowed comments.",
        "identity_terms": [
          "african",
          "africans",
          "arab",
          "arabs",
          "asian",
          "asians",
          "atheist",
          "atheists",
          "autistic",
          "bisexual",
          "black",
          "blacks",
          "buddhist",
          "buddhists",
          "catholic",
          "catholics",
          "chinese",
          "christian",
          "christians",
          "disability",
          "disabled",
          "elderly",
          "female",
          "females",
          "feminist",
          "feminists",
          "gay",
          "gays",
          "heterosexual",
          "hindu",
          "hindus",
          "hispanic",
          "homosexual",
          "homosexuals",
          "immigrant",
          "immigrants",
          "indian",
          "indians",
          "islam",
          "islamic",
          "jew",
          "jewish",
          "jews",
          "latina",
          "latino",
          "lesbian",
          "lesbians",
          "lgbt",
          "male",
          "males",
          "mexican",
          "mexicans",
          "muslim",
          "muslims",
          "queer",
          "refugee",
          "refugees",
          "trans",
          "transgender",
          "white",
          "whites",
          "woman",
          "women"
        ],
        "population_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 810,
              "n_false_positive": 275,
              "false_discovery_rate": 0.3395061728395062,
              "false_discovery_rate_ci95": [
                0.3077209890410385,
                0.3728064685335667
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5482,
              "n_false_positive": 1800,
              "false_discovery_rate": 0.32834731849689897,
              "false_discovery_rate_ci95": [
                0.31603995744110297,
                0.3408950790329937
              ]
            },
            "false_discovery_rate_ratio": 1.0339849108367627,
            "false_discovery_rate_ratio_ci95": [
              0.9325559410243155,
              1.1464457506579027
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 322,
              "n_false_positive": 35,
              "false_discovery_rate": 0.10869565217391304,
              "false_discovery_rate_ci95": [
                0.07919957042736245,
                0.14741818145361116
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 2045,
              "n_false_positive": 202,
              "false_discovery_rate": 0.09877750611246944,
              "false_discovery_rate_ci95": [
                0.08658859263384615,
                0.11247095739301864
              ]
            },
            "false_discovery_rate_ratio": 1.1004089539388722,
            "false_discovery_rate_ratio_ci95": [
              0.7839716194909653,
              1.5445710479865116
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 488,
              "n_false_positive": 240,
              "false_discovery_rate": 0.4918032786885246,
              "false_discovery_rate_ci95": [
                0.44768506900152594,
                0.5360495270648646
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3437,
              "n_false_positive": 1598,
              "false_discovery_rate": 0.46494035496072156,
              "false_discovery_rate_ci95": [
                0.4483140605672861,
                0.4816449326383017
              ]
            },
            "false_discovery_rate_ratio": 1.0577771394571083,
            "false_discovery_rate_ratio_ci95": [
              0.9599352294588238,
              1.1655916382909013
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 272,
                "false_positive_rate": 0.06563706563706563,
                "false_positive_rate_ci95": [
                  0.058492120730319265,
                  0.07358656750118629
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1794,
                "false_positive_rate": 0.033475770185292304,
                "false_positive_rate_ci95": [
                  0.031985987684261995,
                  0.03503242978110185
                ]
              },
              "false_positive_rate_ratio": 1.9607335476900694,
              "false_positive_rate_ratio_ci95": [
                1.7328449760526654,
                2.2185920253494404
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 32,
                "false_positive_rate": 0.007722007722007722,
                "false_positive_rate_ci95": [
                  0.005475285435204754,
                  0.010880561227168243
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 196,
                "false_positive_rate": 0.0036573305219159934,
                "false_positive_rate_ci95": [
                  0.0031806081975381414,
                  0.00420520446849971
                ]
              },
              "false_positive_rate_ratio": 2.1113781419903868,
              "false_positive_rate_ratio_ci95": [
                1.4549748785847052,
                3.0639138339014615
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 240,
                "false_positive_rate": 0.05791505791505792,
                "false_positive_rate_ci95": [
                  0.0512042095070135,
                  0.06544476651527441
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1598,
                "false_positive_rate": 0.029818439663376314,
                "false_positive_rate_ci95": [
                  0.02841176901892619,
                  0.03129251168757164
                ]
              },
              "false_positive_rate_ratio": 1.9422564885643734,
              "false_positive_rate_ratio_ci95": [
                1.7021607433894164,
                2.216218581012932
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4112,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058365758754863814,
              "false_omission_rate_ci95": [
                0.051604023318199826,
                0.06595187957918881
              ]
            },
            "without_identity_term": {
              "n_allowed": 53574,
              "n_truly_positive": 1777,
              "false_omission_rate": 0.033169074551088217,
              "false_omission_rate_ci95": [
                0.03168583339404747,
                0.03471925800202505
              ]
            }
          }
        },
        "model_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 810,
              "n_false_positive": 273,
              "false_discovery_rate": 0.337037037037037,
              "false_discovery_rate_ci95": [
                0.3053212063629801,
                0.37029128902046526
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5482,
              "n_false_positive": 1800,
              "false_discovery_rate": 0.32834731849689897,
              "false_discovery_rate_ci95": [
                0.31603995744110297,
                0.3408950790329937
              ]
            },
            "false_discovery_rate_ratio": 1.0264650205761316,
            "false_discovery_rate_ratio_ci95": [
              0.9253161169100481,
              1.138670794997926
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 225,
              "n_false_positive": 13,
              "false_discovery_rate": 0.057777777777777775,
              "false_discovery_rate_ci95": [
                0.034073070023627444,
                0.09632925754051938
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 2045,
              "n_false_positive": 202,
              "false_discovery_rate": 0.09877750611246944,
              "false_discovery_rate_ci95": [
                0.08658859263384615,
                0.11247095739301864
              ]
            },
            "false_discovery_rate_ratio": 0.5849284928492849,
            "false_discovery_rate_ratio_ci95": [
              0.33962197000956706,
              1.0074181647827376
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 585,
              "n_false_positive": 260,
              "false_discovery_rate": 0.4444444444444444,
              "false_discovery_rate_ci95": [
                0.40467041224149014,
                0.4849433385913918
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3437,
              "n_false_positive": 1598,
              "false_discovery_rate": 0.46494035496072156,
              "false_discovery_rate_ci95": [
                0.4483140605672861,
                0.4816449326383017
              ]
            },
            "false_discovery_rate_ratio": 0.9559171186204978,
            "false_discovery_rate_ratio_ci95": [
              0.8671669934988251,
              1.0537503670254176
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 272,
                "false_positive_rate": 0.06563706563706563,
                "false_positive_rate_ci95": [
                  0.058492120730319265,
                  0.07358656750118629
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1794,
                "false_positive_rate": 0.033475770185292304,
                "false_positive_rate_ci95": [
                  0.031985987684261995,
                  0.03503242978110185
                ]
              },
              "false_positive_rate_ratio": 1.9607335476900694,
              "false_positive_rate_ratio_ci95": [
                1.7328449760526654,
                2.2185920253494404
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 12,
                "false_positive_rate": 0.0028957528957528956,
                "false_positive_rate_ci95": [
                  0.0016573018346735045,
                  0.005054974696938445
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 196,
                "false_positive_rate": 0.0036573305219159934,
                "false_positive_rate_ci95": [
                  0.0031806081975381414,
                  0.00420520446849971
                ]
              },
              "false_positive_rate_ratio": 0.791766803246395,
              "false_positive_rate_ratio_ci95": [
                0.44242389996540127,
                1.4169548045936045
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 260,
                "false_positive_rate": 0.06274131274131274,
                "false_positive_rate_ci95": [
                  0.055755388041875194,
                  0.07053715811117375
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1598,
                "false_positive_rate": 0.029818439663376314,
                "false_positive_rate_ci95": [
                  0.02841176901892619,
                  0.03129251168757164
                ]
              },
              "false_positive_rate_ratio": 2.104111195944738,
              "false_positive_rate_ratio_ci95": [
                1.8527892050046,
                2.3895238124992226
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4112,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058365758754863814,
              "false_omission_rate_ci95": [
                0.051604023318199826,
                0.06595187957918881
              ]
            },
            "without_identity_term": {
              "n_allowed": 53574,
              "n_truly_positive": 1777,
              "false_omission_rate": 0.033169074551088217,
              "false_omission_rate_ci95": [
                0.03168583339404747,
                0.03471925800202505
              ]
            }
          }
        },
        "final_tier": {
          "base_rate": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_truly_positive_any_label": 778,
              "rate": 0.1580658268996343
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_truly_positive_any_label": 5465,
              "rate": 0.09253928474668112
            }
          },
          "predicted_positive": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 810,
              "n_false_positive": 273,
              "false_discovery_rate": 0.337037037037037,
              "false_discovery_rate_ci95": [
                0.3053212063629801,
                0.37029128902046526
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 5500,
              "n_false_positive": 1811,
              "false_discovery_rate": 0.3292727272727273,
              "false_discovery_rate_ci95": [
                0.3169757651349169,
                0.34180801088218515
              ]
            },
            "false_discovery_rate_ratio": 1.023580178743072,
            "false_discovery_rate_ratio_ci95": [
              0.9227630126443263,
              1.1354122000548104
            ]
          },
          "auto_action": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 205,
              "n_false_positive": 12,
              "false_discovery_rate": 0.05853658536585366,
              "false_discovery_rate_ci95": [
                0.033799019999132485,
                0.09951482980078147
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 1920,
              "n_false_positive": 192,
              "false_discovery_rate": 0.1,
              "false_discovery_rate_ci95": [
                0.08736938121151575,
                0.11422803061826353
              ]
            },
            "false_discovery_rate_ratio": 0.5853658536585366,
            "false_discovery_rate_ratio_ci95": [
              0.33265017363980887,
              1.0300706561494526
            ]
          },
          "human_review": {
            "with_identity_term": {
              "n_rows": 4922,
              "n_predicted_positive": 605,
              "n_false_positive": 261,
              "false_discovery_rate": 0.43140495867768597,
              "false_discovery_rate_ci95": [
                0.3924948564625459,
                0.47118065579230817
              ]
            },
            "without_identity_term": {
              "n_rows": 59056,
              "n_predicted_positive": 3580,
              "n_false_positive": 1619,
              "false_discovery_rate": 0.4522346368715084,
              "false_discovery_rate_ci95": [
                0.43599081049049,
                0.46858086101789315
              ]
            },
            "false_discovery_rate_ratio": 0.9539405509982185,
            "false_discovery_rate_ratio_ci95": [
              0.8646054243709012,
              1.052506205938871
            ]
          },
          "clean_negative_false_positives": {
            "predicted_positive": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 272,
                "false_positive_rate": 0.06563706563706563,
                "false_positive_rate_ci95": [
                  0.058492120730319265,
                  0.07358656750118629
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1805,
                "false_positive_rate": 0.03368102853091004,
                "false_positive_rate_ci95": [
                  0.03218673333422573,
                  0.03524217139821733
                ]
              },
              "false_positive_rate_ratio": 1.9487844789783846,
              "false_positive_rate_ratio_ci95": [
                1.7223756747039638,
                2.2049550520735264
              ]
            },
            "auto_action": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 11,
                "false_positive_rate": 0.0026544401544401543,
                "false_positive_rate_ci95": [
                  0.001482871507173892,
                  0.004747226517900006
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 186,
                "false_positive_rate": 0.003470732025899871,
                "false_positive_rate_ci95": [
                  0.003007150849873329,
                  0.004005491573365235
                ]
              },
              "false_positive_rate_ratio": 0.764807001702163,
              "false_positive_rate_ratio_ci95": [
                0.41665992145986336,
                1.403854125933728
              ]
            },
            "human_review": {
              "with_identity_term": {
                "n_actual_negative": 4144,
                "n_false_positive": 261,
                "false_positive_rate": 0.06298262548262548,
                "false_positive_rate_ci95": [
                  0.05598328298901949,
                  0.07079144167056751
                ]
              },
              "without_identity_term": {
                "n_actual_negative": 53591,
                "n_false_positive": 1619,
                "false_positive_rate": 0.03021029650501017,
                "false_positive_rate_ci95": [
                  0.02879446285629727,
                  0.03169347536007906
                ]
              },
              "false_positive_rate_ratio": 2.084806598047796,
              "false_positive_rate_ratio_ci95": [
                1.836424882946234,
                2.366782650151489
              ]
            }
          },
          "allow_false_omission": {
            "with_identity_term": {
              "n_allowed": 4112,
              "n_truly_positive": 240,
              "false_omission_rate": 0.058365758754863814,
              "false_omission_rate_ci95": [
                0.051604023318199826,
                0.06595187957918881
              ]
            },
            "without_identity_term": {
              "n_allowed": 53556,
              "n_truly_positive": 1770,
              "false_omission_rate": 0.03304951826125924,
              "false_omission_rate_ci95": [
                0.031568683248493615,
                0.03459733521520847
              ]
            }
          },
          "by_term": [
            {
              "term": "gay",
              "n_predicted_positive": 360,
              "n_false_positive": 89,
              "false_discovery_rate": 0.24722222222222223
            },
            {
              "term": "black",
              "n_predicted_positive": 79,
              "n_false_positive": 35,
              "false_discovery_rate": 0.4430379746835443
            },
            {
              "term": "white",
              "n_predicted_positive": 70,
              "n_false_positive": 27,
              "false_discovery_rate": 0.38571428571428573
            },
            {
              "term": "woman",
              "n_predicted_positive": 47,
              "n_false_positive": 21,
              "false_discovery_rate": 0.44680851063829785
            },
            {
              "term": "women",
              "n_predicted_positive": 39,
              "n_false_positive": 14,
              "false_discovery_rate": 0.358974358974359
            },
            {
              "term": "jew",
              "n_predicted_positive": 35,
              "n_false_positive": 13,
              "false_discovery_rate": 0.37142857142857144
            },
            {
              "term": "homosexual",
              "n_predicted_positive": 32,
              "n_false_positive": 16,
              "false_discovery_rate": 0.5
            },
            {
              "term": "jews",
              "n_predicted_positive": 31,
              "n_false_positive": 18,
              "false_discovery_rate": 0.5806451612903226
            },
            {
              "term": "christian",
              "n_predicted_positive": 22,
              "n_false_positive": 12,
              "false_discovery_rate": 0.5454545454545454
            },
            {
              "term": "jewish",
              "n_predicted_positive": 21,
              "n_false_positive": 10,
              "false_discovery_rate": 0.47619047619047616
            },
            {
              "term": "male",
              "n_predicted_positive": 19,
              "n_false_positive": 6,
              "false_discovery_rate": 0.3157894736842105
            },
            {
              "term": "queer",
              "n_predicted_positive": 19,
              "n_false_positive": 8,
              "false_discovery_rate": 0.42105263157894735
            }
          ]
        }
      }
    },
    "subgroup_thresholds": {
      "thresholds": {
        "auto_action": {
          "identity_term_present": {
            "toxic": 0.9974480681226716,
            "severe_toxic": null,
            "obscene": null,
            "threat": null,
            "insult": null,
            "identity_hate": null
          }
        }
      },
      "threshold_selection": {
        "auto_action": {
          "n_population_tier": 921,
          "n_removed_by_subgroup_threshold": 31,
          "n_removed_truly_positive_any_label": 31
        },
        "human_review": {
          "n_population_tier": 1158,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "declared": [
          {
            "tier": "auto_action",
            "signal": "identity_term_present"
          }
        ]
      },
      "test": {
        "auto_action": {
          "n_population_tier": 2367,
          "n_removed_by_subgroup_threshold": 97,
          "n_removed_truly_positive_any_label": 77
        },
        "human_review": {
          "n_population_tier": 3925,
          "n_removed_by_subgroup_threshold": 0,
          "n_removed_truly_positive_any_label": 0
        },
        "declared": [
          {
            "tier": "auto_action",
            "signal": "identity_term_present"
          }
        ]
      }
    },
    "thresholds": {
      "auto_action": {
        "toxic": 0.9892913517285165,
        "severe_toxic": null,
        "obscene": 0.9983767822578123,
        "threat": null,
        "insult": 0.9998379195789239,
        "identity_hate": null
      },
      "human_review": {
        "toxic": 0.6576420429168361,
        "severe_toxic": null,
        "obscene": 0.6573428758151054,
        "threat": null,
        "insult": 0.9175983962802222,
        "identity_hate": null
      },
      "subgroup": {
        "auto_action": {
          "identity_term_present": {
            "toxic": 0.9974480681226716,
            "severe_toxic": null,
            "obscene": null,
            "threat": null,
            "insult": null,
            "identity_hate": null
          }
        }
      }
    },
    "identity_summary": {
      "selection_identity_predicted_positive_model": 210,
      "selection_identity_predicted_positive_final": 210,
      "selection_identity_moved_to_positive_by_rules": 0,
      "selection_identity_allowed": 1825,
      "selection_identity_allowed_truly_positive": 155,
      "test_population_tier_predicted_positive": {
        "ratio": 1.0339849108367627,
        "ci95": [
          0.9325559410243155,
          1.1464457506579027
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 275,
          "false_discovery_rate": 0.3395061728395062,
          "false_discovery_rate_ci95": [
            0.3077209890410385,
            0.3728064685335667
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5482,
          "n_false_positive": 1800,
          "false_discovery_rate": 0.32834731849689897,
          "false_discovery_rate_ci95": [
            0.31603995744110297,
            0.3408950790329937
          ]
        }
      },
      "test_population_tier_auto_action": {
        "ratio": 1.1004089539388722,
        "ci95": [
          0.7839716194909653,
          1.5445710479865116
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 322,
          "n_false_positive": 35,
          "false_discovery_rate": 0.10869565217391304,
          "false_discovery_rate_ci95": [
            0.07919957042736245,
            0.14741818145361116
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 2045,
          "n_false_positive": 202,
          "false_discovery_rate": 0.09877750611246944,
          "false_discovery_rate_ci95": [
            0.08658859263384615,
            0.11247095739301864
          ]
        }
      },
      "test_population_tier_human_review": {
        "ratio": 1.0577771394571083,
        "ci95": [
          0.9599352294588238,
          1.1655916382909013
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 488,
          "n_false_positive": 240,
          "false_discovery_rate": 0.4918032786885246,
          "false_discovery_rate_ci95": [
            0.44768506900152594,
            0.5360495270648646
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3437,
          "n_false_positive": 1598,
          "false_discovery_rate": 0.46494035496072156,
          "false_discovery_rate_ci95": [
            0.4483140605672861,
            0.4816449326383017
          ]
        }
      },
      "test_model_tier_predicted_positive": {
        "ratio": 1.0264650205761316,
        "ci95": [
          0.9253161169100481,
          1.138670794997926
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 273,
          "false_discovery_rate": 0.337037037037037,
          "false_discovery_rate_ci95": [
            0.3053212063629801,
            0.37029128902046526
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5482,
          "n_false_positive": 1800,
          "false_discovery_rate": 0.32834731849689897,
          "false_discovery_rate_ci95": [
            0.31603995744110297,
            0.3408950790329937
          ]
        }
      },
      "test_model_tier_auto_action": {
        "ratio": 0.5849284928492849,
        "ci95": [
          0.33962197000956706,
          1.0074181647827376
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 225,
          "n_false_positive": 13,
          "false_discovery_rate": 0.057777777777777775,
          "false_discovery_rate_ci95": [
            0.034073070023627444,
            0.09632925754051938
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 2045,
          "n_false_positive": 202,
          "false_discovery_rate": 0.09877750611246944,
          "false_discovery_rate_ci95": [
            0.08658859263384615,
            0.11247095739301864
          ]
        }
      },
      "test_model_tier_human_review": {
        "ratio": 0.9559171186204978,
        "ci95": [
          0.8671669934988251,
          1.0537503670254176
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 585,
          "n_false_positive": 260,
          "false_discovery_rate": 0.4444444444444444,
          "false_discovery_rate_ci95": [
            0.40467041224149014,
            0.4849433385913918
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3437,
          "n_false_positive": 1598,
          "false_discovery_rate": 0.46494035496072156,
          "false_discovery_rate_ci95": [
            0.4483140605672861,
            0.4816449326383017
          ]
        }
      },
      "test_final_tier_predicted_positive": {
        "ratio": 1.023580178743072,
        "ci95": [
          0.9227630126443263,
          1.1354122000548104
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 810,
          "n_false_positive": 273,
          "false_discovery_rate": 0.337037037037037,
          "false_discovery_rate_ci95": [
            0.3053212063629801,
            0.37029128902046526
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 5500,
          "n_false_positive": 1811,
          "false_discovery_rate": 0.3292727272727273,
          "false_discovery_rate_ci95": [
            0.3169757651349169,
            0.34180801088218515
          ]
        }
      },
      "test_final_tier_auto_action": {
        "ratio": 0.5853658536585366,
        "ci95": [
          0.33265017363980887,
          1.0300706561494526
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 205,
          "n_false_positive": 12,
          "false_discovery_rate": 0.05853658536585366,
          "false_discovery_rate_ci95": [
            0.033799019999132485,
            0.09951482980078147
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 1920,
          "n_false_positive": 192,
          "false_discovery_rate": 0.1,
          "false_discovery_rate_ci95": [
            0.08736938121151575,
            0.11422803061826353
          ]
        }
      },
      "test_final_tier_human_review": {
        "ratio": 0.9539405509982185,
        "ci95": [
          0.8646054243709012,
          1.052506205938871
        ],
        "with": {
          "n_rows": 4922,
          "n_predicted_positive": 605,
          "n_false_positive": 261,
          "false_discovery_rate": 0.43140495867768597,
          "false_discovery_rate_ci95": [
            0.3924948564625459,
            0.47118065579230817
          ]
        },
        "without": {
          "n_rows": 59056,
          "n_predicted_positive": 3580,
          "n_false_positive": 1619,
          "false_discovery_rate": 0.4522346368715084,
          "false_discovery_rate_ci95": [
            0.43599081049049,
            0.46858086101789315
          ]
        }
      },
      "test_final_auto_clean_negative_fpr": {
        "with_identity_term": {
          "n_actual_negative": 4144,
          "n_false_positive": 11,
          "false_positive_rate": 0.0026544401544401543,
          "false_positive_rate_ci95": [
            0.001482871507173892,
            0.004747226517900006
          ]
        },
        "without_identity_term": {
          "n_actual_negative": 53591,
          "n_false_positive": 186,
          "false_positive_rate": 0.003470732025899871,
          "false_positive_rate_ci95": [
            0.003007150849873329,
            0.004005491573365235
          ]
        },
        "false_positive_rate_ratio": 0.764807001702163,
        "false_positive_rate_ratio_ci95": [
          0.41665992145986336,
          1.403854125933728
        ]
      },
      "test_by_term": [
        {
          "term": "gay",
          "n_predicted_positive": 360,
          "n_false_positive": 89,
          "false_discovery_rate": 0.24722222222222223
        },
        {
          "term": "black",
          "n_predicted_positive": 79,
          "n_false_positive": 35,
          "false_discovery_rate": 0.4430379746835443
        },
        {
          "term": "white",
          "n_predicted_positive": 70,
          "n_false_positive": 27,
          "false_discovery_rate": 0.38571428571428573
        },
        {
          "term": "woman",
          "n_predicted_positive": 47,
          "n_false_positive": 21,
          "false_discovery_rate": 0.44680851063829785
        },
        {
          "term": "women",
          "n_predicted_positive": 39,
          "n_false_positive": 14,
          "false_discovery_rate": 0.358974358974359
        },
        {
          "term": "jew",
          "n_predicted_positive": 35,
          "n_false_positive": 13,
          "false_discovery_rate": 0.37142857142857144
        },
        {
          "term": "homosexual",
          "n_predicted_positive": 32,
          "n_false_positive": 16,
          "false_discovery_rate": 0.5
        },
        {
          "term": "jews",
          "n_predicted_positive": 31,
          "n_false_positive": 18,
          "false_discovery_rate": 0.5806451612903226
        },
        {
          "term": "christian",
          "n_predicted_positive": 22,
          "n_false_positive": 12,
          "false_discovery_rate": 0.5454545454545454
        },
        {
          "term": "jewish",
          "n_predicted_positive": 21,
          "n_false_positive": 10,
          "false_discovery_rate": 0.47619047619047616
        },
        {
          "term": "male",
          "n_predicted_positive": 19,
          "n_false_positive": 6,
          "false_discovery_rate": 0.3157894736842105
        },
        {
          "term": "queer",
          "n_predicted_positive": 19,
          "n_false_positive": 8,
          "false_discovery_rate": 0.42105263157894735
        }
      ],
      "identity_terms": [
        "african",
        "africans",
        "arab",
        "arabs",
        "asian",
        "asians",
        "atheist",
        "atheists",
        "autistic",
        "bisexual",
        "black",
        "blacks",
        "buddhist",
        "buddhists",
        "catholic",
        "catholics",
        "chinese",
        "christian",
        "christians",
        "disability",
        "disabled",
        "elderly",
        "female",
        "females",
        "feminist",
        "feminists",
        "gay",
        "gays",
        "heterosexual",
        "hindu",
        "hindus",
        "hispanic",
        "homosexual",
        "homosexuals",
        "immigrant",
        "immigrants",
        "indian",
        "indians",
        "islam",
        "islamic",
        "jew",
        "jewish",
        "jews",
        "latina",
        "latino",
        "lesbian",
        "lesbians",
        "lgbt",
        "male",
        "males",
        "mexican",
        "mexicans",
        "muslim",
        "muslims",
        "queer",
        "refugee",
        "refugees",
        "trans",
        "transgender",
        "white",
        "whites",
        "woman",
        "women"
      ]
    }
  }
}