{
  "accuracy": 0.9086475942782835,
  "challenge": {
    "count": 100,
    "false_acceptance": 0.06,
    "false_acceptance_95ci": [
      0.02778612396318876,
      0.1247681544589569
    ],
    "origin": "original_authored"
  },
  "dataset_hash": "9bad4bbf3ceea09dc3731d6f9cf9e77ec6383fd67fd69faccb32b4dc2f490882",
  "environment": {
    "platform": "macOS-26.3.2-arm64-arm-64bit",
    "processor": "arm",
    "python": "3.12.14"
  },
  "kind": "encoder",
  "latency": {
    "cold_first_prediction_ms": 19.57370899617672,
    "cold_model_load_ms": 1980.1932499976829,
    "queries": 200,
    "scope": "model_logits",
    "warm_p50_ms": 8.529874503437895,
    "warm_p95_ms": 11.71081070933722,
    "warmup_queries": 20
  },
  "macro_f1": 0.9086261106451545,
  "model_version": "337ea9f23fcb64182dcd337067382d9dff01e9ba1565b2315a54883375408d91",
  "policy": {
    "accepted": 475,
    "accepted_accuracy": 0.9957894736842106,
    "challenge_false_acceptance": 0.01,
    "coverage": 0.46027131782945735,
    "margin": 0.0,
    "probability": 0.98,
    "review_only": false
  },
  "routing": {
    "accepted": 1294,
    "accepted_accuracy": 0.9945904173106646,
    "accepted_accuracy_95ci": [
      0.9888758164198189,
      0.9973771587032704
    ],
    "coverage": 0.4206762028608583,
    "review_rate": 0.5793237971391417,
    "total": 3076
  },
  "run_id": "60e353abe4144373aff9fd49",
  "schema_version": 1,
  "temperature": 0.8713410112268448,
  "test_rows": 3076,
  "per_intent": [
    {
      "f1": 0.961038961038961,
      "label": "activate_my_card",
      "label_id": 0,
      "precision": 1.0,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9876543209876543,
      "label": "age_limit",
      "label_id": 1,
      "precision": 0.975609756097561,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 1.0,
      "label": "apple_pay_or_google_pay",
      "label_id": 2,
      "precision": 1.0,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9736842105263158,
      "label": "atm_support",
      "label_id": 3,
      "precision": 1.0,
      "recall": 0.9487179487179487,
      "support": 39
    },
    {
      "f1": 0.9473684210526315,
      "label": "automatic_top_up",
      "label_id": 4,
      "precision": 1.0,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.7692307692307693,
      "label": "balance_not_updated_after_bank_transfer",
      "label_id": 5,
      "precision": 0.7894736842105263,
      "recall": 0.75,
      "support": 40
    },
    {
      "f1": 0.9487179487179487,
      "label": "balance_not_updated_after_cheque_or_cash_deposit",
      "label_id": 6,
      "precision": 0.9736842105263158,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9090909090909091,
      "label": "beneficiary_not_allowed",
      "label_id": 7,
      "precision": 0.9459459459459459,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.961038961038961,
      "label": "cancel_transfer",
      "label_id": 8,
      "precision": 1.0,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9620253164556962,
      "label": "card_about_to_expire",
      "label_id": 9,
      "precision": 0.9743589743589743,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.8611111111111112,
      "label": "card_acceptance",
      "label_id": 10,
      "precision": 0.96875,
      "recall": 0.775,
      "support": 40
    },
    {
      "f1": 0.8571428571428571,
      "label": "card_arrival",
      "label_id": 11,
      "precision": 0.8181818181818182,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.8536585365853658,
      "label": "card_delivery_estimate",
      "label_id": 12,
      "precision": 0.8333333333333334,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9629629629629629,
      "label": "card_linking",
      "label_id": 13,
      "precision": 0.9512195121951219,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.9047619047619048,
      "label": "card_not_working",
      "label_id": 14,
      "precision": 0.8636363636363636,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9047619047619048,
      "label": "card_payment_fee_charged",
      "label_id": 15,
      "precision": 0.8636363636363636,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9210526315789473,
      "label": "card_payment_not_recognised",
      "label_id": 16,
      "precision": 0.9722222222222222,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9487179487179487,
      "label": "card_payment_wrong_exchange_rate",
      "label_id": 17,
      "precision": 0.9736842105263158,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.961038961038961,
      "label": "card_swallowed",
      "label_id": 18,
      "precision": 1.0,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.926829268292683,
      "label": "cash_withdrawal_charge",
      "label_id": 19,
      "precision": 0.9047619047619048,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9156626506024096,
      "label": "cash_withdrawal_not_recognised",
      "label_id": 20,
      "precision": 0.8837209302325582,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9382716049382716,
      "label": "change_pin",
      "label_id": 21,
      "precision": 0.926829268292683,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.868421052631579,
      "label": "compromised_card",
      "label_id": 22,
      "precision": 0.9166666666666666,
      "recall": 0.825,
      "support": 40
    },
    {
      "f1": 0.9333333333333333,
      "label": "contactless_not_working",
      "label_id": 23,
      "precision": 1.0,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9411764705882353,
      "label": "country_support",
      "label_id": 24,
      "precision": 0.8888888888888888,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.8260869565217391,
      "label": "declined_card_payment",
      "label_id": 25,
      "precision": 0.7307692307692307,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.898876404494382,
      "label": "declined_cash_withdrawal",
      "label_id": 26,
      "precision": 0.8163265306122449,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.8235294117647058,
      "label": "declined_transfer",
      "label_id": 27,
      "precision": 1.0,
      "recall": 0.7,
      "support": 40
    },
    {
      "f1": 0.8607594936708861,
      "label": "direct_debit_payment_not_recognised",
      "label_id": 28,
      "precision": 0.8717948717948718,
      "recall": 0.85,
      "support": 40
    },
    {
      "f1": 0.8974358974358975,
      "label": "disposable_card_limits",
      "label_id": 29,
      "precision": 0.9210526315789473,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.975609756097561,
      "label": "edit_personal_details",
      "label_id": 30,
      "precision": 0.9523809523809523,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.935064935064935,
      "label": "exchange_charge",
      "label_id": 31,
      "precision": 0.972972972972973,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.9285714285714286,
      "label": "exchange_rate",
      "label_id": 32,
      "precision": 0.8863636363636364,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8470588235294118,
      "label": "exchange_via_app",
      "label_id": 33,
      "precision": 0.782608695652174,
      "recall": 0.9230769230769231,
      "support": 39
    },
    {
      "f1": 0.9397590361445783,
      "label": "extra_charge_on_statement",
      "label_id": 34,
      "precision": 0.9069767441860465,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8888888888888888,
      "label": "failed_transfer",
      "label_id": 35,
      "precision": 0.8780487804878049,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.868421052631579,
      "label": "fiat_currency_support",
      "label_id": 36,
      "precision": 0.9166666666666666,
      "recall": 0.825,
      "support": 40
    },
    {
      "f1": 0.8571428571428571,
      "label": "get_disposable_virtual_card",
      "label_id": 37,
      "precision": 0.8918918918918919,
      "recall": 0.825,
      "support": 40
    },
    {
      "f1": 0.975609756097561,
      "label": "get_physical_card",
      "label_id": 38,
      "precision": 0.9523809523809523,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9,
      "label": "getting_spare_card",
      "label_id": 39,
      "precision": 0.9,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.8764044943820225,
      "label": "getting_virtual_card",
      "label_id": 40,
      "precision": 0.7959183673469388,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8837209302325582,
      "label": "lost_or_stolen_card",
      "label_id": 41,
      "precision": 0.8260869565217391,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9873417721518988,
      "label": "lost_or_stolen_phone",
      "label_id": 42,
      "precision": 1.0,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8433734939759037,
      "label": "order_physical_card",
      "label_id": 43,
      "precision": 0.813953488372093,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 1.0,
      "label": "passcode_forgotten",
      "label_id": 44,
      "precision": 1.0,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9367088607594937,
      "label": "pending_card_payment",
      "label_id": 45,
      "precision": 0.9487179487179487,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.975,
      "label": "pending_cash_withdrawal",
      "label_id": 46,
      "precision": 0.975,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.9,
      "label": "pending_top_up",
      "label_id": 47,
      "precision": 0.9,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.7894736842105263,
      "label": "pending_transfer",
      "label_id": 48,
      "precision": 0.8108108108108109,
      "recall": 0.7692307692307693,
      "support": 39
    },
    {
      "f1": 0.9090909090909091,
      "label": "pin_blocked",
      "label_id": 49,
      "precision": 0.9459459459459459,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9,
      "label": "receiving_money",
      "label_id": 50,
      "precision": 0.9,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.9487179487179487,
      "label": "Refund_not_showing_up",
      "label_id": 51,
      "precision": 0.9736842105263158,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9629629629629629,
      "label": "request_refund",
      "label_id": 52,
      "precision": 0.9512195121951219,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8333333333333334,
      "label": "reverted_card_payment?",
      "label_id": 53,
      "precision": 0.7954545454545454,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9024390243902439,
      "label": "supported_cards_and_currencies",
      "label_id": 54,
      "precision": 0.8809523809523809,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9876543209876543,
      "label": "terminate_account",
      "label_id": 55,
      "precision": 0.975609756097561,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9210526315789473,
      "label": "top_up_by_bank_transfer_charge",
      "label_id": 56,
      "precision": 0.9722222222222222,
      "recall": 0.875,
      "support": 40
    },
    {
      "f1": 0.9382716049382716,
      "label": "top_up_by_card_charge",
      "label_id": 57,
      "precision": 0.926829268292683,
      "recall": 0.95,
      "support": 40
    },
    {
      "f1": 0.9230769230769231,
      "label": "top_up_by_cash_or_cheque",
      "label_id": 58,
      "precision": 0.9473684210526315,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.9024390243902439,
      "label": "top_up_failed",
      "label_id": 59,
      "precision": 0.8809523809523809,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9629629629629629,
      "label": "top_up_limits",
      "label_id": 60,
      "precision": 0.9512195121951219,
      "recall": 0.975,
      "support": 40
    },
    {
      "f1": 0.8461538461538461,
      "label": "top_up_reverted",
      "label_id": 61,
      "precision": 0.868421052631579,
      "recall": 0.825,
      "support": 40
    },
    {
      "f1": 0.759493670886076,
      "label": "topping_up_by_card",
      "label_id": 62,
      "precision": 0.7692307692307693,
      "recall": 0.75,
      "support": 40
    },
    {
      "f1": 0.963855421686747,
      "label": "transaction_charged_twice",
      "label_id": 63,
      "precision": 0.9302325581395349,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9113924050632911,
      "label": "transfer_fee_charged",
      "label_id": 64,
      "precision": 0.9230769230769231,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.8292682926829268,
      "label": "transfer_into_account",
      "label_id": 65,
      "precision": 0.8095238095238095,
      "recall": 0.85,
      "support": 40
    },
    {
      "f1": 0.8888888888888888,
      "label": "transfer_not_received_by_recipient",
      "label_id": 66,
      "precision": 0.8571428571428571,
      "recall": 0.9230769230769231,
      "support": 39
    },
    {
      "f1": 0.8705882352941177,
      "label": "transfer_timing",
      "label_id": 67,
      "precision": 0.8222222222222222,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.9113924050632911,
      "label": "unable_to_verify_identity",
      "label_id": 68,
      "precision": 0.9230769230769231,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.7816091954022989,
      "label": "verify_my_identity",
      "label_id": 69,
      "precision": 0.723404255319149,
      "recall": 0.85,
      "support": 40
    },
    {
      "f1": 1.0,
      "label": "verify_source_of_funds",
      "label_id": 70,
      "precision": 1.0,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.9876543209876543,
      "label": "verify_top_up",
      "label_id": 71,
      "precision": 0.975609756097561,
      "recall": 1.0,
      "support": 40
    },
    {
      "f1": 0.8732394366197183,
      "label": "virtual_card_not_working",
      "label_id": 72,
      "precision": 1.0,
      "recall": 0.775,
      "support": 40
    },
    {
      "f1": 0.9487179487179487,
      "label": "visa_or_mastercard",
      "label_id": 73,
      "precision": 0.9736842105263158,
      "recall": 0.925,
      "support": 40
    },
    {
      "f1": 0.7123287671232876,
      "label": "why_verify_identity",
      "label_id": 74,
      "precision": 0.7878787878787878,
      "recall": 0.65,
      "support": 40
    },
    {
      "f1": 0.9473684210526315,
      "label": "wrong_amount_of_cash_received",
      "label_id": 75,
      "precision": 1.0,
      "recall": 0.9,
      "support": 40
    },
    {
      "f1": 0.9066666666666666,
      "label": "wrong_exchange_rate_for_cash_withdrawal",
      "label_id": 76,
      "precision": 0.9714285714285714,
      "recall": 0.85,
      "support": 40
    }
  ],
  "selection": {
    "candidates": [
      {
        "C": 0.1,
        "cpu_warm_p50_ms": 0.49897849385160953,
        "cpu_warm_p95_ms": 0.7566121959825968,
        "kind": "tfidf",
        "latency_queries": 200,
        "model_path": "artifacts/classifier/baseline-c0.1",
        "model_version": "6e2f6d38bd424da0a53177a41c84da278b0858ceaec3327c373e063fb3a0e030",
        "selection_accuracy": 0.6234939759036144,
        "selection_macro_f1": 0.5711225425472592,
        "training_seconds": 0.9212144589982927,
        "warmup_queries": 20
      },
      {
        "C": 1.0,
        "cpu_warm_p50_ms": 0.7489374984288588,
        "cpu_warm_p95_ms": 1.7441231990233061,
        "kind": "tfidf",
        "latency_queries": 200,
        "model_path": "artifacts/classifier/baseline-c1",
        "model_version": "4bc19223fddcb81a9c0830e6e63ddfd702019c2ba0a9003830f97f66822a1364",
        "selection_accuracy": 0.8473895582329317,
        "selection_macro_f1": 0.8336635270588467,
        "training_seconds": 2.631425500003388,
        "warmup_queries": 20
      },
      {
        "C": 10.0,
        "cpu_warm_p50_ms": 0.5075420049251989,
        "cpu_warm_p95_ms": 0.9121895986027081,
        "kind": "tfidf",
        "latency_queries": 200,
        "model_path": "artifacts/classifier/baseline-c10",
        "model_version": "1c63ecd603808ab2d16ada51e0b51aecafe67d51602c4e9861d313b779d96e4d",
        "selection_accuracy": 0.8694779116465864,
        "selection_macro_f1": 0.8718995153669015,
        "training_seconds": 5.168581666002865,
        "warmup_queries": 20
      },
      {
        "cpu_warm_p50_ms": 8.442542006378062,
        "cpu_warm_p95_ms": 9.661687145126052,
        "kind": "encoder",
        "latency_queries": 200,
        "model_path": "artifacts/classifier/kaggle/encoder-2e-05/model",
        "model_version": "7229cba5f387bf3746a70aea1cf71b300a70a60bb680185c067eb0b0fd3d6809",
        "selection_accuracy": 0.8263052208835341,
        "selection_macro_f1": 0.7781921089177191,
        "warmup_queries": 20
      },
      {
        "cpu_warm_p50_ms": 8.343958492332604,
        "cpu_warm_p95_ms": 9.798273204069119,
        "kind": "encoder",
        "latency_queries": 200,
        "model_path": "artifacts/classifier/kaggle/encoder-5e-05/model",
        "model_version": "8f3fbbc0ba7ee29e59bc51efeb23b6b856cfd85dbed7f5b3371e64da860e82b6",
        "selection_accuracy": 0.9036144578313253,
        "selection_macro_f1": 0.8994418429006666,
        "warmup_queries": 20
      }
    ],
    "cpu_threads": 2,
    "rule": "Highest selection macro-F1; within .005 prefer lower CPU p95",
    "selected": {
      "cpu_warm_p50_ms": 8.343958492332604,
      "cpu_warm_p95_ms": 9.798273204069119,
      "kind": "encoder",
      "latency_queries": 200,
      "model_path": "artifacts/classifier/kaggle/encoder-5e-05/model",
      "model_version": "8f3fbbc0ba7ee29e59bc51efeb23b6b856cfd85dbed7f5b3371e64da860e82b6",
      "selection_accuracy": 0.9036144578313253,
      "selection_macro_f1": 0.8994418429006666,
      "warmup_queries": 20
    },
    "test_evaluated": false
  }
}