{
  "title": "Selective field recovery development study",
  "date": "2026-09-20",
  "dataset": "Mem2ActBench",
  "dataset_repository": "https://github.com/Cantaloupe-M/Mem2ActBench",
  "dataset_revision": "b00726940b5abbe9bd324bdd7a2cb272f5c62a29",
  "dataset_files_sha256": {
    "qa_dataset.jsonl": "24fc8692567e5fc16457a7e6c9ec12d68ba06245d53b315c7ced5f76a84fd039",
    "toolmem_conversation.jsonl": "8ab30dfb7b2a7455bd5b84250c03b48ebe5cc377a2b5ca03a196bf471c9bdca2"
  },
  "model": "claude-sonnet-5",
  "scope": [
    "Development only; no held-out quality claim",
    "Oracle sessions and known target tool",
    "Exact JSON scoring, not official leaderboard reproduction",
    "Quotes check provenance, not semantic entailment",
    "No executed outcomes or interactive clarification labels",
    "Missing source histories remain in all-case denominator"
  ],
  "bootstrap": {
    "unit": "source_group",
    "draws": 2000,
    "seed": 20260920
  },
  "results": {
    "metrics": {
      "snapshot": {
        "cases": 80,
        "source_groups": 65,
        "measured": 68,
        "errors_or_missing": 12,
        "resolution_correct": 28,
        "all_case_success_rate": 0.35,
        "expected_executable": 80,
        "executed": 38,
        "correct_actions": 28,
        "correct_action_rate": 0.35,
        "wrong_actions": 10,
        "wrong_action_rate_all_cases": 0.125,
        "wrong_action_rate_executed": 0.2631578947368421,
        "unnecessary_clarifications": 0,
        "unnecessary_clarification_rate": 0.0,
        "scope": "Proposed actions only; not execution success, question quality or production safety"
      },
      "typed_once": {
        "cases": 80,
        "source_groups": 65,
        "measured": 68,
        "errors_or_missing": 12,
        "resolution_correct": 25,
        "all_case_success_rate": 0.3125,
        "expected_executable": 80,
        "executed": 30,
        "correct_actions": 25,
        "correct_action_rate": 0.3125,
        "wrong_actions": 5,
        "wrong_action_rate_all_cases": 0.0625,
        "wrong_action_rate_executed": 0.16666666666666666,
        "unnecessary_clarifications": 0,
        "unnecessary_clarification_rate": 0.0,
        "scope": "Proposed actions only; not execution success, question quality or production safety"
      },
      "full_retry": {
        "cases": 80,
        "source_groups": 65,
        "measured": 68,
        "errors_or_missing": 12,
        "resolution_correct": 31,
        "all_case_success_rate": 0.3875,
        "expected_executable": 80,
        "executed": 41,
        "correct_actions": 31,
        "correct_action_rate": 0.3875,
        "wrong_actions": 10,
        "wrong_action_rate_all_cases": 0.125,
        "wrong_action_rate_executed": 0.24390243902439024,
        "unnecessary_clarifications": 0,
        "unnecessary_clarification_rate": 0.0,
        "scope": "Proposed actions only; not execution success, question quality or production safety"
      },
      "selective_retry": {
        "cases": 80,
        "source_groups": 65,
        "measured": 68,
        "errors_or_missing": 12,
        "resolution_correct": 29,
        "all_case_success_rate": 0.3625,
        "expected_executable": 80,
        "executed": 40,
        "correct_actions": 29,
        "correct_action_rate": 0.3625,
        "wrong_actions": 11,
        "wrong_action_rate_all_cases": 0.1375,
        "wrong_action_rate_executed": 0.275,
        "unnecessary_clarifications": 0,
        "unnecessary_clarification_rate": 0.0,
        "scope": "Proposed actions only; not execution success, question quality or production safety"
      }
    },
    "usage": {
      "snapshot": {
        "attempted_calls": 68,
        "returned_usage_records": 68,
        "input_tokens": 699229,
        "output_tokens": 9248,
        "sum_request_seconds": 198.07067120913416
      },
      "typed_once": {
        "attempted_calls": 68,
        "returned_usage_records": 68,
        "input_tokens": 711877,
        "output_tokens": 14266,
        "sum_request_seconds": 245.82629200536758
      },
      "full_retry": {
        "attempted_calls": 106,
        "returned_usage_records": 106,
        "input_tokens": 1136220,
        "output_tokens": 23644,
        "sum_request_seconds": 370.2730835452676
      },
      "selective_retry": {
        "attempted_calls": 106,
        "returned_usage_records": 106,
        "input_tokens": 1134315,
        "output_tokens": 18404,
        "sum_request_seconds": 364.047304793261
      }
    },
    "primary_comparison": {
      "accuracy_difference": -0.025,
      "group_bootstrap_95_percentile_interval": [
        -0.0684931506849315,
        0.0
      ],
      "source_groups": 65,
      "draws": 2000,
      "interpretation": "Errors/unavailable inputs score unsuccessful in this all-case difference; inspect coverage separately"
    }
  }
}