{
  "schema": 1,
  "updated": "2026-06-27T15:23:17+00:00",
  "runs": [
    {
      "run_id": "vsympy_sympy_sequence-base-1763e51-1",
      "timestamp": "2026-06-27T04:37:20+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": false,
      "label": "SymPy base (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.9,
        "efficiency": 0.0,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.9,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vsympy_sympy_sequence-base-1763e51-1\\SUMMARY-20260627T043720Z.json",
      "notes": "SymPy swe_bench_cl sequence, stage=base, harness=cursor, git_sha=1763e51, resolved=45/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vsympy_sympy_sequence-builtin-1763e51-1",
      "timestamp": "2026-06-27T04:37:41+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "SymPy builtin (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.9,
        "efficiency": 0.0,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.9,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vsympy_sympy_sequence-builtin-1763e51-1\\SUMMARY-20260627T043741Z.json",
      "notes": "SymPy swe_bench_cl sequence, stage=builtin, harness=cursor, git_sha=1763e51, resolved=45/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vsympy_sympy_sequence-plugin-accum-1763e51-2",
      "timestamp": "2026-06-27T04:39:02+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "SymPy plugin-accum (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.96,
        "efficiency": 0.44158206923259413,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.96,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vsympy_sympy_sequence-plugin-accum-1763e51-2\\SUMMARY-20260627T043902Z.json",
      "notes": "SymPy swe_bench_cl sequence, stage=plugin-accum, harness=cursor, git_sha=1763e51, resolved=48/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vsympy_sympy_sequence-plugin-dreamed-1763e51-1",
      "timestamp": "2026-06-27T08:14:29+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "SymPy plugin-dreamed (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.9,
        "efficiency": 0.4745209864154167,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.9,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vsympy_sympy_sequence-plugin-dreamed-1763e51-1\\SUMMARY-20260627T081429Z.json",
      "notes": "SymPy swe_bench_cl sequence, stage=plugin-dreamed, harness=cursor, git_sha=1763e51, resolved=45/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vdjango_django_sequence-base-1763e51-1",
      "timestamp": "2026-06-27T04:35:40+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": false,
      "label": "Django base (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.2,
        "efficiency": 0.0,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.2,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vdjango_django_sequence-base-1763e51-1\\SUMMARY-20260627T043540Z.json",
      "notes": "Django swe_bench_cl sequence, stage=base, harness=cursor, git_sha=1763e51, resolved=10/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vdjango_django_sequence-builtin-1763e51-1",
      "timestamp": "2026-06-27T04:36:03+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "Django builtin (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.22,
        "efficiency": 0.0,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.22,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vdjango_django_sequence-builtin-1763e51-1\\SUMMARY-20260627T043603Z.json",
      "notes": "Django swe_bench_cl sequence, stage=builtin, harness=cursor, git_sha=1763e51, resolved=11/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vdjango_django_sequence-plugin-accum-1763e51-1",
      "timestamp": "2026-06-27T04:36:54+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "Django plugin-accum (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.24,
        "efficiency": 0.48922001533265586,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.24,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vdjango_django_sequence-plugin-accum-1763e51-1\\SUMMARY-20260627T043654Z.json",
      "notes": "Django swe_bench_cl sequence, stage=plugin-accum, harness=cursor, git_sha=1763e51, resolved=12/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "vdjango_django_sequence-plugin-dreamed-1763e51-1",
      "timestamp": "2026-06-27T07:49:18+00:00",
      "benchmark": "swe_bench_cl",
      "model": "composer-2.5",
      "memory": true,
      "label": "Django plugin-dreamed (Cursor)",
      "mode": "sequence",
      "metrics": {
        "accuracy": 0.26,
        "efficiency": 0.8866362305611559,
        "recency": 0.0,
        "relevancy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 50,
        "extra": {}
      },
      "native": {
        "metrics": {
          "accuracy": {
            "value": 0.26,
            "label": "Accuracy"
          }
        }
      },
      "n_tasks": 50,
      "cost_usd": 0.0,
      "tokens_in": 0,
      "tokens_out": 0,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\results\\vdjango_django_sequence-plugin-dreamed-1763e51-1\\SUMMARY-20260627T074918Z.json",
      "notes": "Django swe_bench_cl sequence, stage=plugin-dreamed, harness=cursor, git_sha=1763e51, resolved=13/50 graded; Cursor composer-2.5 same-commit comparison"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:28:50+00:00",
      "benchmark": "memoryagentbench",
      "model": "claude-haiku-4-5",
      "memory": true,
      "label": "claude-haiku-4-5+mem",
      "mode": "single",
      "metrics": {
        "recency": 0.5,
        "efficiency": 0.2810498883097543,
        "relevancy": 0.23083951866846603,
        "accuracy": 1.0,
        "recency_decayed": 1.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.000797,
      "tokens_in": 112,
      "tokens_out": 137,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\memoryagentbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:28:52+00:00",
      "benchmark": "memoryagentbench",
      "model": "claude-haiku-4-5",
      "memory": false,
      "label": "claude-haiku-4-5",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.000801,
      "tokens_in": 46,
      "tokens_out": 151,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\memoryagentbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-opus-4-8",
      "timestamp": "2026-06-17T20:28:59+00:00",
      "benchmark": "memoryagentbench",
      "model": "claude-opus-4-8",
      "memory": false,
      "label": "claude-opus-4-8",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.007045,
      "tokens_in": 54,
      "tokens_out": 271,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\memoryagentbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-sonnet-4-6",
      "timestamp": "2026-06-17T20:29:02+00:00",
      "benchmark": "memoryagentbench",
      "model": "claude-sonnet-4-6",
      "memory": false,
      "label": "claude-sonnet-4-6",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.001773,
      "tokens_in": 46,
      "tokens_out": 109,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\memoryagentbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:29:05+00:00",
      "benchmark": "longmemeval",
      "model": "claude-haiku-4-5",
      "memory": true,
      "label": "claude-haiku-4-5+mem",
      "mode": "single",
      "metrics": {
        "recency": 1.0,
        "efficiency": 0.33601450096295454,
        "relevancy": 0.11802054154995333,
        "accuracy": 0.5,
        "recency_decayed": 1.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.000488,
      "tokens_in": 113,
      "tokens_out": 75,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\longmemeval.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:29:07+00:00",
      "benchmark": "longmemeval",
      "model": "claude-haiku-4-5",
      "memory": false,
      "label": "claude-haiku-4-5",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.000777,
      "tokens_in": 42,
      "tokens_out": 147,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\longmemeval.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-opus-4-8",
      "timestamp": "2026-06-17T20:29:14+00:00",
      "benchmark": "longmemeval",
      "model": "claude-opus-4-8",
      "memory": false,
      "label": "claude-opus-4-8",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.00873,
      "tokens_in": 51,
      "tokens_out": 339,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\longmemeval.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-sonnet-4-6",
      "timestamp": "2026-06-17T20:29:18+00:00",
      "benchmark": "longmemeval",
      "model": "claude-sonnet-4-6",
      "memory": false,
      "label": "claude-sonnet-4-6",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.002016,
      "tokens_in": 42,
      "tokens_out": 126,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\longmemeval.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:29:25+00:00",
      "benchmark": "swe_contextbench",
      "model": "claude-haiku-4-5",
      "memory": true,
      "label": "claude-haiku-4-5+mem",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.04798264803280897,
        "relevancy": 0.19478844169246645,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.004236,
      "tokens_in": 106,
      "tokens_out": 826,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:29:36+00:00",
      "benchmark": "swe_contextbench",
      "model": "claude-haiku-4-5",
      "memory": false,
      "label": "claude-haiku-4-5",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.006324,
      "tokens_in": 49,
      "tokens_out": 1255,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-opus-4-8",
      "timestamp": "2026-06-17T20:29:55+00:00",
      "benchmark": "swe_contextbench",
      "model": "claude-opus-4-8",
      "memory": false,
      "label": "claude-opus-4-8",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.03823,
      "tokens_in": 66,
      "tokens_out": 1516,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-sonnet-4-6",
      "timestamp": "2026-06-17T20:30:25+00:00",
      "benchmark": "swe_contextbench",
      "model": "claude-sonnet-4-6",
      "memory": false,
      "label": "claude-sonnet-4-6",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.030867,
      "tokens_in": 49,
      "tokens_out": 2048,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:30:32+00:00",
      "benchmark": "swe_bench_cl",
      "model": "claude-haiku-4-5",
      "memory": true,
      "label": "claude-haiku-4-5+mem",
      "mode": "single",
      "metrics": {
        "recency": 1.0,
        "efficiency": 0.040380047505938245,
        "relevancy": 0.09090909090909091,
        "accuracy": 0.0,
        "recency_decayed": 1.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.003317,
      "tokens_in": 92,
      "tokens_out": 645,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_bench_cl.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:30:41+00:00",
      "benchmark": "swe_bench_cl",
      "model": "claude-haiku-4-5",
      "memory": false,
      "label": "claude-haiku-4-5",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.003627,
      "tokens_in": 47,
      "tokens_out": 716,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_bench_cl.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-opus-4-8",
      "timestamp": "2026-06-17T20:30:58+00:00",
      "benchmark": "swe_bench_cl",
      "model": "claude-opus-4-8",
      "memory": false,
      "label": "claude-opus-4-8",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.031025,
      "tokens_in": 65,
      "tokens_out": 1228,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_bench_cl.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-sonnet-4-6",
      "timestamp": "2026-06-17T20:31:18+00:00",
      "benchmark": "swe_bench_cl",
      "model": "claude-sonnet-4-6",
      "memory": false,
      "label": "claude-sonnet-4-6",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.014436,
      "tokens_in": 47,
      "tokens_out": 953,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\swe_bench_cl.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:31:26+00:00",
      "benchmark": "contextbench",
      "model": "claude-haiku-4-5",
      "memory": true,
      "label": "claude-haiku-4-5+mem",
      "mode": "single",
      "metrics": {
        "recency": 1.0,
        "efficiency": 0.03796115919908421,
        "relevancy": 0.09103641456582633,
        "accuracy": 0.0,
        "recency_decayed": 1.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.00457,
      "tokens_in": 105,
      "tokens_out": 893,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-haiku-4-5",
      "timestamp": "2026-06-17T20:31:34+00:00",
      "benchmark": "contextbench",
      "model": "claude-haiku-4-5",
      "memory": false,
      "label": "claude-haiku-4-5",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.004578,
      "tokens_in": 53,
      "tokens_out": 905,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-opus-4-8",
      "timestamp": "2026-06-17T20:31:57+00:00",
      "benchmark": "contextbench",
      "model": "claude-opus-4-8",
      "memory": false,
      "label": "claude-opus-4-8",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.044475,
      "tokens_in": 75,
      "tokens_out": 1764,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    },
    {
      "run_id": "real-claude-sonnet-4-6",
      "timestamp": "2026-06-17T20:32:23+00:00",
      "benchmark": "contextbench",
      "model": "claude-sonnet-4-6",
      "memory": false,
      "label": "claude-sonnet-4-6",
      "mode": "single",
      "metrics": {
        "recency": 0.0,
        "efficiency": 0.0,
        "relevancy": 0.0,
        "accuracy": 0.0,
        "recency_decayed": 0.0,
        "precision_at_k": 0.0,
        "accuracy_memory_off": null,
        "n": 2,
        "extra": {}
      },
      "n_tasks": 2,
      "cost_usd": 0.024639,
      "tokens_in": 53,
      "tokens_out": 1632,
      "partial": false,
      "budget_exceeded": false,
      "source": "C:\\Users\\kenhu\\agent-memory-harness\\eval\\tests\\fixtures\\contextbench.json",
      "notes": "real model on bundled fixture (n=2); InMemoryStore"
    }
  ]
}