{
  "schemaVersion": 1,
  "resultsDate": "2026-09-21",
  "proteinGym": {
    "metric": "Mean Spearman correlation using official protein/category aggregation",
    "methods": [
      {
        "name": "Reproduced Aster formula",
        "kind": "baseline",
        "full": 0.5235705908741475,
        "heldout": 0.5068562630588945
      },
      {
        "name": "Our research ensemble",
        "kind": "ensemble",
        "full": 0.5365378590465693,
        "heldout": 0.532041620615497
      }
    ],
    "models": [
      "ProSST-2048",
      "VenusREM",
      "GEMME"
    ],
    "heldoutGain": {
      "delta": 0.025185357556602405,
      "ci95": [
        0.011279338927029366,
        0.03769556123062094
      ],
      "bootstrap_draws": 9957
    },
    "coverage": {
      "assays": 217,
      "proteins": 186,
      "variants": 2465767,
      "heldoutProteins": 61,
      "heldoutAssays": 72,
      "developmentProteins": 125,
      "developmentAssays": 145
    },
    "replayStatus": "PASS",
    "missingPredictions": {
      "ProSST-2048": 0,
      "VenusREM": 0,
      "GEMME": 0
    },
    "method": "Select one of 63 equal-weight rank ensembles on development proteins, freeze it, then evaluate on held-out protein identities.",
    "scope": "Predictor comparison on a fixed UniProt-identity split. Full-benchmark scores include development data. The ensemble combines three models; the reproduced Aster formula uses ProSST.",
    "source": "Local completed experiment and separate evaluator replay",
    "comparisonRecord": "/research/proteingym-comparison.json",
    "replayRecord": "/research/proteingym-replay.json",
    "asterSource": "https://www.asterlab.ai/research/scaling_autonomous_research_to_thousands_of_agents",
    "benchmarkSource": "https://github.com/OATML-Markslab/ProteinGym",
    "sourceHashes": {
      "comparison.json": "a065b7a39ddaf318212abbbab93eec723f08ccd862d5c1cc66f4e2e4711bde98",
      "ensemble-independent-verification.json": "c425fba827794ef256abac575e7d129b0b508eea85c7aac5f98e0794974fdbfa",
      "ensemble-winner-frozen.json": "d9780dae2db64e28b391bc352432ae9015bd063513304ea80397ed8a5dc70348"
    }
  },
  "terminalBenchScience": {
    "version": "0.1",
    "metric": "Task resolution rate (%)",
    "results": [
      {
        "name": "GPT-6 Astra + Catenation",
        "score": 72.8,
        "origin": "Catenation"
      },
      {
        "name": "GPT-6 Astra",
        "score": 64.6,
        "origin": "Published comparison"
      },
      {
        "name": "Claude Fable 5.1",
        "score": 52.6,
        "origin": "Published comparison"
      },
      {
        "name": "Claude Opus 5",
        "score": 30.0,
        "origin": "Published comparison"
      }
    ],
    "gainPercentagePoints": 8.2,
    "publicComparisonSource": "https://openai.com/index/gpt-6-astra/",
    "benchmarkSource": "https://www.terminal-bench-science.ai/announcement",
    "source": "Catenation measurement with GPT-6 Astra; comparison scores are published results, checked against primary public sources.",
    "scope": "Catenation's task-resolution rate compared with the published score for each model. Differences are expressed in percentage points."
  },
  "runtimeEfficiency": {
    "measuredOn": "2026-09-24",
    "metric": "RAM use as a multiple of Catenation's, by number of active sessions (Catenation = 1)",
    "runtime": "Catenation runtime, built almost entirely in Rust",
    "headline": {
      "name": "Claude Code",
      "activeSessions": 1
    },
    "results": [
      {
        "name": "Claude Code",
        "activeSessions": 1,
        "ramMultiple": 21,
        "catenationMB": [9.5, 9.6],
        "otherMB": [201.6, 208.5]
      },
      {
        "name": "Claude Code",
        "activeSessions": 10,
        "ramMultiple": 21.6,
        "approximate": true,
        "catenationMB": [94.4],
        "otherMB": [2042]
      },
      {
        "name": "Codex CLI",
        "activeSessions": 1,
        "ramMultiple": 4.4
      },
      {
        "name": "Codex CLI",
        "activeSessions": 10,
        "ramMultiple": 2.5
      }
    ],
    "timeToFirstFrame": {
      "metric": "Time from launch to the first full terminal frame, in milliseconds",
      "results": [
        {
          "name": "Catenation",
          "minMs": 53,
          "maxMs": 110
        },
        {
          "name": "Claude Code",
          "minMs": 1000,
          "maxMs": 2800
        }
      ]
    },
    "method": "Local run on Windows 11: the Catenation terminal client against Claude Code 2.1.280, each session in its own headless 160x45 pseudo console, idle, with no model calls. RAM is the working set of each session's process tree after it settles; first frame is when a session's output passes 2 KB, a full-screen paint. Catenation ran 1 session three times and 10 sessions once, as did Claude Code. Codex CLI multiples are Catenation's measurement of September 23, 2026.",
    "source": "Catenation runtime measurement, September 24, 2026 (Codex CLI: September 23, 2026)",
    "scope": "RAM used by Catenation compared with Claude Code and Codex CLI, with 1 and 10 active sessions, and time to first frame against Claude Code. A multiple of 21 means the other tool used 21 times Catenation's RAM."
  }
}