Download results/agentic-v6.json from MMLA-ORG/MMLA-Orbit-1B: direct link, hf CLI and curl.
- Browser
- Download file 79.7 kB
-
https://huggingface.co/MMLA-ORG/MMLA-Orbit-1B/resolve/main/results/agentic-v6.json
- Command line
-
hf download hf://MMLA-ORG/MMLA-Orbit-1B/results/agentic-v6.json
-
curl -L -o agentic-v6.json https://huggingface.co/MMLA-ORG/MMLA-Orbit-1B/resolve/main/results/agentic-v6.json
79.7 kB
{
"schema": "orbit-agentic-public-results-v6",
"evaluation_date": "2026-11-06",
"model_weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276",
"coverage": {
"expected_tasks": 3816,
"completed_tasks": 3816,
"unscored_tasks": 0
},
"models": {
"Orbit": {
"label": "Orbit-1B (M + RTMA)",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0.417910447761194,
"primary_unit": "task_success_fraction",
"success_count": 56,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Complete 2026-11-06 published-formatter instruction arm; all134 tasks, fixed published weights",
"evaluation_date": "2026-11-06",
"measurement_protocol": "published-ALF-formatter-original-instruction-fixed-batch8"
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Complete published-reader raw-schema control from calibration comparison; before-calibration arm excluded",
"evaluation_date": "2026-11-06",
"measurement_protocol": "published-reader-raw-schema-control-local"
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -28.27777777777778,
"primary_unit": "native_score_points",
"success_count": 1,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Complete revised-instruction control from 2026-11-06 paired run; original instruction is a separate diagnostic",
"evaluation_date": "2026-11-06",
"measurement_protocol": "revised-instruction-fixed-batch8"
}
},
"identity": {
"weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276",
"config_sha256": "1a42f6b5708ae12d2e485c77fdbe228c7c503a3afbdc771c1cf22f3719015a89",
"deployment_sha256": "e1ef2bde757c64e61b2c8300d6f73f7d6b613ed1597d9ad6ff58c811236ef3d0",
"runtime": {
"api": "mmla_orbit.load_orbit",
"extra": "orbit",
"name": "mmla-rsi",
"repository": "https://github.com/MMLA-org/MMLA-RSI",
"revision": "ea0e4e2ffd140f7f192c6bd23040c521f3672aea",
"version": "0.2.3"
}
},
"inference": {
"alfworld": {
"contextTokens": 4096,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 8192,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 16384,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"openbmb--MiniCPM5-1B-SFT": {
"label": "MiniCPM5-1B-SFT",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.015,
"primary_unit": "task_success_fraction",
"success_count": 3,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -16.5,
"primary_unit": "native_score_points",
"success_count": 0,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "a60b37f1fc409c54e1e337b0723aaac6f92dfec0",
"files": {
"tokenizer.model": {
"bytes": 1181204,
"sha256": "bb74d51116831c3bf65db812c553f94ab0c88dcf97a5bbb37e3504f6d359c530"
},
"special_tokens_map.json": {
"bytes": 551,
"sha256": "82d96d7a9e6ced037f12394b7ea6a5b02e6ca87e0d11edaa8d60d9be857ce7db"
},
"tokenizer_config.json": {
"bytes": 94416,
"sha256": "094efb3cf1ff412284cc5945fc99dff58673a912760d04483a04aa1c716f66fd"
},
"chat_template.jinja": {
"bytes": 9062,
"sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c"
},
"tokenizer.json": {
"bytes": 9894271,
"sha256": "3e065a558a034185fe299917b398685c1facd0169a9eea1e629eb30c171fed81"
},
"UPSTREAM_README.md": {
"bytes": 23444,
"sha256": "b9e4a7e8abf5b41f08489b5a87478f49028f3adb4356d6e1748218c14dc0ebee"
},
"model.safetensors.index.json": {
"bytes": 18004,
"sha256": "1b2eaa65742b9cadec7a95d65d8533208d6294b1952ee3450af30b6aa4b3f298"
},
"generation_config.json": {
"bytes": 213,
"sha256": "c32cc1dbec6aef8cd6d3310ccd1f6ecebbb124b2c7b3d471b19e136b4567e0b1"
},
"config.json": {
"bytes": 730,
"sha256": "2c129d79de518fd7ba22df4f14a2f8b737157e385dd4c2f8cdbe1c8dec4e0091"
},
"model-00000-of-00001.safetensors": {
"bytes": 2161290944,
"sha256": "3d55612299c140fb1f16180df61ccf8166f48fb4ba7b499435aac6a4055f1675"
}
}
},
"inference": {
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
},
"alfworld": {
"contextTokens": 32768,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17,
"provenance": "unchanged complete revision2 panel, verified reuse",
"max_observed_prompt_tokens": 3179
}
}
},
"openbmb--MiniCPM5-1B": {
"label": "MiniCPM5-1B",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.02,
"primary_unit": "task_success_fraction",
"success_count": 4,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -8.277777777777779,
"primary_unit": "native_score_points",
"success_count": 0,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "87179e5c1f455ef22e6223592d2d61351b525bfc",
"files": {
"README.md": {
"bytes": 23472,
"sha256": "b032e36400dc26ba8d78aa5ad63e4cf8bd17f70bf8551e500335185d8e7d061a"
},
"special_tokens_map.json": {
"bytes": 551,
"sha256": "82d96d7a9e6ced037f12394b7ea6a5b02e6ca87e0d11edaa8d60d9be857ce7db"
},
"tokenizer_config.json": {
"bytes": 94416,
"sha256": "094efb3cf1ff412284cc5945fc99dff58673a912760d04483a04aa1c716f66fd"
},
"tokenizer.json": {
"bytes": 9894271,
"sha256": "3e065a558a034185fe299917b398685c1facd0169a9eea1e629eb30c171fed81"
},
"model.safetensors.index.json": {
"bytes": 18004,
"sha256": "162add042e75abc3d571c4a8679523fa4f1ffc55d1fea25fc6658a19d6e957ee"
},
"generation_config.json": {
"bytes": 213,
"sha256": "92afd6424501426eddcf7e1542f013d19e5987544977b4ee7bd26359bd5fd2ab"
},
"config.json": {
"bytes": 726,
"sha256": "6a6509b646cb3169616c5ffc3196e7ccaf9d4d6bc17b266581d241a31c217714"
},
"model-00000-of-00001.safetensors": {
"bytes": 2161290912,
"sha256": "7ab8fd86563125929be78aeec8cb3969c7ed2ead3be1ab9d3ec0a9fa69c8660d"
},
"chat_template.jinja": {
"bytes": 9062,
"sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c"
}
}
},
"inference": {
"alfworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"Qwen--Qwen2.5-1.5B-Instruct": {
"label": "Qwen2.5-1.5B-Instruct",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.005,
"primary_unit": "task_success_fraction",
"success_count": 1,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -25.355555555555554,
"primary_unit": "native_score_points",
"success_count": 1,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306",
"files": {
"README.md": {
"bytes": 4917,
"sha256": "2e1bcd8bd964728a820be709fa0f7b9dd54817a94fd2254c535df70c5e67fada"
},
"tokenizer_config.json": {
"bytes": 7305,
"sha256": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583"
},
"tokenizer.json": {
"bytes": 7031645,
"sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539"
},
"LICENSE": {
"bytes": 11343,
"sha256": "832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e"
},
"merges.txt": {
"bytes": 1671839,
"sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
},
"generation_config.json": {
"bytes": 242,
"sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6"
},
"vocab.json": {
"bytes": 2776833,
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
},
"config.json": {
"bytes": 660,
"sha256": "98d2ff8cc47488d08a2b0b3acf4eb99ef210779b42bd48605f6b8e36acdbf670"
},
"model.safetensors": {
"bytes": 3087467144,
"sha256": "dd924a11b4c220f385b51ffa522daea7c9f3d850e31b162bb5661df483c6d3ee"
}
}
},
"inference": {
"alfworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"HuggingFaceTB--SmolLM2-1.7B-Instruct": {
"label": "SmolLM2-1.7B-Instruct",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": 1.9222222222222223,
"primary_unit": "native_score_points",
"success_count": 0,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "31b70e2e869a7173562077fd711b654946d38674",
"files": {
"README.md": {
"bytes": 14928,
"sha256": "b00c0570dbc43674a0028d088fc72f40d83eea90f47cd987706ca2f95ed114c0"
},
"special_tokens_map.json": {
"bytes": 655,
"sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52"
},
"tokenizer_config.json": {
"bytes": 3764,
"sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df"
},
"tokenizer.json": {
"bytes": 2104556,
"sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c"
},
"merges.txt": {
"bytes": 466391,
"sha256": "0b54e8aa4e53d5383e2e4bc635a56b43f9647f7b13832d5d9ecd8f82dac4f510"
},
"generation_config.json": {
"bytes": 132,
"sha256": "87b916edaaab66b3899b9d0dd0752727dff6666686da0504d89ae0a6e055a013"
},
"vocab.json": {
"bytes": 800662,
"sha256": "82b84012e3add4d01d12ba14442026e49b8cbbaead1f79ecf3d919784f82dc79"
},
"config.json": {
"bytes": 908,
"sha256": "994f50b16abb4ae00880baefe03c10260b5bd608d2bf586f7056ca05a534feea"
},
"model.safetensors": {
"bytes": 3422777952,
"sha256": "f55217be716b6a997b97b9d8d7eb6fad02e00858f5010ec24f64603c3a98a0e8"
}
}
},
"inference": {
"alfworld": {
"contextTokens": 8192,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 8192,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 8192,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"LiquidAI--LFM2.5-1.2B-Instruct": {
"label": "LFM2.5-1.2B-Instruct",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.015,
"primary_unit": "task_success_fraction",
"success_count": 3,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": 2.922222222222222,
"primary_unit": "native_score_points",
"success_count": 1,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "0f604ada3f766f9f257460c4c9f0b5d6f69d431b",
"files": {
"LICENSE": {
"bytes": 10644,
"sha256": "61d7e939a05911c765b7e98ffaa1ab5ca6c0174a65350766c25cb10197d19fc8"
},
"README.md": {
"bytes": 16291,
"sha256": "682de876c2d172eaeffff775ad8422b5fd7ada701543916e62129e3be8eab077"
},
"chat_template.jinja": {
"bytes": 5487,
"sha256": "ba551d58630afa3190b1be3602e28301f3d2e9bbac978dfc49d6d825171648b6"
},
"config.json": {
"bytes": 1224,
"sha256": "15d6157fb6df3f8272e2fe90e18f57727ccf02a125c94469198b0f3281510185"
},
"generation_config.json": {
"bytes": 132,
"sha256": "5ffd97da1dec4308543894569662d96e923ed01f7a9d8c7ff5aea7f800738cbd"
},
"model.safetensors": {
"bytes": 2340697936,
"sha256": "1ba63d9adb03ae43581db0e136e4416febe0441aff7296397bd455fb6017f73a"
},
"special_tokens_map.json": {
"bytes": 434,
"sha256": "742aefe2b7dec496e8caffdba03a75d0c1a9925d53bd3f3e0d388c96b591b6f4"
},
"tokenizer.json": {
"bytes": 4733389,
"sha256": "df1d8d5ec5d091b460562ffd545e4a5e91d17d4a0db7ebe733be34ed374377bd"
},
"tokenizer_config.json": {
"bytes": 92225,
"sha256": "2a52ec012d3df831ba434b081bef3726a6ee22501f062ad8353c557a0cfa0d01"
}
}
},
"inference": {
"alfworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"HuggingFaceTB--SmolLM3-3B": {
"label": "SmolLM3-3B",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.035,
"primary_unit": "task_success_fraction",
"success_count": 7,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -22.055555555555557,
"primary_unit": "native_score_points",
"success_count": 2,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "a07cc9a04f16550a088caea529712d1d335b0ac1",
"files": {
"README.md": {
"bytes": 17452,
"sha256": "24f16f90b883af3618425a589783a944356a7fc2e3cfa69b579c9ff06963e178"
},
"chat_template.jinja": {
"bytes": 5598,
"sha256": "3e0e275a527f249670056e0078e474eadbdd007c134b4fdfae5a7b3089702814"
},
"config.json": {
"bytes": 1915,
"sha256": "c72b1031274ff4626e434d0019e88e95a767460135db9ee492eb80652b786af1"
},
"generation_config.json": {
"bytes": 182,
"sha256": "3458d62b3f9654f4bbe285223b0f4e5a1e67cae61c92bb6aeaadf04468bf9e01"
},
"model-00001-of-00002.safetensors": {
"bytes": 4966315264,
"sha256": "e5eed4113a925264c33e3cdf76cd9f547c06a207ef2270914a7274a2bf685ffd"
},
"model-00002-of-00002.safetensors": {
"bytes": 1183919744,
"sha256": "6c3ad90646457295723e4da5ee8afac47006e20407b122df5174693dd8d68a43"
},
"model.safetensors.index.json": {
"bytes": 26899,
"sha256": "e3a7254c086c4a78f95ad5864f435d165ffed1ed8998326ce5107b4423c15b76"
},
"special_tokens_map.json": {
"bytes": 289,
"sha256": "12403efb970f069ba62ddf3b66166a27254be4a198166b61ad1ee604bd3a4515"
},
"tokenizer.json": {
"bytes": 17208819,
"sha256": "7b6a500b662a34eb3f0374db856ba4ad7de4c81040571d78dc0d357238930005"
},
"tokenizer_config.json": {
"bytes": 50387,
"sha256": "59a64b39f05ee0acf7479bbccf99a44d9e481c1a4a494eb03838ae3c35eb29be"
}
}
},
"inference": {
"alfworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"Qwen--Qwen3.5-4B": {
"label": "Qwen3.5-4B",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0.03731343283582089,
"primary_unit": "task_success_fraction",
"success_count": 5,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.465,
"primary_unit": "task_success_fraction",
"success_count": 93,
"status_counts": {
"COMPLETE": 200
},
"measurement_protocol": "orbit-agentic-interface-v3.1-qwen-prelude",
"evaluation_date": "2026-11-05",
"historical_v3_success_count": 69
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -9.311111111111112,
"primary_unit": "native_score_points",
"success_count": 3,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
"files": {
"LICENSE": {
"bytes": 11544,
"sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
},
"README.md": {
"bytes": 77661,
"sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb"
},
"chat_template.jinja": {
"bytes": 7756,
"sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715"
},
"config.json": {
"bytes": 3161,
"sha256": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670"
},
"merges.txt": {
"bytes": 3353259,
"sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
},
"model.safetensors-00001-of-00002.safetensors": {
"bytes": 5329398688,
"sha256": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61"
},
"model.safetensors-00002-of-00002.safetensors": {
"bytes": 3990429408,
"sha256": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188"
},
"model.safetensors.index.json": {
"bytes": 76196,
"sha256": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d"
},
"preprocessor_config.json": {
"bytes": 390,
"sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
},
"tokenizer.json": {
"bytes": 12807982,
"sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"
},
"tokenizer_config.json": {
"bytes": 16710,
"sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8"
},
"video_preprocessor_config.json": {
"bytes": 385,
"sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
},
"vocab.json": {
"bytes": 6722759,
"sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
}
}
},
"inference": {
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
},
"alfworld": {
"contextTokens": 32768,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17,
"provenance": "unchanged complete revision2 panel, verified reuse",
"max_observed_prompt_tokens": 4673
}
}
},
"Qwen--Qwen3.5-9B": {
"label": "Qwen3.5-9B",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0.2462686567164179,
"primary_unit": "task_success_fraction",
"success_count": 33,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.605,
"primary_unit": "task_success_fraction",
"success_count": 121,
"status_counts": {
"COMPLETE": 200
},
"measurement_protocol": "orbit-agentic-interface-v3.1-qwen-prelude",
"evaluation_date": "2026-11-05",
"historical_v3_success_count": 91
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -24.3,
"primary_unit": "native_score_points",
"success_count": 8,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction."
}
},
"identity": {
"revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
"files": {
"LICENSE": {
"bytes": 11544,
"sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
},
"README.md": {
"bytes": 77643,
"sha256": "c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0"
},
"chat_template.jinja": {
"bytes": 7756,
"sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715"
},
"config.json": {
"bytes": 3126,
"sha256": "d0883072e01861ed0b2d47be3c16c36a8e81c224c7ffaa310c6558fb3f932b05"
},
"merges.txt": {
"bytes": 3353259,
"sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
},
"model.safetensors-00001-of-00004.safetensors": {
"bytes": 5276436216,
"sha256": "db6f444b43d318c92f360a13a25561a6a65b10c0631b8ed305a426dbaa6c380e"
},
"model.safetensors-00002-of-00004.safetensors": {
"bytes": 5335161512,
"sha256": "31c7d7e2dd5d207840b31cc59083c8f4c4718959149e0358c0364052bb9a0330"
},
"model.safetensors-00003-of-00004.safetensors": {
"bytes": 5368717440,
"sha256": "7ec36ba3a4176a44c3c0876ad80c56a2f70c84bf008d82e9501df642f17dadec"
},
"model.safetensors-00004-of-00004.safetensors": {
"bytes": 3325995712,
"sha256": "b62b0c4cd7e44edee103ee8f4fe225f246d5e768e07bfd5f25b63a8aa1fdd0c6"
},
"model.safetensors.index.json": {
"bytes": 79657,
"sha256": "26d3539b516be613f39563617cb9d33b3f83d401298125be392c80cefb8f7fe5"
},
"preprocessor_config.json": {
"bytes": 390,
"sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
},
"tokenizer.json": {
"bytes": 12807982,
"sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42"
},
"tokenizer_config.json": {
"bytes": 16710,
"sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8"
},
"video_preprocessor_config.json": {
"bytes": 385,
"sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
},
"vocab.json": {
"bytes": 6722759,
"sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
}
}
},
"inference": {
"scienceworld": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 32768,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
},
"alfworld": {
"contextTokens": 32768,
"nativeTools": false,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17,
"provenance": "unchanged complete revision2 panel, verified reuse",
"max_observed_prompt_tokens": 6040
}
}
}
},
"protocol": {
"host": "External and earlier panels use the OhMyPi-based dispatcher. The latest Orbit ALF/Science controls use the frozen Python task/environment adapters with the direct Orbit owner and a fixed synchronous schedule; the BFCL control uses its retained reviewed dispatcher. Host implementation and scheduling are not identical across all cells.",
"panels": {
"alfworld": "134 official seen-regression task identities",
"bfcl": "200 official multi_turn_base cases only",
"scienceworld": "30 task classes, 3 fixed official test variations per class"
},
"primary_metrics": {
"alfworld": "complete task success fraction",
"bfcl": "official multi-turn audit success fraction",
"scienceworld": "mean native final score, range -100..100"
},
"decode": "greedy; thinking disabled",
"reply_limit": 64,
"output_token_limits": {
"alfworld": 64,
"bfcl": 512,
"scienceworld": 64
},
"orbit": "Saved trained Phi fixed; current observation plus eight-slot authoritative M; deployed RTMA rule, no host transcript.",
"external": "Saved chat/tool templates where available; MiniCPM5-1B uses the MiniCPM5-1B-SFT template override; ordinary conversation history, no added memory module.",
"feedback": "Tools rendered in model templates, raw textual calls decoded by bounded adapters, tool results returned as ordinary user observations; tool_choice=none in external BFCL requests. Not the full native tool-role message protocol.",
"same_information_boundary": "Public instructions, tools and returned observations only; no hidden valid-action list, future questions or test answers.",
"runtime": {
"external": "vLLM0.29.0 / torch2.13 cu130 / transformers5.17",
"orbit": "torch2.11.0+cu128 / transformers5.12.1"
},
"reuse": "All panels have been inspected previously. ALF134 is a seen regression panel. Latest complete Orbit runs are used in full; all3392 external rows are unchanged from their reviewed sources. No per-task best-of selection.",
"scope": "Finite whole-system regression comparison; not unrestricted official leaderboards or a causal architecture ablation.",
"MiniCPM5_1B_template_override_sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c",
"seeds": {
"generation": 17,
"declared_task_seed": 20261017,
"scienceworld_environment_seed_applied": false,
"scienceworld_scope": "Host task-seed argument was not forwarded. Official1.2.3 doLoad initializes Scala Random and EnvironmentMaker with variationIdx. Fixed task/variation identities do not prove exact replay of every historical JVM state or collection ordering."
},
"context_tokens": {
"Orbit": {
"alfworld": 4096,
"scienceworld": 8192,
"bfcl": 16384
},
"openbmb--MiniCPM5-1B-SFT": {
"scienceworld": 32768,
"bfcl": 32768,
"alfworld": 32768
},
"openbmb--MiniCPM5-1B": {
"alfworld": 32768,
"scienceworld": 32768,
"bfcl": 32768
},
"Qwen--Qwen2.5-1.5B-Instruct": {
"alfworld": 32768,
"scienceworld": 32768,
"bfcl": 32768
},
"HuggingFaceTB--SmolLM2-1.7B-Instruct": {
"alfworld": 8192,
"scienceworld": 8192,
"bfcl": 8192
},
"LiquidAI--LFM2.5-1.2B-Instruct": {
"alfworld": 32768,
"scienceworld": 32768,
"bfcl": 32768
},
"HuggingFaceTB--SmolLM3-3B": {
"alfworld": 32768,
"scienceworld": 32768,
"bfcl": 32768
},
"Qwen--Qwen3.5-4B": {
"scienceworld": 32768,
"bfcl": 32768,
"alfworld": 32768
},
"Qwen--Qwen3.5-9B": {
"scienceworld": 32768,
"bfcl": 32768,
"alfworld": 32768
}
},
"BFCL_correction": {
"model_ids": [
"Qwen--Qwen3.5-4B",
"Qwen--Qwen3.5-9B"
],
"tasks_per_model": 200,
"decoder_sha256": "4e5d917ad1780b96dc858989431725930257d1c63f46baf3671a5dd539ecb495",
"plan_sha256": "6c30bd4e3e52cf22bc8e26bca944621a042c2ef8b208ee951d89def6a327f0cf",
"change": "Permit plain prose before a complete native Qwen function-call block; preserve names, arguments, complete suffix validation and originally valid literal calls.",
"unchanged": [
"weights",
"prompts",
"task IDs",
"scorer",
"reply/token limits",
"generation seed"
],
"hardware": "Local RTX4090; historical Qwen panels ran on remote RTX PRO6000.",
"score_difference_interpretation": "Descriptive protocol revision, not a paired causal estimate of parser effect."
},
"recovery": "Saved replies and submitted prefixes retained; nine SmolLM3 BFCL tasks explicitly continued after an unreturned request. No double-scoring; unreturned-request physical compute cost is unknown. Orbit timeout/continuation traces independently reconciled to final checkpoints.",
"BFCL_turn_termination": "[] or a parsing rejection ends the current user turn; the 21st nonempty reply in a turn triggers termination, alongside the64-reply task limit. Counts replies, not individual functions.",
"BFCL_schema_diagnostic": {
"arms": {
"gorilla": {
"completed": 200,
"successes": 0,
"calls": 735,
"function_calls": 1,
"input_tokens": 4228908,
"output_tokens": 8244,
"expected": 200,
"primary": 0.0
},
"openapi-types": {
"completed": 200,
"successes": 0,
"calls": 734,
"function_calls": 0,
"input_tokens": 4223973,
"output_tokens": 8481,
"expected": 200,
"primary": 0.0
}
},
"paired_task_outcomes": {
"both_fail": 200
},
"plan_sha256": "5045d3d4e14cd0d3da469848148d6ca132de00d6594c263b555f99a709702e97",
"change": "Only parameter-schema dict/float types converted to object/number; published weights, roles, task IDs, seeds, scorer, parser and budgets fixed.",
"interpretation": "Neither arm completed a task successfully. This does not support this type-only conversion as a sufficient repair; it does not identify every cause of failed tool use or prove all tool interfaces equivalent.",
"table_policy": "Report the raw-schema control; retain the normalized arm as a separate diagnostic. No candidate was installed."
},
"Orbit_ALF_protocol_repair": {
"reason": "Use the ACTION_INSTRUCTION already defined by the released ALFWorldFormatter and its task-state factory; remove the evaluation-specific instruction rewrite, without tuning new text or selecting tasks.",
"original_instruction": "Complete the household task using the current observation and accessible memory. Return exactly one environment command, with no explanation. Public command forms: look; inventory; go to (receptacle); open (receptacle); close (receptacle); take (object) from (receptacle); move (object) to (receptacle); put (object) into (receptacle object); put (receptacle object) in (receptacle); examine (object or receptacle); use (object); heat (object) with (receptacle); clean (object) with (receptacle); cool (object) with (receptacle); slice (object) with (sharp object); help. Use names and numbers from observations.\n",
"original_instruction_sha256": "bfd95b4465189c67f1ea458bbea253ca68a9baa52bb20834412364d691471c3b",
"alternative_instruction": "Complete the household task. Return exactly one executable ALFWorld command, without explanation, quotes, JSON, Markdown, or a numbered list.\nUse complete object and receptacle names with their numbers, exactly as observed. Parenthesized words in command templates are placeholders: replace the whole placeholder; never print parentheses or the placeholder word.\nCommands include: look; inventory; go to RECEPTACLE; open RECEPTACLE; close RECEPTACLE; take OBJECT from RECEPTACLE; move OBJECT to RECEPTACLE; examine OBJECT; use OBJECT; heat OBJECT with RECEPTACLE; cool OBJECT with RECEPTACLE; clean OBJECT with RECEPTACLE.\nFor example, if you see a countertop 3, the command is go to countertop 3. If you are at a shelf 2 and see a book 4 on it, take book 4 from shelf 2 picks it up. These are syntax examples, not facts about your current room.\nGo to a receptacle before interacting with its contents. Open a closed container before taking from it. Hold an object before moving, heating, cooling, or cleaning it. To examine a held object under a lamp, go to the lamp's location and use the lamp by its observed name. Read the returned observation before choosing the next command. A response of Nothing happens means the last command had no effect; use the observation to choose a different valid action.\n",
"alternative_successes": 35,
"default_successes": 56,
"tasks_per_arm": 134,
"full_pairs": "agentic-instruction-pairs.json",
"scope": "One fixed reused panel and seed; conditional total effect, not unique historical cause or unseen generalization.",
"external_models": "External panels retain their original revised text and model-specific templates. This is not an instruction-matched external comparison.",
"implementation": "The released runtime already defines this instruction. Model/runtime weights and deployment.json are unchanged; the benchmark instruction override is the repaired choice."
},
"Orbit_panel_runs": {
"alfworld": {
"source": "01c3202aad900ece8ae4e5b29e3cfa055598180262d6ffb7ec6ab2840390d8ff",
"arm": "original",
"scheduling": "groups8, last6; fixed slots, original thenv3; no refill",
"primary_successes": 56,
"historical_v3_successes": 36
},
"bfcl": {
"source": "b85b7a04a0bfaec4b85ce898ef3e86d223094da3c435b274e4385e8adea850e3",
"arm": "calibrated",
"scheduling": "published-reader arm first, seven shards with up to32 environment episodes; physical generation batch1",
"primary_successes": 0
},
"scienceworld": {
"source": "79efb1fb6d0f16d3459a65d690bd00c9eb6c56e3f89756d6f9411708717c1fd5",
"arm": "v3",
"scheduling": "fixed8-wide groups; original thenv3",
"mean_score": -28.27777777777778,
"successes": 1,
"historical_v3_mean_score": -28.27777777777778,
"historical_v3_successes": 0,
"historical_note": "Seven individual scores changed despite the unchanged mean. New and historical runs are separately retained."
}
}
},
"performance": {
"definitions": "Job wall seconds sum per-device active host intervals across original and continuation segments; exclude model startup/cleanup and 402 reused ALF historical cost. Parallel workers have additive time, not total end-to-end wall time. Returned work excludes unreturned responses, whose elapsed cost remains included. Saved replies count once.",
"UTL": "Whole-card samples during the matching job/device intervals; not MFU.",
"MFU": "Not measured.",
"hardware_note": "Historical v3: Orbit ALF134 and BFCL160 on local RTX4090; BFCL40 and Science90 on remote RTX PRO6000; external newly evaluated v3 panels on RTX PRO6000. Current Qwen3.5 BFCL correction: local RTX4090. Mixed hardware/runtime is not a controlled speed benchmark. The complete Orbit BFCL control and normalized-schema diagnostic now both ran locally on RTX4090 with batch1/head256/cache; historical Orbit BFCL hardware is no longer the provenance of the current table cell.",
"historical_v3_rows": [
{
"model": "HuggingFaceTB--SmolLM2-1.7B-Instruct",
"panel": "alfworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 259.0365707874298,
"returned_work": {
"replies": 8576,
"input_tokens": 18604396,
"output_tokens": 190888,
"length_finishes": 1792
},
"returned_output_tokens_per_worker_second": 736.9152526214,
"samples": 51,
"mean_UTL": 57.470588235294116,
"mean_W": 249.1407843137255,
"MFU": null
},
{
"model": "HuggingFaceTB--SmolLM2-1.7B-Instruct",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 181.028724193573,
"returned_work": {
"replies": 1293,
"input_tokens": 7497335,
"output_tokens": 185818,
"length_finishes": 183
},
"returned_output_tokens_per_worker_second": 1026.455888852787,
"samples": 36,
"mean_UTL": 80.5,
"mean_W": 374.84277777777777,
"MFU": null
},
{
"model": "HuggingFaceTB--SmolLM2-1.7B-Instruct",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 282.05963945388794,
"returned_work": {
"replies": 5760,
"input_tokens": 17446306,
"output_tokens": 92370,
"length_finishes": 5
},
"returned_output_tokens_per_worker_second": 327.4839327556503,
"samples": 55,
"mean_UTL": 62.018181818181816,
"mean_W": 236.7987272727273,
"MFU": null
},
{
"model": "HuggingFaceTB--SmolLM3-3B",
"panel": "alfworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 242.03773832321167,
"returned_work": {
"replies": 8576,
"input_tokens": 16628138,
"output_tokens": 123997,
"length_finishes": 300
},
"returned_output_tokens_per_worker_second": 512.3044069863901,
"samples": 47,
"mean_UTL": 47.1063829787234,
"mean_W": 216.55489361702126,
"MFU": null
},
{
"model": "HuggingFaceTB--SmolLM3-3B",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 190.18724155426025,
"returned_work": {
"replies": 2325,
"input_tokens": 15063016,
"output_tokens": 111950,
"length_finishes": 13,
"unreturned_model_calls": 9
},
"returned_output_tokens_per_worker_second": 588.6304416905945,
"samples": 38,
"mean_UTL": 73.5,
"mean_W": 295.78473684210525,
"MFU": null
},
{
"model": "HuggingFaceTB--SmolLM3-3B",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 223.03466820716858,
"returned_work": {
"replies": 4189,
"input_tokens": 12390270,
"output_tokens": 39946,
"length_finishes": 103
},
"returned_output_tokens_per_worker_second": 179.10220111115484,
"samples": 44,
"mean_UTL": 53.13636363636363,
"mean_W": 211.91886363636365,
"MFU": null
},
{
"model": "LiquidAI--LFM2.5-1.2B-Instruct",
"panel": "alfworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 281.0382556915283,
"returned_work": {
"replies": 8576,
"input_tokens": 15082957,
"output_tokens": 63816,
"length_finishes": 70
},
"returned_output_tokens_per_worker_second": 227.07228894149333,
"samples": 55,
"mean_UTL": 15.127272727272727,
"mean_W": 126.92236363636363,
"MFU": null
},
{
"model": "LiquidAI--LFM2.5-1.2B-Instruct",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 77.01419711112976,
"returned_work": {
"replies": 1215,
"input_tokens": 8600438,
"output_tokens": 52585,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 682.79618528154,
"samples": 14,
"mean_UTL": 43.857142857142854,
"mean_W": 218.01642857142858,
"MFU": null
},
{
"model": "LiquidAI--LFM2.5-1.2B-Instruct",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 297.06167936325073,
"returned_work": {
"replies": 5699,
"input_tokens": 17273159,
"output_tokens": 82705,
"length_finishes": 130
},
"returned_output_tokens_per_worker_second": 278.4101947355764,
"samples": 58,
"mean_UTL": 31.551724137931036,
"mean_W": 165.83982758620692,
"MFU": null
},
{
"model": "Orbit",
"panel": "alfworld",
"hardware": "local-4090",
"job_wall_seconds": 937.1650896072388,
"returned_work": {
"replies": 6523,
"input_tokens": 3388281,
"output_tokens": 45529,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 48.58162185606058,
"samples": 187,
"mean_UTL": 62.45454545454545,
"mean_W": 317.73561497326205,
"MFU": null
},
{
"model": "Orbit",
"panel": "bfcl",
"hardware": "local-4090",
"job_wall_seconds": 4504.695177793503,
"returned_work": {
"replies": 567,
"input_tokens": 3153962,
"output_tokens": 5739,
"length_finishes": 1,
"saved_reply_reuses": 20,
"unreturned_model_calls": 4
},
"returned_output_tokens_per_worker_second": 1.27400407208265,
"samples": 898,
"mean_UTL": 98.70601336302896,
"mean_W": 389.1228173719377,
"MFU": null
},
{
"model": "Orbit",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 1362.1481802463531,
"returned_work": {
"replies": 168,
"input_tokens": 1074946,
"output_tokens": 2673,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 1.9623415710297913,
"samples": 267,
"mean_UTL": 96.12359550561797,
"mean_W": 419.8287265917603,
"MFU": null
},
{
"model": "Orbit",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 799.1107132434845,
"returned_work": {
"replies": 4004,
"input_tokens": 2882017,
"output_tokens": 21507,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 26.913667460051858,
"samples": 157,
"mean_UTL": 44.33757961783439,
"mean_W": 247.38681528662417,
"MFU": null
},
{
"model": "Qwen--Qwen2.5-1.5B-Instruct",
"panel": "alfworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 225.05171990394592,
"returned_work": {
"replies": 8576,
"input_tokens": 18472352,
"output_tokens": 215577,
"length_finishes": 4
},
"returned_output_tokens_per_worker_second": 957.8998111723394,
"samples": 44,
"mean_UTL": 47.70454545454545,
"mean_W": 191.15840909090912,
"MFU": null
},
{
"model": "Qwen--Qwen2.5-1.5B-Instruct",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 174.0313355922699,
"returned_work": {
"replies": 4320,
"input_tokens": 29030655,
"output_tokens": 157181,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 903.1764277684579,
"samples": 34,
"mean_UTL": 61.294117647058826,
"mean_W": 236.54617647058822,
"MFU": null
},
{
"model": "Qwen--Qwen2.5-1.5B-Instruct",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 184.02459573745728,
"returned_work": {
"replies": 4124,
"input_tokens": 12236946,
"output_tokens": 24886,
"length_finishes": 34
},
"returned_output_tokens_per_worker_second": 135.23192321261314,
"samples": 36,
"mean_UTL": 33.75,
"mean_W": 158.09333333333333,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 317.0433175563812,
"returned_work": {
"replies": 3403,
"input_tokens": 24051404,
"output_tokens": 172457,
"length_finishes": 0,
"unreturned_model_calls": 1
},
"returned_output_tokens_per_worker_second": 543.9540606918208,
"samples": 62,
"mean_UTL": 71.20967741935483,
"mean_W": 307.3567741935484,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 281.0388207435608,
"returned_work": {
"replies": 4393,
"input_tokens": 17336798,
"output_tokens": 21089,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 75.03945520481335,
"samples": 55,
"mean_UTL": 42.89090909090909,
"mean_W": 214.77236363636362,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 214.0323269367218,
"returned_work": {
"replies": 2071,
"input_tokens": 12747517,
"output_tokens": 104506,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 488.2720357981109,
"samples": 41,
"mean_UTL": 72.63414634146342,
"mean_W": 356.6329268292683,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 229.05741500854492,
"returned_work": {
"replies": 3036,
"input_tokens": 8960212,
"output_tokens": 19179,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 83.73009884567384,
"samples": 45,
"mean_UTL": 56.24444444444445,
"mean_W": 299.9644444444445,
"MFU": null
},
{
"model": "openbmb--MiniCPM5-1B",
"panel": "alfworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 213.0453381538391,
"returned_work": {
"replies": 8576,
"input_tokens": 13913501,
"output_tokens": 62761,
"length_finishes": 128
},
"returned_output_tokens_per_worker_second": 294.5898771776013,
"samples": 42,
"mean_UTL": 21.023809523809526,
"mean_W": 135.07190476190476,
"MFU": null
},
{
"model": "openbmb--MiniCPM5-1B",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 143.0234076976776,
"returned_work": {
"replies": 4196,
"input_tokens": 28075473,
"output_tokens": 110918,
"length_finishes": 9
},
"returned_output_tokens_per_worker_second": 775.5234040742343,
"samples": 28,
"mean_UTL": 56.142857142857146,
"mean_W": 210.02535714285713,
"MFU": null
},
{
"model": "openbmb--MiniCPM5-1B",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 217.0302734375,
"returned_work": {
"replies": 5209,
"input_tokens": 14038149,
"output_tokens": 35960,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 165.6911703166411,
"samples": 43,
"mean_UTL": 25.558139534883722,
"mean_W": 151.32302325581395,
"MFU": null
},
{
"model": "openbmb--MiniCPM5-1B-SFT",
"panel": "bfcl",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 134.02470660209656,
"returned_work": {
"replies": 4119,
"input_tokens": 27551807,
"output_tokens": 102810,
"length_finishes": 1
},
"returned_output_tokens_per_worker_second": 767.0973703768715,
"samples": 26,
"mean_UTL": 54.80769230769231,
"mean_W": 209.4953846153846,
"MFU": null
},
{
"model": "openbmb--MiniCPM5-1B-SFT",
"panel": "scienceworld",
"hardware": "rtx-pro6000-2",
"job_wall_seconds": 204.0337872505188,
"returned_work": {
"replies": 4693,
"input_tokens": 12643305,
"output_tokens": 39153,
"length_finishes": 0
},
"returned_output_tokens_per_worker_second": 191.8946882651684,
"samples": 40,
"mean_UTL": 28.6,
"mean_W": 154.87824999999998,
"MFU": null
}
],
"correction_rows": [
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 54.01022696495056,
"returned_work": {
"replies": 424,
"input_tokens": 2473167,
"output_tokens": 15811,
"length_finishes": 0
},
"returned_tokens_per_job_second": 292.7408546211147,
"mean_UTL": 100.0,
"mean_W": 397.965,
"samples": 10,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 50.00976538658142,
"returned_work": {
"replies": 381,
"input_tokens": 2562091,
"output_tokens": 16803,
"length_finishes": 0
},
"returned_tokens_per_job_second": 335.99437770025145,
"mean_UTL": 90.0,
"mean_W": 372.612,
"samples": 10,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 41.00820732116699,
"returned_work": {
"replies": 404,
"input_tokens": 2717661,
"output_tokens": 18745,
"length_finishes": 0
},
"returned_tokens_per_job_second": 457.10361960456856,
"mean_UTL": 77.77777777777777,
"mean_W": 350.5855555555556,
"samples": 9,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 37.007455587387085,
"returned_work": {
"replies": 334,
"input_tokens": 1776928,
"output_tokens": 16748,
"length_finishes": 0
},
"returned_tokens_per_job_second": 452.5574572521562,
"mean_UTL": 100.0,
"mean_W": 411.06,
"samples": 7,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 45.008026123046875,
"returned_work": {
"replies": 302,
"input_tokens": 1707315,
"output_tokens": 17328,
"length_finishes": 0
},
"returned_tokens_per_job_second": 384.99799908192375,
"mean_UTL": 88.88888888888889,
"mean_W": 384.6388888888889,
"samples": 9,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 57.01064419746399,
"returned_work": {
"replies": 349,
"input_tokens": 2479362,
"output_tokens": 28498,
"length_finishes": 0
},
"returned_tokens_per_job_second": 499.87156611129257,
"mean_UTL": 100.0,
"mean_W": 400.16636363636366,
"samples": 11,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-9B",
"seconds": 35.00768256187439,
"returned_work": {
"replies": 83,
"input_tokens": 562334,
"output_tokens": 7417,
"length_finishes": 0
},
"returned_tokens_per_job_second": 211.86778036194798,
"mean_UTL": 94.57142857142857,
"mean_W": 337.7657142857143,
"samples": 7,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 38.0064799785614,
"returned_work": {
"replies": 534,
"input_tokens": 3220502,
"output_tokens": 19397,
"length_finishes": 0
},
"returned_tokens_per_job_second": 510.36033884067695,
"mean_UTL": 81.875,
"mean_W": 357.9125,
"samples": 8,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 34.0051965713501,
"returned_work": {
"replies": 493,
"input_tokens": 3453983,
"output_tokens": 19236,
"length_finishes": 0
},
"returned_tokens_per_job_second": 565.6782474301774,
"mean_UTL": 85.71428571428571,
"mean_W": 380.8442857142857,
"samples": 7,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 52.008928537368774,
"returned_work": {
"replies": 605,
"input_tokens": 4492497,
"output_tokens": 27984,
"length_finishes": 0
},
"returned_tokens_per_job_second": 538.0614595798355,
"mean_UTL": 100.0,
"mean_W": 393.69,
"samples": 10,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 45.008227825164795,
"returned_work": {
"replies": 458,
"input_tokens": 2725874,
"output_tokens": 26274,
"length_finishes": 0
},
"returned_tokens_per_job_second": 583.7599316743105,
"mean_UTL": 100.0,
"mean_W": 399.8077777777778,
"samples": 9,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 47.0073606967926,
"returned_work": {
"replies": 480,
"input_tokens": 3140364,
"output_tokens": 28119,
"length_finishes": 0
},
"returned_tokens_per_job_second": 598.1829139775255,
"mean_UTL": 100.0,
"mean_W": 398.57,
"samples": 9,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 57.00828671455383,
"returned_work": {
"replies": 596,
"input_tokens": 4792821,
"output_tokens": 38110,
"length_finishes": 0
},
"returned_tokens_per_job_second": 668.499304159421,
"mean_UTL": 83.33333333333333,
"mean_W": 361.60833333333335,
"samples": 12,
"MFU": null
},
{
"model": "Qwen--Qwen3.5-4B",
"seconds": 46.00865983963013,
"returned_work": {
"replies": 161,
"input_tokens": 1305314,
"output_tokens": 12610,
"length_finishes": 0
},
"returned_tokens_per_job_second": 274.07883741786844,
"mean_UTL": 88.88888888888889,
"mean_W": 336.5588888888889,
"samples": 9,
"MFU": null
}
],
"correction_cost_scope": "Job intervals include the failed first runner interval but exclude server warmup/cleanup. Original failed segment preserved. Zero rental/API charges; local compute was used. MFU not measured.",
"Orbit_schema_pair_rows": [
{
"arm": "gorilla",
"wall_seconds": 545.0869147777557,
"returned_work": {
"returned_calls": 106,
"input_tokens": 555495,
"output_tokens": 919,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.685969659305979,
"mean_GPU_UTL_percent": 98.68518518518519,
"mean_GPU_power_W": 385.2319444444444,
"GPU_samples": 108,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 496.0800075531006,
"returned_work": {
"returned_calls": 104,
"input_tokens": 648013,
"output_tokens": 649,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.308256712866081,
"mean_GPU_UTL_percent": 96.84848484848484,
"mean_GPU_power_W": 384.8429292929293,
"GPU_samples": 99,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 666.1066203117371,
"returned_work": {
"returned_calls": 105,
"input_tokens": 668554,
"output_tokens": 796,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.1950038863560206,
"mean_GPU_UTL_percent": 97.39097744360902,
"mean_GPU_power_W": 385.1439097744361,
"GPU_samples": 133,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 656.1069223880768,
"returned_work": {
"returned_calls": 131,
"input_tokens": 643616,
"output_tokens": 1385,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 2.1109364232264487,
"mean_GPU_UTL_percent": 98.17557251908397,
"mean_GPU_power_W": 388.6147328244275,
"GPU_samples": 131,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 1296.2045104503632,
"returned_work": {
"returned_calls": 121,
"input_tokens": 638284,
"output_tokens": 1990,
"length_finishes": 1
},
"returned_output_tokens_per_job_second": 1.535251562508897,
"mean_GPU_UTL_percent": 99.06589147286822,
"mean_GPU_power_W": 389.8410465116279,
"GPU_samples": 258,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 1469.1985957622528,
"returned_work": {
"returned_calls": 135,
"input_tokens": 874233,
"output_tokens": 1892,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.2877768910597063,
"mean_GPU_UTL_percent": 99.18150684931507,
"mean_GPU_power_W": 388.42,
"GPU_samples": 292,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "gorilla",
"wall_seconds": 422.0683341026306,
"returned_work": {
"returned_calls": 33,
"input_tokens": 200713,
"output_tokens": 613,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.4523714537915138,
"mean_GPU_UTL_percent": 97.07058823529412,
"mean_GPU_power_W": 381.9274117647059,
"GPU_samples": 85,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 688.0890183448792,
"returned_work": {
"returned_calls": 106,
"input_tokens": 555495,
"output_tokens": 1428,
"length_finishes": 1
},
"returned_output_tokens_per_job_second": 2.075312876573577,
"mean_GPU_UTL_percent": 98.61313868613139,
"mean_GPU_power_W": 388.10335766423356,
"GPU_samples": 137,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 475.07875299453735,
"returned_work": {
"returned_calls": 104,
"input_tokens": 648013,
"output_tokens": 632,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.3303057567115972,
"mean_GPU_UTL_percent": 98.21052631578948,
"mean_GPU_power_W": 383.17726315789474,
"GPU_samples": 95,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 698.1144533157349,
"returned_work": {
"returned_calls": 105,
"input_tokens": 668554,
"output_tokens": 809,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.1588357699193992,
"mean_GPU_UTL_percent": 97.79856115107914,
"mean_GPU_power_W": 386.07755395683455,
"GPU_samples": 139,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 618.0961883068085,
"returned_work": {
"returned_calls": 131,
"input_tokens": 643616,
"output_tokens": 1309,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 2.1177933544386187,
"mean_GPU_UTL_percent": 98.21951219512195,
"mean_GPU_power_W": 389.5132520325203,
"GPU_samples": 123,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 924.1439681053162,
"returned_work": {
"returned_calls": 121,
"input_tokens": 638284,
"output_tokens": 1571,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.6999515813762986,
"mean_GPU_UTL_percent": 99.22826086956522,
"mean_GPU_power_W": 389.6759239130435,
"GPU_samples": 184,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 1571.249398469925,
"returned_work": {
"returned_calls": 134,
"input_tokens": 869298,
"output_tokens": 2018,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.2843282562049785,
"mean_GPU_UTL_percent": 99.33226837060703,
"mean_GPU_power_W": 388.8350479233227,
"GPU_samples": 313,
"MFU": null,
"CPU_seconds": null
},
{
"arm": "openapi-types",
"wall_seconds": 505.08206009864807,
"returned_work": {
"returned_calls": 33,
"input_tokens": 200713,
"output_tokens": 714,
"length_finishes": 0
},
"returned_output_tokens_per_job_second": 1.4136316777130196,
"mean_GPU_UTL_percent": 98.01980198019803,
"mean_GPU_power_W": 386.79940594059406,
"GPU_samples": 101,
"MFU": null,
"CPU_seconds": null
}
],
"Orbit_schema_pair_resource_observation": {
"reservation_start_unix": 1793857185.8012683,
"terminal_file_mtime_unix": 1793868501.913097,
"reservation_wall_seconds_by_file_mtime": 11316.111828565598,
"hard_stop_unix": 1793871585.8012683,
"terminal_file_written_after_hard_stop": false,
"CPU_seconds_total": null,
"GPU_reserved_hours_upper_charged": 3.143364396823777,
"rental_CNY": 0,
"paid_API_USD": 0,
"limits": [
"Reservation includes preparation/review/load/save/cleanup and is not GPU kernel time; endpoint uses preserved file mtime",
"Original launcher did not record process CPU totals; unknown, not zero. Collection CPU is separate",
"Per-job observed returned tokens include partial tasks; unreturned requests may have additional work",
"GPU telemetry is whole-card sampling, not MFU or uniquely attributable if shared"
]
},
"current_Orbit_complete_pair_costs": {
"ALF_instruction_pair": {
"resources": {
"wall_seconds": 1657.1963202879997,
"GPU_allocation_seconds": 1657.1963202879997,
"total_CPU_seconds": 2733.054583558,
"total_output_bytes": 151777040,
"CPU_accounting_complete": true,
"hard_limit_exceeded": []
},
"performance": {
"samples_total": 329,
"samples_parsed": 329,
"mean_UTL_percent": 55.18844984802432,
"mean_power_W": 284.32039513677813,
"max_memory_MiB": 24744.0,
"output_tokens_per_GPU_allocation_second": 49.52098854874235,
"MFU": "unknown",
"sample_scope": "whole card, point samples, not per-kernel activity integration"
},
"cost_scope": "Both268 diagnostic endpoints; not134-only model-table cost"
},
"Science_instruction_pair": {
"resources": {
"wall_seconds": 1983.108780568,
"GPU_allocation_seconds": 1983.108780568,
"total_CPU_seconds": 4030.990388231,
"total_output_bytes": 95510422,
"CPU_accounting_complete": true,
"hard_limit_exceeded": []
},
"performance": {
"GPU_samples": 394,
"usable_GPU_samples": 394,
"sampled_mean_UTL_percent": 60.93908629441624,
"sampled_mean_power_W": 296.9608883248731,
"output_tokens_per_allocation_second": 26.21137100966894,
"MFU": null,
"scope": "Whole allocation includes verification/load/init/generation/feedback/M/cleanup. Card samples are not per-task MFU; completed-task tokens are a lower bound if INCOMPLETE."
},
"cost_scope": "Both180 diagnostic endpoints"
},
"BFCL_calibration_pair": {
"GPU_allocation_seconds": 14162.458663880941,
"CPU_seconds": 14259.603018855001,
"cost_scope": "Both400 diagnostic endpoints; not the200 published-reader endpoints alone"
}
}
},
"evidence_sha256": {
"private_summary": "f8494a0f820daeeaaefc3694797c810b4c8ca46fde3401a94a45504a0b3bde8a",
"task_evidence": "b58cbdb37d2b95476aeab69bf37b57f296bc845892ce0d792da78a931406b955",
"collector": "386f3680d65e9dd5c0db13bc32855097cf69d141bafe459b20e3cd1a463b401a",
"ALF_reuse_manifest": "b89c81e9865016047b14500572429f1dbb19eb38029e4892f118b0070b543936",
"historical_public_v3": "68491e44beede30c3429135437383acd7bfdc4ccfff4e0eb8144ce1cb0641dac",
"BFCL_correction_summary": "1c81962f06af035c032c31bf8807bf7e64c527742efc5300fac3dff3ce29feb2",
"BFCL_correction_task_index": "3ee62a66238ef9e7b4732eb9e56326f2e361d09018ccf006de6548ce3c822f4c",
"BFCL_schema_pair_summary": "29b6c23751c7bdfb95e9fdabdce07ff951c7cb6e73ed693013f4dcc8825e1be6",
"private_v4_candidate": "f8b7fd1b924402e79826a75d14e472924886e5d3362c43f734f1ebd7d53c14c7",
"bfcl-schema-result-review.json": "c420fcab372ede427bce7e33dd3ee5cdd894720197fdb22af72e4abf90f840a9",
"alf-science-measurement-review.json": "f7f911dcd9e0c11487d8921a659ab6d9654a66e1ac9533e428c4e3c7a505f0a5",
"ALF_instruction_collection": "0b0cec43606f1f4f29f86f95383c5d9e313703ad57c63c2e36fea74a334f05b3",
"ALF_instruction_result_review": "a5063f53b908be0c10654194e855eb7d60e7038a45e0f76da3cf92be5a993a9e",
"Science_instruction_result": "457f8b048d2dcf20eb44cade6f961f1aa76b4e61b41925c99430673529abde54",
"Science_instruction_result_review": "46bc8f57c0e52a1f1f81b9e1778634e72cd15236a013cc04b31a90502577ac23",
"BFCL_calibration_result_review": "a24520db68013d33b8732f4a9de79d5ae8ec357dfc9bffdc67f425e43538d138"
},
"access": "Aggregate results, identities and hashes are public in this package; raw task traces and detailed execution evidence remain in the private research archive.",
"warnings": [
"ALF134 is a seen regression panel; BFCL200 and Science90 are repeated after a declared interface revision, not new held-out tests.",
"Native tool-template serialization where supported, strict public adapters and bounded64 calls; not a reproduction of unrestricted official leaderboards.",
"Three exact complete ALF panels may be reused with explicit provenance; no best-of selection.",
"ScienceWorld negative scores are retained. Missing positions are not zero-filled.",
"UTL and power are whole-card samples; MFU is not measured. This table evaluates the packaged deployed manager.",
"MiniCPM5-1B template override, context/precision/runtime/hardware differences prevent a strictly matched model ranking.",
"Orbit historical ALF54/35/36 used different protocols. Available evidence does not identify a unique cause of those differences.",
"No accepted/installed RSI successor is represented by this table.",
"ScienceWorld uses variation-based initialization in the pinned engine; the host task seed was not forwarded. Exact historical physical replay is not established.",
"Orbit ALF uses its packaged original instruction; external ALF panels retain their revised instruction. These system configurations do not constitute a prompt-matched causal ranking.",
"The higher ScienceWorld mean under the original instruction is disclosed as a diagnostic; it had0/90 successes and is not selected for the main table."
],
"evaluation_date_scope": "Orbit panels use complete fixed-published-weight runs ending2026-11-06; external models retain their full reviewed panels including Qwen BFCL correction. No new science is performed by this aggregation.",
"historical_results": "agentic-v3.json",
"reproduction_access": "Published per-task IDs/variations/metrics, version identities and hashes support score recomputation; exact runtime snapshots and raw traces are retained privately. This package does not provide a public standalone end-to-end benchmark runner.",
"task_index": {
"file": "results/agentic-tasks-v6.jsonl",
"sha256": "c6fab11838721ec53045184b569233d69e162e05bd4aaad314d57a69d716e5ea",
"scope": "All3816 complete positions; no missing task or zero fill. All424 Orbit rows use whole complete current runs;3392 external rows retained exactly."
}
}
{
"schema": "orbit-agentic-public-results-v6",
"evaluation_date": "2026-11-06",
"model_weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276",
"coverage": {
"expected_tasks": 3816,
"completed_tasks": 3816,
"unscored_tasks": 0
},
"models": {
"Orbit": {
"label": "Orbit-1B (M + RTMA)",
"panels": {
"alfworld": {
"status": "COMPLETE",
"expected": 134,
"completed": 134,
"unscored": 0,
"primary": 0.417910447761194,
"primary_unit": "task_success_fraction",
"success_count": 56,
"status_counts": {
"COMPLETE": 134
},
"provenance": "Complete 2026-11-06 published-formatter instruction arm; all134 tasks, fixed published weights",
"evaluation_date": "2026-11-06",
"measurement_protocol": "published-ALF-formatter-original-instruction-fixed-batch8"
},
"bfcl": {
"status": "COMPLETE",
"expected": 200,
"completed": 200,
"unscored": 0,
"primary": 0.0,
"primary_unit": "task_success_fraction",
"success_count": 0,
"status_counts": {
"COMPLETE": 200
},
"provenance": "Complete published-reader raw-schema control from calibration comparison; before-calibration arm excluded",
"evaluation_date": "2026-11-06",
"measurement_protocol": "published-reader-raw-schema-control-local"
},
"scienceworld": {
"status": "COMPLETE",
"expected": 90,
"completed": 90,
"unscored": 0,
"primary": -28.27777777777778,
"primary_unit": "native_score_points",
"success_count": 1,
"status_counts": {
"COMPLETE": 90
},
"provenance": "Complete revised-instruction control from 2026-11-06 paired run; original instruction is a separate diagnostic",
"evaluation_date": "2026-11-06",
"measurement_protocol": "revised-instruction-fixed-batch8"
}
},
"identity": {
"weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276",
"config_sha256": "1a42f6b5708ae12d2e485c77fdbe228c7c503a3afbdc771c1cf22f3719015a89",
"deployment_sha256": "e1ef2bde757c64e61b2c8300d6f73f7d6b613ed1597d9ad6ff58c811236ef3d0",
"runtime": {
"api": "mmla_orbit.load_orbit",
"extra": "orbit",
"name": "mmla-rsi",
"repository": "https://github.com/MMLA-org/MMLA-RSI",
"revision": "ea0e4e2ffd140f7f192c6bd23040c521f3672aea",
"version": "0.2.3"
}
},
"inference": {
"alfworld": {
"contextTokens": 4096,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"scienceworld": {
"contextTokens": 8192,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 64,
"seed": 17
},
"bfcl": {
"contextTokens": 16384,
"nativeTools": true,
"maxCalls": 64,
"maxTokens": 512,
"seed": 17
}
}
},
"openbmb--MiniCPM5-1B-SFT": {
"label": "MiniCPM5-1B-SFT",
"panels": {
"alfworld": {