{ "schema": "orbit-agentic-public-results-v6", "evaluation_date": "2026-11-06", "model_weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276", "coverage": { "expected_tasks": 3816, "completed_tasks": 3816, "unscored_tasks": 0 }, "models": { "Orbit": { "label": "Orbit-1B (M + RTMA)", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0.417910447761194, "primary_unit": "task_success_fraction", "success_count": 56, "status_counts": { "COMPLETE": 134 }, "provenance": "Complete 2026-11-06 published-formatter instruction arm; all134 tasks, fixed published weights", "evaluation_date": "2026-11-06", "measurement_protocol": "published-ALF-formatter-original-instruction-fixed-batch8" }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 200 }, "provenance": "Complete published-reader raw-schema control from calibration comparison; before-calibration arm excluded", "evaluation_date": "2026-11-06", "measurement_protocol": "published-reader-raw-schema-control-local" }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -28.27777777777778, "primary_unit": "native_score_points", "success_count": 1, "status_counts": { "COMPLETE": 90 }, "provenance": "Complete revised-instruction control from 2026-11-06 paired run; original instruction is a separate diagnostic", "evaluation_date": "2026-11-06", "measurement_protocol": "revised-instruction-fixed-batch8" } }, "identity": { "weight_sha256": "62d83295cb154ce7869c60602e66677b84cde6f33ab886101752caa6805af276", "config_sha256": "1a42f6b5708ae12d2e485c77fdbe228c7c503a3afbdc771c1cf22f3719015a89", "deployment_sha256": "e1ef2bde757c64e61b2c8300d6f73f7d6b613ed1597d9ad6ff58c811236ef3d0", "runtime": { "api": "mmla_orbit.load_orbit", "extra": "orbit", "name": "mmla-rsi", "repository": "https://github.com/MMLA-org/MMLA-RSI", "revision": "ea0e4e2ffd140f7f192c6bd23040c521f3672aea", "version": "0.2.3" } }, "inference": { "alfworld": { "contextTokens": 4096, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 8192, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 16384, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "openbmb--MiniCPM5-1B-SFT": { "label": "MiniCPM5-1B-SFT", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.015, "primary_unit": "task_success_fraction", "success_count": 3, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -16.5, "primary_unit": "native_score_points", "success_count": 0, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "a60b37f1fc409c54e1e337b0723aaac6f92dfec0", "files": { "tokenizer.model": { "bytes": 1181204, "sha256": "bb74d51116831c3bf65db812c553f94ab0c88dcf97a5bbb37e3504f6d359c530" }, "special_tokens_map.json": { "bytes": 551, "sha256": "82d96d7a9e6ced037f12394b7ea6a5b02e6ca87e0d11edaa8d60d9be857ce7db" }, "tokenizer_config.json": { "bytes": 94416, "sha256": "094efb3cf1ff412284cc5945fc99dff58673a912760d04483a04aa1c716f66fd" }, "chat_template.jinja": { "bytes": 9062, "sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c" }, "tokenizer.json": { "bytes": 9894271, "sha256": "3e065a558a034185fe299917b398685c1facd0169a9eea1e629eb30c171fed81" }, "UPSTREAM_README.md": { "bytes": 23444, "sha256": "b9e4a7e8abf5b41f08489b5a87478f49028f3adb4356d6e1748218c14dc0ebee" }, "model.safetensors.index.json": { "bytes": 18004, "sha256": "1b2eaa65742b9cadec7a95d65d8533208d6294b1952ee3450af30b6aa4b3f298" }, "generation_config.json": { "bytes": 213, "sha256": "c32cc1dbec6aef8cd6d3310ccd1f6ecebbb124b2c7b3d471b19e136b4567e0b1" }, "config.json": { "bytes": 730, "sha256": "2c129d79de518fd7ba22df4f14a2f8b737157e385dd4c2f8cdbe1c8dec4e0091" }, "model-00000-of-00001.safetensors": { "bytes": 2161290944, "sha256": "3d55612299c140fb1f16180df61ccf8166f48fb4ba7b499435aac6a4055f1675" } } }, "inference": { "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 }, "alfworld": { "contextTokens": 32768, "nativeTools": false, "maxCalls": 64, "maxTokens": 64, "seed": 17, "provenance": "unchanged complete revision2 panel, verified reuse", "max_observed_prompt_tokens": 3179 } } }, "openbmb--MiniCPM5-1B": { "label": "MiniCPM5-1B", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.02, "primary_unit": "task_success_fraction", "success_count": 4, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -8.277777777777779, "primary_unit": "native_score_points", "success_count": 0, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "87179e5c1f455ef22e6223592d2d61351b525bfc", "files": { "README.md": { "bytes": 23472, "sha256": "b032e36400dc26ba8d78aa5ad63e4cf8bd17f70bf8551e500335185d8e7d061a" }, "special_tokens_map.json": { "bytes": 551, "sha256": "82d96d7a9e6ced037f12394b7ea6a5b02e6ca87e0d11edaa8d60d9be857ce7db" }, "tokenizer_config.json": { "bytes": 94416, "sha256": "094efb3cf1ff412284cc5945fc99dff58673a912760d04483a04aa1c716f66fd" }, "tokenizer.json": { "bytes": 9894271, "sha256": "3e065a558a034185fe299917b398685c1facd0169a9eea1e629eb30c171fed81" }, "model.safetensors.index.json": { "bytes": 18004, "sha256": "162add042e75abc3d571c4a8679523fa4f1ffc55d1fea25fc6658a19d6e957ee" }, "generation_config.json": { "bytes": 213, "sha256": "92afd6424501426eddcf7e1542f013d19e5987544977b4ee7bd26359bd5fd2ab" }, "config.json": { "bytes": 726, "sha256": "6a6509b646cb3169616c5ffc3196e7ccaf9d4d6bc17b266581d241a31c217714" }, "model-00000-of-00001.safetensors": { "bytes": 2161290912, "sha256": "7ab8fd86563125929be78aeec8cb3969c7ed2ead3be1ab9d3ec0a9fa69c8660d" }, "chat_template.jinja": { "bytes": 9062, "sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c" } } }, "inference": { "alfworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "Qwen--Qwen2.5-1.5B-Instruct": { "label": "Qwen2.5-1.5B-Instruct", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.005, "primary_unit": "task_success_fraction", "success_count": 1, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -25.355555555555554, "primary_unit": "native_score_points", "success_count": 1, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306", "files": { "README.md": { "bytes": 4917, "sha256": "2e1bcd8bd964728a820be709fa0f7b9dd54817a94fd2254c535df70c5e67fada" }, "tokenizer_config.json": { "bytes": 7305, "sha256": "5b5d4f65d0acd3b2d56a35b56d374a36cbc1c8fa5cf3b3febbbfabf22f359583" }, "tokenizer.json": { "bytes": 7031645, "sha256": "c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539" }, "LICENSE": { "bytes": 11343, "sha256": "832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e" }, "merges.txt": { "bytes": 1671839, "sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3" }, "generation_config.json": { "bytes": 242, "sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6" }, "vocab.json": { "bytes": 2776833, "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910" }, "config.json": { "bytes": 660, "sha256": "98d2ff8cc47488d08a2b0b3acf4eb99ef210779b42bd48605f6b8e36acdbf670" }, "model.safetensors": { "bytes": 3087467144, "sha256": "dd924a11b4c220f385b51ffa522daea7c9f3d850e31b162bb5661df483c6d3ee" } } }, "inference": { "alfworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "HuggingFaceTB--SmolLM2-1.7B-Instruct": { "label": "SmolLM2-1.7B-Instruct", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": 1.9222222222222223, "primary_unit": "native_score_points", "success_count": 0, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "31b70e2e869a7173562077fd711b654946d38674", "files": { "README.md": { "bytes": 14928, "sha256": "b00c0570dbc43674a0028d088fc72f40d83eea90f47cd987706ca2f95ed114c0" }, "special_tokens_map.json": { "bytes": 655, "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52" }, "tokenizer_config.json": { "bytes": 3764, "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df" }, "tokenizer.json": { "bytes": 2104556, "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" }, "merges.txt": { "bytes": 466391, "sha256": "0b54e8aa4e53d5383e2e4bc635a56b43f9647f7b13832d5d9ecd8f82dac4f510" }, "generation_config.json": { "bytes": 132, "sha256": "87b916edaaab66b3899b9d0dd0752727dff6666686da0504d89ae0a6e055a013" }, "vocab.json": { "bytes": 800662, "sha256": "82b84012e3add4d01d12ba14442026e49b8cbbaead1f79ecf3d919784f82dc79" }, "config.json": { "bytes": 908, "sha256": "994f50b16abb4ae00880baefe03c10260b5bd608d2bf586f7056ca05a534feea" }, "model.safetensors": { "bytes": 3422777952, "sha256": "f55217be716b6a997b97b9d8d7eb6fad02e00858f5010ec24f64603c3a98a0e8" } } }, "inference": { "alfworld": { "contextTokens": 8192, "nativeTools": false, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 8192, "nativeTools": false, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 8192, "nativeTools": false, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "LiquidAI--LFM2.5-1.2B-Instruct": { "label": "LFM2.5-1.2B-Instruct", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.015, "primary_unit": "task_success_fraction", "success_count": 3, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": 2.922222222222222, "primary_unit": "native_score_points", "success_count": 1, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "0f604ada3f766f9f257460c4c9f0b5d6f69d431b", "files": { "LICENSE": { "bytes": 10644, "sha256": "61d7e939a05911c765b7e98ffaa1ab5ca6c0174a65350766c25cb10197d19fc8" }, "README.md": { "bytes": 16291, "sha256": "682de876c2d172eaeffff775ad8422b5fd7ada701543916e62129e3be8eab077" }, "chat_template.jinja": { "bytes": 5487, "sha256": "ba551d58630afa3190b1be3602e28301f3d2e9bbac978dfc49d6d825171648b6" }, "config.json": { "bytes": 1224, "sha256": "15d6157fb6df3f8272e2fe90e18f57727ccf02a125c94469198b0f3281510185" }, "generation_config.json": { "bytes": 132, "sha256": "5ffd97da1dec4308543894569662d96e923ed01f7a9d8c7ff5aea7f800738cbd" }, "model.safetensors": { "bytes": 2340697936, "sha256": "1ba63d9adb03ae43581db0e136e4416febe0441aff7296397bd455fb6017f73a" }, "special_tokens_map.json": { "bytes": 434, "sha256": "742aefe2b7dec496e8caffdba03a75d0c1a9925d53bd3f3e0d388c96b591b6f4" }, "tokenizer.json": { "bytes": 4733389, "sha256": "df1d8d5ec5d091b460562ffd545e4a5e91d17d4a0db7ebe733be34ed374377bd" }, "tokenizer_config.json": { "bytes": 92225, "sha256": "2a52ec012d3df831ba434b081bef3726a6ee22501f062ad8353c557a0cfa0d01" } } }, "inference": { "alfworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "HuggingFaceTB--SmolLM3-3B": { "label": "SmolLM3-3B", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0, "primary_unit": "task_success_fraction", "success_count": 0, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.035, "primary_unit": "task_success_fraction", "success_count": 7, "status_counts": { "COMPLETE": 200 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -22.055555555555557, "primary_unit": "native_score_points", "success_count": 2, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "a07cc9a04f16550a088caea529712d1d335b0ac1", "files": { "README.md": { "bytes": 17452, "sha256": "24f16f90b883af3618425a589783a944356a7fc2e3cfa69b579c9ff06963e178" }, "chat_template.jinja": { "bytes": 5598, "sha256": "3e0e275a527f249670056e0078e474eadbdd007c134b4fdfae5a7b3089702814" }, "config.json": { "bytes": 1915, "sha256": "c72b1031274ff4626e434d0019e88e95a767460135db9ee492eb80652b786af1" }, "generation_config.json": { "bytes": 182, "sha256": "3458d62b3f9654f4bbe285223b0f4e5a1e67cae61c92bb6aeaadf04468bf9e01" }, "model-00001-of-00002.safetensors": { "bytes": 4966315264, "sha256": "e5eed4113a925264c33e3cdf76cd9f547c06a207ef2270914a7274a2bf685ffd" }, "model-00002-of-00002.safetensors": { "bytes": 1183919744, "sha256": "6c3ad90646457295723e4da5ee8afac47006e20407b122df5174693dd8d68a43" }, "model.safetensors.index.json": { "bytes": 26899, "sha256": "e3a7254c086c4a78f95ad5864f435d165ffed1ed8998326ce5107b4423c15b76" }, "special_tokens_map.json": { "bytes": 289, "sha256": "12403efb970f069ba62ddf3b66166a27254be4a198166b61ad1ee604bd3a4515" }, "tokenizer.json": { "bytes": 17208819, "sha256": "7b6a500b662a34eb3f0374db856ba4ad7de4c81040571d78dc0d357238930005" }, "tokenizer_config.json": { "bytes": 50387, "sha256": "59a64b39f05ee0acf7479bbccf99a44d9e481c1a4a494eb03838ae3c35eb29be" } } }, "inference": { "alfworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 } } }, "Qwen--Qwen3.5-4B": { "label": "Qwen3.5-4B", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0.03731343283582089, "primary_unit": "task_success_fraction", "success_count": 5, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.465, "primary_unit": "task_success_fraction", "success_count": 93, "status_counts": { "COMPLETE": 200 }, "measurement_protocol": "orbit-agentic-interface-v3.1-qwen-prelude", "evaluation_date": "2026-11-05", "historical_v3_success_count": 69 }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -9.311111111111112, "primary_unit": "native_score_points", "success_count": 3, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", "files": { "LICENSE": { "bytes": 11544, "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a" }, "README.md": { "bytes": 77661, "sha256": "1406be1b6b8fd8a6545870da516912804756593628a1d0fb0a7965211e82a7bb" }, "chat_template.jinja": { "bytes": 7756, "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" }, "config.json": { "bytes": 3161, "sha256": "ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670" }, "merges.txt": { "bytes": 3353259, "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d" }, "model.safetensors-00001-of-00002.safetensors": { "bytes": 5329398688, "sha256": "26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61" }, "model.safetensors-00002-of-00002.safetensors": { "bytes": 3990429408, "sha256": "cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188" }, "model.safetensors.index.json": { "bytes": 76196, "sha256": "cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d" }, "preprocessor_config.json": { "bytes": 390, "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" }, "tokenizer.json": { "bytes": 12807982, "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" }, "tokenizer_config.json": { "bytes": 16710, "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8" }, "video_preprocessor_config.json": { "bytes": 385, "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13" }, "vocab.json": { "bytes": 6722759, "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" } } }, "inference": { "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 }, "alfworld": { "contextTokens": 32768, "nativeTools": false, "maxCalls": 64, "maxTokens": 64, "seed": 17, "provenance": "unchanged complete revision2 panel, verified reuse", "max_observed_prompt_tokens": 4673 } } }, "Qwen--Qwen3.5-9B": { "label": "Qwen3.5-9B", "panels": { "alfworld": { "status": "COMPLETE", "expected": 134, "completed": 134, "unscored": 0, "primary": 0.2462686567164179, "primary_unit": "task_success_fraction", "success_count": 33, "status_counts": { "COMPLETE": 134 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." }, "bfcl": { "status": "COMPLETE", "expected": 200, "completed": 200, "unscored": 0, "primary": 0.605, "primary_unit": "task_success_fraction", "success_count": 121, "status_counts": { "COMPLETE": 200 }, "measurement_protocol": "orbit-agentic-interface-v3.1-qwen-prelude", "evaluation_date": "2026-11-05", "historical_v3_success_count": 91 }, "scienceworld": { "status": "COMPLETE", "expected": 90, "completed": 90, "unscored": 0, "primary": -24.3, "primary_unit": "native_score_points", "success_count": 8, "status_counts": { "COMPLETE": 90 }, "provenance": "Retained complete panel from historical agentic-v3.json; not rerun in the correction." } }, "identity": { "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "files": { "LICENSE": { "bytes": 11544, "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a" }, "README.md": { "bytes": 77643, "sha256": "c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0" }, "chat_template.jinja": { "bytes": 7756, "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" }, "config.json": { "bytes": 3126, "sha256": "d0883072e01861ed0b2d47be3c16c36a8e81c224c7ffaa310c6558fb3f932b05" }, "merges.txt": { "bytes": 3353259, "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d" }, "model.safetensors-00001-of-00004.safetensors": { "bytes": 5276436216, "sha256": "db6f444b43d318c92f360a13a25561a6a65b10c0631b8ed305a426dbaa6c380e" }, "model.safetensors-00002-of-00004.safetensors": { "bytes": 5335161512, "sha256": "31c7d7e2dd5d207840b31cc59083c8f4c4718959149e0358c0364052bb9a0330" }, "model.safetensors-00003-of-00004.safetensors": { "bytes": 5368717440, "sha256": "7ec36ba3a4176a44c3c0876ad80c56a2f70c84bf008d82e9501df642f17dadec" }, "model.safetensors-00004-of-00004.safetensors": { "bytes": 3325995712, "sha256": "b62b0c4cd7e44edee103ee8f4fe225f246d5e768e07bfd5f25b63a8aa1fdd0c6" }, "model.safetensors.index.json": { "bytes": 79657, "sha256": "26d3539b516be613f39563617cb9d33b3f83d401298125be392c80cefb8f7fe5" }, "preprocessor_config.json": { "bytes": 390, "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" }, "tokenizer.json": { "bytes": 12807982, "sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42" }, "tokenizer_config.json": { "bytes": 16710, "sha256": "316230d6a809701f4db5ea8f8fc862bc3a6f3229c937c174e674ff3ca0a64ac8" }, "video_preprocessor_config.json": { "bytes": 385, "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13" }, "vocab.json": { "bytes": 6722759, "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" } } }, "inference": { "scienceworld": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 64, "seed": 17 }, "bfcl": { "contextTokens": 32768, "nativeTools": true, "maxCalls": 64, "maxTokens": 512, "seed": 17 }, "alfworld": { "contextTokens": 32768, "nativeTools": false, "maxCalls": 64, "maxTokens": 64, "seed": 17, "provenance": "unchanged complete revision2 panel, verified reuse", "max_observed_prompt_tokens": 6040 } } } }, "protocol": { "host": "External and earlier panels use the OhMyPi-based dispatcher. The latest Orbit ALF/Science controls use the frozen Python task/environment adapters with the direct Orbit owner and a fixed synchronous schedule; the BFCL control uses its retained reviewed dispatcher. Host implementation and scheduling are not identical across all cells.", "panels": { "alfworld": "134 official seen-regression task identities", "bfcl": "200 official multi_turn_base cases only", "scienceworld": "30 task classes, 3 fixed official test variations per class" }, "primary_metrics": { "alfworld": "complete task success fraction", "bfcl": "official multi-turn audit success fraction", "scienceworld": "mean native final score, range -100..100" }, "decode": "greedy; thinking disabled", "reply_limit": 64, "output_token_limits": { "alfworld": 64, "bfcl": 512, "scienceworld": 64 }, "orbit": "Saved trained Phi fixed; current observation plus eight-slot authoritative M; deployed RTMA rule, no host transcript.", "external": "Saved chat/tool templates where available; MiniCPM5-1B uses the MiniCPM5-1B-SFT template override; ordinary conversation history, no added memory module.", "feedback": "Tools rendered in model templates, raw textual calls decoded by bounded adapters, tool results returned as ordinary user observations; tool_choice=none in external BFCL requests. Not the full native tool-role message protocol.", "same_information_boundary": "Public instructions, tools and returned observations only; no hidden valid-action list, future questions or test answers.", "runtime": { "external": "vLLM0.29.0 / torch2.13 cu130 / transformers5.17", "orbit": "torch2.11.0+cu128 / transformers5.12.1" }, "reuse": "All panels have been inspected previously. ALF134 is a seen regression panel. Latest complete Orbit runs are used in full; all3392 external rows are unchanged from their reviewed sources. No per-task best-of selection.", "scope": "Finite whole-system regression comparison; not unrestricted official leaderboards or a causal architecture ablation.", "MiniCPM5_1B_template_override_sha256": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c", "seeds": { "generation": 17, "declared_task_seed": 20261017, "scienceworld_environment_seed_applied": false, "scienceworld_scope": "Host task-seed argument was not forwarded. Official1.2.3 doLoad initializes Scala Random and EnvironmentMaker with variationIdx. Fixed task/variation identities do not prove exact replay of every historical JVM state or collection ordering." }, "context_tokens": { "Orbit": { "alfworld": 4096, "scienceworld": 8192, "bfcl": 16384 }, "openbmb--MiniCPM5-1B-SFT": { "scienceworld": 32768, "bfcl": 32768, "alfworld": 32768 }, "openbmb--MiniCPM5-1B": { "alfworld": 32768, "scienceworld": 32768, "bfcl": 32768 }, "Qwen--Qwen2.5-1.5B-Instruct": { "alfworld": 32768, "scienceworld": 32768, "bfcl": 32768 }, "HuggingFaceTB--SmolLM2-1.7B-Instruct": { "alfworld": 8192, "scienceworld": 8192, "bfcl": 8192 }, "LiquidAI--LFM2.5-1.2B-Instruct": { "alfworld": 32768, "scienceworld": 32768, "bfcl": 32768 }, "HuggingFaceTB--SmolLM3-3B": { "alfworld": 32768, "scienceworld": 32768, "bfcl": 32768 }, "Qwen--Qwen3.5-4B": { "scienceworld": 32768, "bfcl": 32768, "alfworld": 32768 }, "Qwen--Qwen3.5-9B": { "scienceworld": 32768, "bfcl": 32768, "alfworld": 32768 } }, "BFCL_correction": { "model_ids": [ "Qwen--Qwen3.5-4B", "Qwen--Qwen3.5-9B" ], "tasks_per_model": 200, "decoder_sha256": "4e5d917ad1780b96dc858989431725930257d1c63f46baf3671a5dd539ecb495", "plan_sha256": "6c30bd4e3e52cf22bc8e26bca944621a042c2ef8b208ee951d89def6a327f0cf", "change": "Permit plain prose before a complete native Qwen function-call block; preserve names, arguments, complete suffix validation and originally valid literal calls.", "unchanged": [ "weights", "prompts", "task IDs", "scorer", "reply/token limits", "generation seed" ], "hardware": "Local RTX4090; historical Qwen panels ran on remote RTX PRO6000.", "score_difference_interpretation": "Descriptive protocol revision, not a paired causal estimate of parser effect." }, "recovery": "Saved replies and submitted prefixes retained; nine SmolLM3 BFCL tasks explicitly continued after an unreturned request. No double-scoring; unreturned-request physical compute cost is unknown. Orbit timeout/continuation traces independently reconciled to final checkpoints.", "BFCL_turn_termination": "[] or a parsing rejection ends the current user turn; the 21st nonempty reply in a turn triggers termination, alongside the64-reply task limit. Counts replies, not individual functions.", "BFCL_schema_diagnostic": { "arms": { "gorilla": { "completed": 200, "successes": 0, "calls": 735, "function_calls": 1, "input_tokens": 4228908, "output_tokens": 8244, "expected": 200, "primary": 0.0 }, "openapi-types": { "completed": 200, "successes": 0, "calls": 734, "function_calls": 0, "input_tokens": 4223973, "output_tokens": 8481, "expected": 200, "primary": 0.0 } }, "paired_task_outcomes": { "both_fail": 200 }, "plan_sha256": "5045d3d4e14cd0d3da469848148d6ca132de00d6594c263b555f99a709702e97", "change": "Only parameter-schema dict/float types converted to object/number; published weights, roles, task IDs, seeds, scorer, parser and budgets fixed.", "interpretation": "Neither arm completed a task successfully. This does not support this type-only conversion as a sufficient repair; it does not identify every cause of failed tool use or prove all tool interfaces equivalent.", "table_policy": "Report the raw-schema control; retain the normalized arm as a separate diagnostic. No candidate was installed." }, "Orbit_ALF_protocol_repair": { "reason": "Use the ACTION_INSTRUCTION already defined by the released ALFWorldFormatter and its task-state factory; remove the evaluation-specific instruction rewrite, without tuning new text or selecting tasks.", "original_instruction": "Complete the household task using the current observation and accessible memory. Return exactly one environment command, with no explanation. Public command forms: look; inventory; go to (receptacle); open (receptacle); close (receptacle); take (object) from (receptacle); move (object) to (receptacle); put (object) into (receptacle object); put (receptacle object) in (receptacle); examine (object or receptacle); use (object); heat (object) with (receptacle); clean (object) with (receptacle); cool (object) with (receptacle); slice (object) with (sharp object); help. Use names and numbers from observations.\n", "original_instruction_sha256": "bfd95b4465189c67f1ea458bbea253ca68a9baa52bb20834412364d691471c3b", "alternative_instruction": "Complete the household task. Return exactly one executable ALFWorld command, without explanation, quotes, JSON, Markdown, or a numbered list.\nUse complete object and receptacle names with their numbers, exactly as observed. Parenthesized words in command templates are placeholders: replace the whole placeholder; never print parentheses or the placeholder word.\nCommands include: look; inventory; go to RECEPTACLE; open RECEPTACLE; close RECEPTACLE; take OBJECT from RECEPTACLE; move OBJECT to RECEPTACLE; examine OBJECT; use OBJECT; heat OBJECT with RECEPTACLE; cool OBJECT with RECEPTACLE; clean OBJECT with RECEPTACLE.\nFor example, if you see a countertop 3, the command is go to countertop 3. If you are at a shelf 2 and see a book 4 on it, take book 4 from shelf 2 picks it up. These are syntax examples, not facts about your current room.\nGo to a receptacle before interacting with its contents. Open a closed container before taking from it. Hold an object before moving, heating, cooling, or cleaning it. To examine a held object under a lamp, go to the lamp's location and use the lamp by its observed name. Read the returned observation before choosing the next command. A response of Nothing happens means the last command had no effect; use the observation to choose a different valid action.\n", "alternative_successes": 35, "default_successes": 56, "tasks_per_arm": 134, "full_pairs": "agentic-instruction-pairs.json", "scope": "One fixed reused panel and seed; conditional total effect, not unique historical cause or unseen generalization.", "external_models": "External panels retain their original revised text and model-specific templates. This is not an instruction-matched external comparison.", "implementation": "The released runtime already defines this instruction. Model/runtime weights and deployment.json are unchanged; the benchmark instruction override is the repaired choice." }, "Orbit_panel_runs": { "alfworld": { "source": "01c3202aad900ece8ae4e5b29e3cfa055598180262d6ffb7ec6ab2840390d8ff", "arm": "original", "scheduling": "groups8, last6; fixed slots, original thenv3; no refill", "primary_successes": 56, "historical_v3_successes": 36 }, "bfcl": { "source": "b85b7a04a0bfaec4b85ce898ef3e86d223094da3c435b274e4385e8adea850e3", "arm": "calibrated", "scheduling": "published-reader arm first, seven shards with up to32 environment episodes; physical generation batch1", "primary_successes": 0 }, "scienceworld": { "source": "79efb1fb6d0f16d3459a65d690bd00c9eb6c56e3f89756d6f9411708717c1fd5", "arm": "v3", "scheduling": "fixed8-wide groups; original thenv3", "mean_score": -28.27777777777778, "successes": 1, "historical_v3_mean_score": -28.27777777777778, "historical_v3_successes": 0, "historical_note": "Seven individual scores changed despite the unchanged mean. New and historical runs are separately retained." } } }, "performance": { "definitions": "Job wall seconds sum per-device active host intervals across original and continuation segments; exclude model startup/cleanup and 402 reused ALF historical cost. Parallel workers have additive time, not total end-to-end wall time. Returned work excludes unreturned responses, whose elapsed cost remains included. Saved replies count once.", "UTL": "Whole-card samples during the matching job/device intervals; not MFU.", "MFU": "Not measured.", "hardware_note": "Historical v3: Orbit ALF134 and BFCL160 on local RTX4090; BFCL40 and Science90 on remote RTX PRO6000; external newly evaluated v3 panels on RTX PRO6000. Current Qwen3.5 BFCL correction: local RTX4090. Mixed hardware/runtime is not a controlled speed benchmark. The complete Orbit BFCL control and normalized-schema diagnostic now both ran locally on RTX4090 with batch1/head256/cache; historical Orbit BFCL hardware is no longer the provenance of the current table cell.", "historical_v3_rows": [ { "model": "HuggingFaceTB--SmolLM2-1.7B-Instruct", "panel": "alfworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 259.0365707874298, "returned_work": { "replies": 8576, "input_tokens": 18604396, "output_tokens": 190888, "length_finishes": 1792 }, "returned_output_tokens_per_worker_second": 736.9152526214, "samples": 51, "mean_UTL": 57.470588235294116, "mean_W": 249.1407843137255, "MFU": null }, { "model": "HuggingFaceTB--SmolLM2-1.7B-Instruct", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 181.028724193573, "returned_work": { "replies": 1293, "input_tokens": 7497335, "output_tokens": 185818, "length_finishes": 183 }, "returned_output_tokens_per_worker_second": 1026.455888852787, "samples": 36, "mean_UTL": 80.5, "mean_W": 374.84277777777777, "MFU": null }, { "model": "HuggingFaceTB--SmolLM2-1.7B-Instruct", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 282.05963945388794, "returned_work": { "replies": 5760, "input_tokens": 17446306, "output_tokens": 92370, "length_finishes": 5 }, "returned_output_tokens_per_worker_second": 327.4839327556503, "samples": 55, "mean_UTL": 62.018181818181816, "mean_W": 236.7987272727273, "MFU": null }, { "model": "HuggingFaceTB--SmolLM3-3B", "panel": "alfworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 242.03773832321167, "returned_work": { "replies": 8576, "input_tokens": 16628138, "output_tokens": 123997, "length_finishes": 300 }, "returned_output_tokens_per_worker_second": 512.3044069863901, "samples": 47, "mean_UTL": 47.1063829787234, "mean_W": 216.55489361702126, "MFU": null }, { "model": "HuggingFaceTB--SmolLM3-3B", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 190.18724155426025, "returned_work": { "replies": 2325, "input_tokens": 15063016, "output_tokens": 111950, "length_finishes": 13, "unreturned_model_calls": 9 }, "returned_output_tokens_per_worker_second": 588.6304416905945, "samples": 38, "mean_UTL": 73.5, "mean_W": 295.78473684210525, "MFU": null }, { "model": "HuggingFaceTB--SmolLM3-3B", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 223.03466820716858, "returned_work": { "replies": 4189, "input_tokens": 12390270, "output_tokens": 39946, "length_finishes": 103 }, "returned_output_tokens_per_worker_second": 179.10220111115484, "samples": 44, "mean_UTL": 53.13636363636363, "mean_W": 211.91886363636365, "MFU": null }, { "model": "LiquidAI--LFM2.5-1.2B-Instruct", "panel": "alfworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 281.0382556915283, "returned_work": { "replies": 8576, "input_tokens": 15082957, "output_tokens": 63816, "length_finishes": 70 }, "returned_output_tokens_per_worker_second": 227.07228894149333, "samples": 55, "mean_UTL": 15.127272727272727, "mean_W": 126.92236363636363, "MFU": null }, { "model": "LiquidAI--LFM2.5-1.2B-Instruct", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 77.01419711112976, "returned_work": { "replies": 1215, "input_tokens": 8600438, "output_tokens": 52585, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 682.79618528154, "samples": 14, "mean_UTL": 43.857142857142854, "mean_W": 218.01642857142858, "MFU": null }, { "model": "LiquidAI--LFM2.5-1.2B-Instruct", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 297.06167936325073, "returned_work": { "replies": 5699, "input_tokens": 17273159, "output_tokens": 82705, "length_finishes": 130 }, "returned_output_tokens_per_worker_second": 278.4101947355764, "samples": 58, "mean_UTL": 31.551724137931036, "mean_W": 165.83982758620692, "MFU": null }, { "model": "Orbit", "panel": "alfworld", "hardware": "local-4090", "job_wall_seconds": 937.1650896072388, "returned_work": { "replies": 6523, "input_tokens": 3388281, "output_tokens": 45529, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 48.58162185606058, "samples": 187, "mean_UTL": 62.45454545454545, "mean_W": 317.73561497326205, "MFU": null }, { "model": "Orbit", "panel": "bfcl", "hardware": "local-4090", "job_wall_seconds": 4504.695177793503, "returned_work": { "replies": 567, "input_tokens": 3153962, "output_tokens": 5739, "length_finishes": 1, "saved_reply_reuses": 20, "unreturned_model_calls": 4 }, "returned_output_tokens_per_worker_second": 1.27400407208265, "samples": 898, "mean_UTL": 98.70601336302896, "mean_W": 389.1228173719377, "MFU": null }, { "model": "Orbit", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 1362.1481802463531, "returned_work": { "replies": 168, "input_tokens": 1074946, "output_tokens": 2673, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 1.9623415710297913, "samples": 267, "mean_UTL": 96.12359550561797, "mean_W": 419.8287265917603, "MFU": null }, { "model": "Orbit", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 799.1107132434845, "returned_work": { "replies": 4004, "input_tokens": 2882017, "output_tokens": 21507, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 26.913667460051858, "samples": 157, "mean_UTL": 44.33757961783439, "mean_W": 247.38681528662417, "MFU": null }, { "model": "Qwen--Qwen2.5-1.5B-Instruct", "panel": "alfworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 225.05171990394592, "returned_work": { "replies": 8576, "input_tokens": 18472352, "output_tokens": 215577, "length_finishes": 4 }, "returned_output_tokens_per_worker_second": 957.8998111723394, "samples": 44, "mean_UTL": 47.70454545454545, "mean_W": 191.15840909090912, "MFU": null }, { "model": "Qwen--Qwen2.5-1.5B-Instruct", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 174.0313355922699, "returned_work": { "replies": 4320, "input_tokens": 29030655, "output_tokens": 157181, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 903.1764277684579, "samples": 34, "mean_UTL": 61.294117647058826, "mean_W": 236.54617647058822, "MFU": null }, { "model": "Qwen--Qwen2.5-1.5B-Instruct", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 184.02459573745728, "returned_work": { "replies": 4124, "input_tokens": 12236946, "output_tokens": 24886, "length_finishes": 34 }, "returned_output_tokens_per_worker_second": 135.23192321261314, "samples": 36, "mean_UTL": 33.75, "mean_W": 158.09333333333333, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 317.0433175563812, "returned_work": { "replies": 3403, "input_tokens": 24051404, "output_tokens": 172457, "length_finishes": 0, "unreturned_model_calls": 1 }, "returned_output_tokens_per_worker_second": 543.9540606918208, "samples": 62, "mean_UTL": 71.20967741935483, "mean_W": 307.3567741935484, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 281.0388207435608, "returned_work": { "replies": 4393, "input_tokens": 17336798, "output_tokens": 21089, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 75.03945520481335, "samples": 55, "mean_UTL": 42.89090909090909, "mean_W": 214.77236363636362, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 214.0323269367218, "returned_work": { "replies": 2071, "input_tokens": 12747517, "output_tokens": 104506, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 488.2720357981109, "samples": 41, "mean_UTL": 72.63414634146342, "mean_W": 356.6329268292683, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 229.05741500854492, "returned_work": { "replies": 3036, "input_tokens": 8960212, "output_tokens": 19179, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 83.73009884567384, "samples": 45, "mean_UTL": 56.24444444444445, "mean_W": 299.9644444444445, "MFU": null }, { "model": "openbmb--MiniCPM5-1B", "panel": "alfworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 213.0453381538391, "returned_work": { "replies": 8576, "input_tokens": 13913501, "output_tokens": 62761, "length_finishes": 128 }, "returned_output_tokens_per_worker_second": 294.5898771776013, "samples": 42, "mean_UTL": 21.023809523809526, "mean_W": 135.07190476190476, "MFU": null }, { "model": "openbmb--MiniCPM5-1B", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 143.0234076976776, "returned_work": { "replies": 4196, "input_tokens": 28075473, "output_tokens": 110918, "length_finishes": 9 }, "returned_output_tokens_per_worker_second": 775.5234040742343, "samples": 28, "mean_UTL": 56.142857142857146, "mean_W": 210.02535714285713, "MFU": null }, { "model": "openbmb--MiniCPM5-1B", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 217.0302734375, "returned_work": { "replies": 5209, "input_tokens": 14038149, "output_tokens": 35960, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 165.6911703166411, "samples": 43, "mean_UTL": 25.558139534883722, "mean_W": 151.32302325581395, "MFU": null }, { "model": "openbmb--MiniCPM5-1B-SFT", "panel": "bfcl", "hardware": "rtx-pro6000-2", "job_wall_seconds": 134.02470660209656, "returned_work": { "replies": 4119, "input_tokens": 27551807, "output_tokens": 102810, "length_finishes": 1 }, "returned_output_tokens_per_worker_second": 767.0973703768715, "samples": 26, "mean_UTL": 54.80769230769231, "mean_W": 209.4953846153846, "MFU": null }, { "model": "openbmb--MiniCPM5-1B-SFT", "panel": "scienceworld", "hardware": "rtx-pro6000-2", "job_wall_seconds": 204.0337872505188, "returned_work": { "replies": 4693, "input_tokens": 12643305, "output_tokens": 39153, "length_finishes": 0 }, "returned_output_tokens_per_worker_second": 191.8946882651684, "samples": 40, "mean_UTL": 28.6, "mean_W": 154.87824999999998, "MFU": null } ], "correction_rows": [ { "model": "Qwen--Qwen3.5-9B", "seconds": 54.01022696495056, "returned_work": { "replies": 424, "input_tokens": 2473167, "output_tokens": 15811, "length_finishes": 0 }, "returned_tokens_per_job_second": 292.7408546211147, "mean_UTL": 100.0, "mean_W": 397.965, "samples": 10, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 50.00976538658142, "returned_work": { "replies": 381, "input_tokens": 2562091, "output_tokens": 16803, "length_finishes": 0 }, "returned_tokens_per_job_second": 335.99437770025145, "mean_UTL": 90.0, "mean_W": 372.612, "samples": 10, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 41.00820732116699, "returned_work": { "replies": 404, "input_tokens": 2717661, "output_tokens": 18745, "length_finishes": 0 }, "returned_tokens_per_job_second": 457.10361960456856, "mean_UTL": 77.77777777777777, "mean_W": 350.5855555555556, "samples": 9, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 37.007455587387085, "returned_work": { "replies": 334, "input_tokens": 1776928, "output_tokens": 16748, "length_finishes": 0 }, "returned_tokens_per_job_second": 452.5574572521562, "mean_UTL": 100.0, "mean_W": 411.06, "samples": 7, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 45.008026123046875, "returned_work": { "replies": 302, "input_tokens": 1707315, "output_tokens": 17328, "length_finishes": 0 }, "returned_tokens_per_job_second": 384.99799908192375, "mean_UTL": 88.88888888888889, "mean_W": 384.6388888888889, "samples": 9, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 57.01064419746399, "returned_work": { "replies": 349, "input_tokens": 2479362, "output_tokens": 28498, "length_finishes": 0 }, "returned_tokens_per_job_second": 499.87156611129257, "mean_UTL": 100.0, "mean_W": 400.16636363636366, "samples": 11, "MFU": null }, { "model": "Qwen--Qwen3.5-9B", "seconds": 35.00768256187439, "returned_work": { "replies": 83, "input_tokens": 562334, "output_tokens": 7417, "length_finishes": 0 }, "returned_tokens_per_job_second": 211.86778036194798, "mean_UTL": 94.57142857142857, "mean_W": 337.7657142857143, "samples": 7, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 38.0064799785614, "returned_work": { "replies": 534, "input_tokens": 3220502, "output_tokens": 19397, "length_finishes": 0 }, "returned_tokens_per_job_second": 510.36033884067695, "mean_UTL": 81.875, "mean_W": 357.9125, "samples": 8, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 34.0051965713501, "returned_work": { "replies": 493, "input_tokens": 3453983, "output_tokens": 19236, "length_finishes": 0 }, "returned_tokens_per_job_second": 565.6782474301774, "mean_UTL": 85.71428571428571, "mean_W": 380.8442857142857, "samples": 7, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 52.008928537368774, "returned_work": { "replies": 605, "input_tokens": 4492497, "output_tokens": 27984, "length_finishes": 0 }, "returned_tokens_per_job_second": 538.0614595798355, "mean_UTL": 100.0, "mean_W": 393.69, "samples": 10, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 45.008227825164795, "returned_work": { "replies": 458, "input_tokens": 2725874, "output_tokens": 26274, "length_finishes": 0 }, "returned_tokens_per_job_second": 583.7599316743105, "mean_UTL": 100.0, "mean_W": 399.8077777777778, "samples": 9, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 47.0073606967926, "returned_work": { "replies": 480, "input_tokens": 3140364, "output_tokens": 28119, "length_finishes": 0 }, "returned_tokens_per_job_second": 598.1829139775255, "mean_UTL": 100.0, "mean_W": 398.57, "samples": 9, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 57.00828671455383, "returned_work": { "replies": 596, "input_tokens": 4792821, "output_tokens": 38110, "length_finishes": 0 }, "returned_tokens_per_job_second": 668.499304159421, "mean_UTL": 83.33333333333333, "mean_W": 361.60833333333335, "samples": 12, "MFU": null }, { "model": "Qwen--Qwen3.5-4B", "seconds": 46.00865983963013, "returned_work": { "replies": 161, "input_tokens": 1305314, "output_tokens": 12610, "length_finishes": 0 }, "returned_tokens_per_job_second": 274.07883741786844, "mean_UTL": 88.88888888888889, "mean_W": 336.5588888888889, "samples": 9, "MFU": null } ], "correction_cost_scope": "Job intervals include the failed first runner interval but exclude server warmup/cleanup. Original failed segment preserved. Zero rental/API charges; local compute was used. MFU not measured.", "Orbit_schema_pair_rows": [ { "arm": "gorilla", "wall_seconds": 545.0869147777557, "returned_work": { "returned_calls": 106, "input_tokens": 555495, "output_tokens": 919, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.685969659305979, "mean_GPU_UTL_percent": 98.68518518518519, "mean_GPU_power_W": 385.2319444444444, "GPU_samples": 108, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 496.0800075531006, "returned_work": { "returned_calls": 104, "input_tokens": 648013, "output_tokens": 649, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.308256712866081, "mean_GPU_UTL_percent": 96.84848484848484, "mean_GPU_power_W": 384.8429292929293, "GPU_samples": 99, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 666.1066203117371, "returned_work": { "returned_calls": 105, "input_tokens": 668554, "output_tokens": 796, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.1950038863560206, "mean_GPU_UTL_percent": 97.39097744360902, "mean_GPU_power_W": 385.1439097744361, "GPU_samples": 133, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 656.1069223880768, "returned_work": { "returned_calls": 131, "input_tokens": 643616, "output_tokens": 1385, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 2.1109364232264487, "mean_GPU_UTL_percent": 98.17557251908397, "mean_GPU_power_W": 388.6147328244275, "GPU_samples": 131, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 1296.2045104503632, "returned_work": { "returned_calls": 121, "input_tokens": 638284, "output_tokens": 1990, "length_finishes": 1 }, "returned_output_tokens_per_job_second": 1.535251562508897, "mean_GPU_UTL_percent": 99.06589147286822, "mean_GPU_power_W": 389.8410465116279, "GPU_samples": 258, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 1469.1985957622528, "returned_work": { "returned_calls": 135, "input_tokens": 874233, "output_tokens": 1892, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.2877768910597063, "mean_GPU_UTL_percent": 99.18150684931507, "mean_GPU_power_W": 388.42, "GPU_samples": 292, "MFU": null, "CPU_seconds": null }, { "arm": "gorilla", "wall_seconds": 422.0683341026306, "returned_work": { "returned_calls": 33, "input_tokens": 200713, "output_tokens": 613, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.4523714537915138, "mean_GPU_UTL_percent": 97.07058823529412, "mean_GPU_power_W": 381.9274117647059, "GPU_samples": 85, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 688.0890183448792, "returned_work": { "returned_calls": 106, "input_tokens": 555495, "output_tokens": 1428, "length_finishes": 1 }, "returned_output_tokens_per_job_second": 2.075312876573577, "mean_GPU_UTL_percent": 98.61313868613139, "mean_GPU_power_W": 388.10335766423356, "GPU_samples": 137, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 475.07875299453735, "returned_work": { "returned_calls": 104, "input_tokens": 648013, "output_tokens": 632, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.3303057567115972, "mean_GPU_UTL_percent": 98.21052631578948, "mean_GPU_power_W": 383.17726315789474, "GPU_samples": 95, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 698.1144533157349, "returned_work": { "returned_calls": 105, "input_tokens": 668554, "output_tokens": 809, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.1588357699193992, "mean_GPU_UTL_percent": 97.79856115107914, "mean_GPU_power_W": 386.07755395683455, "GPU_samples": 139, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 618.0961883068085, "returned_work": { "returned_calls": 131, "input_tokens": 643616, "output_tokens": 1309, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 2.1177933544386187, "mean_GPU_UTL_percent": 98.21951219512195, "mean_GPU_power_W": 389.5132520325203, "GPU_samples": 123, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 924.1439681053162, "returned_work": { "returned_calls": 121, "input_tokens": 638284, "output_tokens": 1571, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.6999515813762986, "mean_GPU_UTL_percent": 99.22826086956522, "mean_GPU_power_W": 389.6759239130435, "GPU_samples": 184, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 1571.249398469925, "returned_work": { "returned_calls": 134, "input_tokens": 869298, "output_tokens": 2018, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.2843282562049785, "mean_GPU_UTL_percent": 99.33226837060703, "mean_GPU_power_W": 388.8350479233227, "GPU_samples": 313, "MFU": null, "CPU_seconds": null }, { "arm": "openapi-types", "wall_seconds": 505.08206009864807, "returned_work": { "returned_calls": 33, "input_tokens": 200713, "output_tokens": 714, "length_finishes": 0 }, "returned_output_tokens_per_job_second": 1.4136316777130196, "mean_GPU_UTL_percent": 98.01980198019803, "mean_GPU_power_W": 386.79940594059406, "GPU_samples": 101, "MFU": null, "CPU_seconds": null } ], "Orbit_schema_pair_resource_observation": { "reservation_start_unix": 1793857185.8012683, "terminal_file_mtime_unix": 1793868501.913097, "reservation_wall_seconds_by_file_mtime": 11316.111828565598, "hard_stop_unix": 1793871585.8012683, "terminal_file_written_after_hard_stop": false, "CPU_seconds_total": null, "GPU_reserved_hours_upper_charged": 3.143364396823777, "rental_CNY": 0, "paid_API_USD": 0, "limits": [ "Reservation includes preparation/review/load/save/cleanup and is not GPU kernel time; endpoint uses preserved file mtime", "Original launcher did not record process CPU totals; unknown, not zero. Collection CPU is separate", "Per-job observed returned tokens include partial tasks; unreturned requests may have additional work", "GPU telemetry is whole-card sampling, not MFU or uniquely attributable if shared" ] }, "current_Orbit_complete_pair_costs": { "ALF_instruction_pair": { "resources": { "wall_seconds": 1657.1963202879997, "GPU_allocation_seconds": 1657.1963202879997, "total_CPU_seconds": 2733.054583558, "total_output_bytes": 151777040, "CPU_accounting_complete": true, "hard_limit_exceeded": [] }, "performance": { "samples_total": 329, "samples_parsed": 329, "mean_UTL_percent": 55.18844984802432, "mean_power_W": 284.32039513677813, "max_memory_MiB": 24744.0, "output_tokens_per_GPU_allocation_second": 49.52098854874235, "MFU": "unknown", "sample_scope": "whole card, point samples, not per-kernel activity integration" }, "cost_scope": "Both268 diagnostic endpoints; not134-only model-table cost" }, "Science_instruction_pair": { "resources": { "wall_seconds": 1983.108780568, "GPU_allocation_seconds": 1983.108780568, "total_CPU_seconds": 4030.990388231, "total_output_bytes": 95510422, "CPU_accounting_complete": true, "hard_limit_exceeded": [] }, "performance": { "GPU_samples": 394, "usable_GPU_samples": 394, "sampled_mean_UTL_percent": 60.93908629441624, "sampled_mean_power_W": 296.9608883248731, "output_tokens_per_allocation_second": 26.21137100966894, "MFU": null, "scope": "Whole allocation includes verification/load/init/generation/feedback/M/cleanup. Card samples are not per-task MFU; completed-task tokens are a lower bound if INCOMPLETE." }, "cost_scope": "Both180 diagnostic endpoints" }, "BFCL_calibration_pair": { "GPU_allocation_seconds": 14162.458663880941, "CPU_seconds": 14259.603018855001, "cost_scope": "Both400 diagnostic endpoints; not the200 published-reader endpoints alone" } } }, "evidence_sha256": { "private_summary": "f8494a0f820daeeaaefc3694797c810b4c8ca46fde3401a94a45504a0b3bde8a", "task_evidence": "b58cbdb37d2b95476aeab69bf37b57f296bc845892ce0d792da78a931406b955", "collector": "386f3680d65e9dd5c0db13bc32855097cf69d141bafe459b20e3cd1a463b401a", "ALF_reuse_manifest": "b89c81e9865016047b14500572429f1dbb19eb38029e4892f118b0070b543936", "historical_public_v3": "68491e44beede30c3429135437383acd7bfdc4ccfff4e0eb8144ce1cb0641dac", "BFCL_correction_summary": "1c81962f06af035c032c31bf8807bf7e64c527742efc5300fac3dff3ce29feb2", "BFCL_correction_task_index": "3ee62a66238ef9e7b4732eb9e56326f2e361d09018ccf006de6548ce3c822f4c", "BFCL_schema_pair_summary": "29b6c23751c7bdfb95e9fdabdce07ff951c7cb6e73ed693013f4dcc8825e1be6", "private_v4_candidate": "f8b7fd1b924402e79826a75d14e472924886e5d3362c43f734f1ebd7d53c14c7", "bfcl-schema-result-review.json": "c420fcab372ede427bce7e33dd3ee5cdd894720197fdb22af72e4abf90f840a9", "alf-science-measurement-review.json": "f7f911dcd9e0c11487d8921a659ab6d9654a66e1ac9533e428c4e3c7a505f0a5", "ALF_instruction_collection": "0b0cec43606f1f4f29f86f95383c5d9e313703ad57c63c2e36fea74a334f05b3", "ALF_instruction_result_review": "a5063f53b908be0c10654194e855eb7d60e7038a45e0f76da3cf92be5a993a9e", "Science_instruction_result": "457f8b048d2dcf20eb44cade6f961f1aa76b4e61b41925c99430673529abde54", "Science_instruction_result_review": "46bc8f57c0e52a1f1f81b9e1778634e72cd15236a013cc04b31a90502577ac23", "BFCL_calibration_result_review": "a24520db68013d33b8732f4a9de79d5ae8ec357dfc9bffdc67f425e43538d138" }, "access": "Aggregate results, identities and hashes are public in this package; raw task traces and detailed execution evidence remain in the private research archive.", "warnings": [ "ALF134 is a seen regression panel; BFCL200 and Science90 are repeated after a declared interface revision, not new held-out tests.", "Native tool-template serialization where supported, strict public adapters and bounded64 calls; not a reproduction of unrestricted official leaderboards.", "Three exact complete ALF panels may be reused with explicit provenance; no best-of selection.", "ScienceWorld negative scores are retained. Missing positions are not zero-filled.", "UTL and power are whole-card samples; MFU is not measured. This table evaluates the packaged deployed manager.", "MiniCPM5-1B template override, context/precision/runtime/hardware differences prevent a strictly matched model ranking.", "Orbit historical ALF54/35/36 used different protocols. Available evidence does not identify a unique cause of those differences.", "No accepted/installed RSI successor is represented by this table.", "ScienceWorld uses variation-based initialization in the pinned engine; the host task seed was not forwarded. Exact historical physical replay is not established.", "Orbit ALF uses its packaged original instruction; external ALF panels retain their revised instruction. These system configurations do not constitute a prompt-matched causal ranking.", "The higher ScienceWorld mean under the original instruction is disclosed as a diagnostic; it had0/90 successes and is not selected for the main table." ], "evaluation_date_scope": "Orbit panels use complete fixed-published-weight runs ending2026-11-06; external models retain their full reviewed panels including Qwen BFCL correction. No new science is performed by this aggregation.", "historical_results": "agentic-v3.json", "reproduction_access": "Published per-task IDs/variations/metrics, version identities and hashes support score recomputation; exact runtime snapshots and raw traces are retained privately. This package does not provide a public standalone end-to-end benchmark runner.", "task_index": { "file": "results/agentic-tasks-v6.jsonl", "sha256": "c6fab11838721ec53045184b569233d69e162e05bd4aaad314d57a69d716e5ea", "scope": "All3816 complete positions; no missing task or zero fill. All424 Orbit rows use whole complete current runs;3392 external rows retained exactly." } }