{ "format": "arti.reading-qa-benchmark", "version": 1, "complete": true, "measured_on": "2026-10-12", "base": { "id": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca" }, "results": { "squad": { "rows": [ { "method": "base", "label": "base", "seeds": [ 17 ], "trainable_parameters": 0, "em": 55.078125, "em_std": 0, "f1": 62.562165105031, "f1_std": 0, "adapter_kib": 0.0 }, { "method": "torch", "label": "torch", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 55.598958333333336, "em_std": 0.8131507810414581, "f1": 62.430029901984454, "f1_std": 0.5369590312075996, "adapter_kib": 512.359375 }, { "method": "peft", "label": "peft", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 56.770833333333336, "em_std": 1.8460217290049252, "f1": 64.21988995333979, "f1_std": 1.502822247037492, "adapter_kib": 512.546875 }, { "method": "arti", "label": "arti", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 55.598958333333336, "em_std": 0.8131507810414581, "f1": 62.430029901984454, "f1_std": 0.5369590312075996, "adapter_kib": 545.3828125 } ], "sample": { "id": "57291f153f37b31900478044", "context": "Kenya has proven deposits of oil in Turkana and the commercial viability was just discovered. Tullow Oil estimates Kenya's oil reserves to be around 10 billion barrels. Exploration is still continuing to determine if there are more reserves. Kenya currently imports all crude petroleum requirements. Kenya, east Africa's largest economy, has no strategic reserves and relies solely on oil marketers' 21-day oil reserves required under industry regulations. Petroleum accounts for 20% to 25% of the national import bill.", "question": "How many barrels of oil is it estimated Kenya has?", "references": [ "around 10 billion barrels.", "10 billion" ], "predictions": { "base": "10 billion", "torch": "10 billion", "peft": "10 billion", "arti": "10 billion" } }, "eligibility": { "train": { "source_rows": 87599, "eligible_rows": 84688, "overlength_excluded": 2707, "answer_over_32_excluded": 204, "eligible_contexts": 18390 }, "dev": { "source_rows": 10570, "eligible_rows": 10169, "overlength_excluded": 394, "answer_over_32_excluded": 7, "eligible_contexts": 2002 } }, "split_manifest": { "dataset": "squad", "selection": "sha256 context-first then original-ID order; validation contexts removed from train; public dev heldout", "counts": { "train": 512, "validation": 128, "heldout": 256 }, "eligibility": { "train": { "source_rows": 87599, "eligible_rows": 84688, "overlength_excluded": 2707, "answer_over_32_excluded": 204, "eligible_contexts": 18390 }, "dev": { "source_rows": 10570, "eligible_rows": 10169, "overlength_excluded": 394, "answer_over_32_excluded": 7, "eligible_contexts": 2002 } }, "splits": { "train": { "ids": [ "56f8c1a09b226e1400dd0f3e", "56f8c1a09b226e1400dd0f40", "56f8c1a09b226e1400dd0f3f", "56f8c1a09b226e1400dd0f3d", "56bfae97a10cfb1400551236", "56beab283aeaaa14008c91cd", "56bfae97a10cfb1400551237", "56d4d2f12ccc5a1400d8326f", "56d4d2f12ccc5a1400d83270", "56bfae97a10cfb1400551235", "56beab283aeaaa14008c91cc", "56bfae97a10cfb1400551234", "56d4d2f12ccc5a1400d8326c", "56d4d2f12ccc5a1400d8326d", "56e02060231d4119001abf3f", "56e02060231d4119001abf3e", "5706d9820eeca41400aa0e82", "5706d9820eeca41400aa0e83", "5706d9820eeca41400aa0e81", "5706d9820eeca41400aa0e85", "5706d9820eeca41400aa0e84", "570bd9336b8089140040fa8c", "570bd9336b8089140040fa8d", "570bd9336b8089140040fa8a", "570bd9336b8089140040fa8b", "5733e7c4d058e614000b6559", "5733e7c4d058e614000b655b", "5733e7c4d058e614000b6558", "5733e7c4d058e614000b6557", "5733e7c4d058e614000b655a", "57265fb55951b619008f70f3", "57265fb55951b619008f70f2", "57265fb55951b619008f70f1", "57265fb55951b619008f70f0", "57265fb55951b619008f70ef", "570affb86b8089140040f696", "570affb86b8089140040f697", "570affb86b8089140040f698", "570affb86b8089140040f695", "570affb86b8089140040f694", "5729141f6aef051400154a3a", "5729141f6aef051400154a3b", "5729141f6aef051400154a3c", "572713ce708984140094d969", "572713ce708984140094d968", "572713ce708984140094d967", "572713ce708984140094d965", "572713ce708984140094d966", "570be5f1ec8fbc190045bbd0", "570be5f1ec8fbc190045bbd4", "570be5f1ec8fbc190045bbd2", "570be5f1ec8fbc190045bbd3", "570be5f1ec8fbc190045bbd1", "56db7a68e7c41114004b5185", "56db7a68e7c41114004b5186", "56d98906dc89441400fdb503", "56db7a68e7c41114004b5183", "56db7a68e7c41114004b5187", "56d98906dc89441400fdb506", "56d98906dc89441400fdb505", "56d98906dc89441400fdb502", "572fed4e04bcaa1900d76ecf", "572fed4e04bcaa1900d76ed1", "572fed4e04bcaa1900d76ecd", "572fed4e04bcaa1900d76ed0", "572fed4e04bcaa1900d76ece", "570e366b0b85d914000d7d79", "570e366b0b85d914000d7d77", "570e366b0b85d914000d7d7a", "570e366b0b85d914000d7d7b", "570e366b0b85d914000d7d78", "570ad1354103511400d59a38", "570ad1354103511400d59a37", "570ad1354103511400d59a35", "570ad1354103511400d59a36", "570ad1354103511400d59a34", "56f82fdaaef2371900625ec5", "56f82fdaaef2371900625ec7", "56f82fdaaef2371900625ec6", "56f82fdaaef2371900625ec9", "56f82fdaaef2371900625ec8", "5727e7822ca10214002d9925", "5727e7822ca10214002d9928", "5727e7822ca10214002d9927", "5727e7812ca10214002d9924", "5727e7822ca10214002d9926", "56e7825400c9c71400d771ec", "56e7825400c9c71400d771ea", "56e7825400c9c71400d771ed", "56e7825400c9c71400d771eb", "56e7825400c9c71400d771e9", "5730e7c0b54a4f140068ccd1", "5730e7c0b54a4f140068ccd4", "5730e7c0b54a4f140068ccd3", "5730e7c0b54a4f140068ccd2", "5730e7c0b54a4f140068ccd0", "56d64f7c1c85041400947097", "56d64f7c1c85041400947099", "56cee003aab44d1400b88bb1", "56cee003aab44d1400b88bb3", "56d64f7c1c85041400947098", "56cee003aab44d1400b88bb2", "56d64f7c1c85041400947095", "56cee003aab44d1400b88bb5", "56d1109d17492d1400aab894", "570cd692b3d812140066d2b3", "570cd692b3d812140066d2af", "570cd692b3d812140066d2b2", "570cd692b3d812140066d2b1", "570cd692b3d812140066d2b0", "573426c64776f41900661989", "573426c64776f41900661988", "573426c64776f4190066198a", "573426c64776f4190066198b", "573426c64776f41900661987", "5728fb541d04691400778ef0", "5728fb541d04691400778ef1", "5728fb541d04691400778eef", "5726de80dd62a815002e9350", "5726de80dd62a815002e9351", "5726de80dd62a815002e9352", "5726de80dd62a815002e9354", "5726de80dd62a815002e9353", "5732aa731d5d2e14009ff892", "5732aa731d5d2e14009ff890", "5732aa731d5d2e14009ff88f", "5732aa731d5d2e14009ff891", "5706bf590eeca41400aa0dfb", "5706bf590eeca41400aa0dfd", "5706bf590eeca41400aa0dfe", "5706bf590eeca41400aa0dfc", "5727d72bff5b5019007d96a5", "5727d72bff5b5019007d96a4", "5727d72bff5b5019007d96a2", "5727d72bff5b5019007d96a6", "5727d72bff5b5019007d96a3", "572ea043c246551400ce4426", "572ea043c246551400ce4423", "572ea043c246551400ce4424", "572ea043c246551400ce4422", "572ea043c246551400ce4425", "572ec78cc246551400ce464f", "572ec78cc246551400ce464e", "572ec78cc246551400ce464c", "572ec78cc246551400ce4650", "572ec78cc246551400ce464d", "573028d9a23a5019007fcebb", "573028d9a23a5019007fcebd", "573028d9a23a5019007fcebf", "573028d9a23a5019007fcebe", "573028d9a23a5019007fcebc", "57068e2575f01819005e7bff", "57068e2575f01819005e7bfd", "57068e2575f01819005e7bfe", "57068e2575f01819005e7c00", "57068e2575f01819005e7bfc", "572ea15acb0c0d14000f139f", "572ea15acb0c0d14000f139c", "572ea15acb0c0d14000f139d", "572ea15acb0c0d14000f139e", "5726872cf1498d1400e8e2de", "5726872cf1498d1400e8e2dc", "5726872cf1498d1400e8e2dd", "57303a4904bcaa1900d773da", "57303a4904bcaa1900d773d9", "57303a4904bcaa1900d773d7", "57303a4904bcaa1900d773d8", "57303a4904bcaa1900d773db", "572ea7e9dfa6aa1500f8d269", "5730d9efb54a4f140068ccad", "572ea7e9dfa6aa1500f8d265", "572ea7e9dfa6aa1500f8d266", "5730d9efb54a4f140068ccae", "572ea7e9dfa6aa1500f8d268", "5730d9efb54a4f140068ccaf", "572ea7e9dfa6aa1500f8d267", "5730d9efb54a4f140068ccac", "5731860d05b4da19006bd22d", "5731860d05b4da19006bd22e", "5731860d05b4da19006bd22c", "5731860d05b4da19006bd230", "5731860d05b4da19006bd22f", "571df3d555697319006390c6", "571df3d555697319006390c7", "572a586fb8ce0319002e2ac2", "572a586fb8ce0319002e2ac5", "572a586fb8ce0319002e2ac4", "572a586fb8ce0319002e2ac1", "572a586fb8ce0319002e2ac3", "57262e04ec44d21400f3dbc0", "572645e3dd62a815002e8028", "57262e04ec44d21400f3dbc1", "57262e04ec44d21400f3dbbf", "572645e3dd62a815002e8029", "57262e04ec44d21400f3dbc2", "572645e3dd62a815002e802a", "56f7f695aef2371900625d00", "56f7f695aef2371900625cfe", "56f7f695aef2371900625cfd", "56f7f695aef2371900625cfc", "56f7f695aef2371900625cff", "57296daa1d04691400779427", "57296daa1d04691400779428", "57296daa1d04691400779429", "56f75218aef2371900625b14", "56f75218aef2371900625b11", "56f75218aef2371900625b12", "56f75218aef2371900625b13", "56f75218aef2371900625b15", "572821842ca10214002d9e99", "572821842ca10214002d9e97", "572821842ca10214002d9e98", "572821842ca10214002d9e96", "572f8446947a6a140053ca05", "572f8446947a6a140053ca08", "572f8446947a6a140053ca04", "572f8446947a6a140053ca07", "572f8446947a6a140053ca06", "5727bce93acd2414000deb0d", "5727bce93acd2414000deb0c", "5727bce93acd2414000deb0e", "5727bce93acd2414000deb0f", "5727bce93acd2414000deb0b", "56e7b06e37bdd419002c4360", "56e7b06e37bdd419002c435d", "56e7b06e37bdd419002c435c", "56e7b06e37bdd419002c435e", "56e7b06e37bdd419002c435f", "572ac5ba111d821400f38d57", "572ac5ba111d821400f38d58", "572ac5ba111d821400f38d56", "572ac5ba111d821400f38d54", "572ac5ba111d821400f38d55", "57280f912ca10214002d9cee", "57280f912ca10214002d9cec", "57280f912ca10214002d9cef", "57280f912ca10214002d9cf0", "57280f912ca10214002d9ced", "56f8e05b9e9bad19000a0654", "56f8e05b9e9bad19000a0656", "56f8e05b9e9bad19000a0652", "56f8e05b9e9bad19000a0653", "56f8e05b9e9bad19000a0655", "56e787d900c9c71400d77253", "56e787d900c9c71400d77254", "56e787d900c9c71400d77252", "56e787d900c9c71400d77255", "56d4df312ccc5a1400d832b9", "56beb9203aeaaa14008c92d5", "56bfbacaa10cfb1400551284", "56beb9203aeaaa14008c92d4", "56beb9203aeaaa14008c92d2", "56d4df312ccc5a1400d832ba", "56d4df312ccc5a1400d832b6", "56d4df312ccc5a1400d832b7", "56bfbacaa10cfb1400551283", "56beb9203aeaaa14008c92d1", "56bfbacaa10cfb1400551285", "56bfbacaa10cfb1400551287", "5731b75cb9d445190005e4b2", "5731b75cb9d445190005e4b3", "5731b75cb9d445190005e4b1", "5731b75cb9d445190005e4b4", "5731b75cb9d445190005e4b5", "5726bd02dd62a815002e8ed6", "5726bd02dd62a815002e8ed5", "5726bd02dd62a815002e8ed4", "56d38b8959d6e414001466e0", "56d38b8959d6e414001466e1", "56d38b8959d6e414001466df", "56db63dbe7c41114004b5064", "56d38b8959d6e414001466e2", "570d5dcdfed7b91900d45f57", "570d5dcdfed7b91900d45f55", "570d5dcdfed7b91900d45f56", "570d5dcdfed7b91900d45f53", "570d5dcdfed7b91900d45f54", "56df206e3277331400b4d98f", "56df206e3277331400b4d98d", "56df206e3277331400b4d98e", "56df206e3277331400b4d991", "56df206e3277331400b4d990", "56e19c6ee3433e1400423023", "56e19c6ee3433e1400423021", "56e19c6ee3433e1400423022", "56f96f4f9b226e1400dd1442", "56f96f4f9b226e1400dd1444", "56f96f4f9b226e1400dd1441", "56f96f4f9b226e1400dd1443", "56f96f4f9b226e1400dd1440", "5728a39f3acd2414000dfbae", "5728a39f3acd2414000dfbaf", "57291034af94a219006aa014", "5728a39f3acd2414000dfbad", "57291034af94a219006aa013", "5728a39f3acd2414000dfbb1", "5728a39f3acd2414000dfbb0", "57291034af94a219006aa015", "57291034af94a219006aa016", "57291034af94a219006aa017", "5728359e2ca10214002da0d2", "5728359e2ca10214002da0d1", "5728359e2ca10214002da0d0", "56e14e56cd28a01900c67784", "56e14e56cd28a01900c67787", "56e14e56cd28a01900c67783", "56e14e56cd28a01900c67786", "56e14e56cd28a01900c67785", "5727f6c94b864d19001640c0", "5727f6c94b864d19001640be", "5727f6c94b864d19001640bd", "5727f6c94b864d19001640bc", "5727f6c94b864d19001640bf", "570da3fadf2f5219002ed094", "570da3fadf2f5219002ed098", "570da3fadf2f5219002ed097", "570da3fadf2f5219002ed096", "570da3fadf2f5219002ed095", "56e42e3739bdeb140034791c", "56e42e3739bdeb1400347920", "56e42e3739bdeb140034791f", "56e42e3739bdeb140034791d", "56e42e3739bdeb140034791e", "57279dae4b864d19001638a7", "57279dae4b864d19001638a9", "57279dae4b864d19001638a8", "57279dae4b864d19001638a6", "56f89f569b226e1400dd0ce1", "56f89f569b226e1400dd0cdf", "56f89f569b226e1400dd0ce3", "56f89f569b226e1400dd0ce2", "56f89f569b226e1400dd0ce0", "56e427248c00841900fbaeff", "56e427248c00841900fbaf00", "56e427248c00841900fbaf01", "56e427248c00841900fbaf03", "56e427248c00841900fbaf02", "57312407a5e9cc1400cdbc96", "57312407a5e9cc1400cdbc95", "57312407a5e9cc1400cdbc97", "56de7997cffd8e1900b4b955", "56de7997cffd8e1900b4b954", "56de7997cffd8e1900b4b956", "56de7997cffd8e1900b4b957", "56dfb970231d4119001abd0f", "56dfb970231d4119001abd11", "56dfb970231d4119001abd10", "572a3ecfaf94a219006aa908", "572a3ecfaf94a219006aa909", "572a3ecfaf94a219006aa907", "5726dc6d708984140094d3ee", "5726dc6d708984140094d3ed", "5726dc6d708984140094d3eb", "5726dc6d708984140094d3ec", "5726dc6d708984140094d3ef", "572ed29703f9891900756a53", "572ed29703f9891900756a56", "572ed29703f9891900756a57", "572ed29703f9891900756a54", "572ed29703f9891900756a55", "56fad7698f12f319006301e6", "56fad7698f12f319006301e7", "56fad7698f12f319006301e5", "56d62c011c85041400946f6e", "56d9cadedc89441400fdb82d", "56d9cadedc89441400fdb82c", "56d62c011c85041400946f6f", "56d9cadedc89441400fdb82e", "56d62c011c85041400946f70", "56dfd7cf231d4119001abe30", "56dfd7cf231d4119001abe31", "56dfd7cf231d4119001abe2f", "56dfd7cf231d4119001abe33", "56dfd7cf231d4119001abe32", "570d3734fed7b91900d45d2d", "570d3734fed7b91900d45d2c", "570d3734fed7b91900d45d2a", "570d3734fed7b91900d45d2b", "570d3734fed7b91900d45d29", "5708085c9e06ca38007e951e", "5708085c9e06ca38007e951f", "5708085c9e06ca38007e951d", "5708085c9e06ca38007e951b", "5708085c9e06ca38007e951c", "56f964559e9bad19000a08be", "56f964559e9bad19000a08bf", "56f964559e9bad19000a08bd", "56f964559e9bad19000a08c0", "572713e6f1498d1400e8f342", "572713e6f1498d1400e8f346", "572713e6f1498d1400e8f344", "572713e6f1498d1400e8f343", "572713e6f1498d1400e8f345", "57279112708984140094e118", "57279112708984140094e11b", "57279112708984140094e119", "57279112708984140094e117", "57279112708984140094e11a", "56d240dbb329da140004ec78", "56d240dbb329da140004ec76", "56d240dbb329da140004ec77", "571a711910f8ca1400305039", "571a711910f8ca140030503a", "571a711910f8ca1400305038", "571a711910f8ca1400305037", "572fa78c04bcaa1900d76b54", "572fa78c04bcaa1900d76b55", "572fa78c04bcaa1900d76b53", "572e995fdfa6aa1500f8d1c7", "572e995fdfa6aa1500f8d1c9", "572e995fdfa6aa1500f8d1c6", "572fa78c04bcaa1900d76b57", "572e995fdfa6aa1500f8d1c8", "572e995fdfa6aa1500f8d1c5", "572fa78c04bcaa1900d76b56", "57300e8a04bcaa1900d770c5", "57300e8a04bcaa1900d770c2", "57300e8a04bcaa1900d770c3", "57300e8a04bcaa1900d770c1", "57300e8a04bcaa1900d770c4", "56e86f5b99e8941900975e04", "570d5c15fed7b91900d45f24", "570d5c15fed7b91900d45f22", "570d5c15fed7b91900d45f21", "570d5c15fed7b91900d45f23", "570d5c15fed7b91900d45f25", "5732833db9988014000c7655", "5732833db9988014000c7653", "5732833db9988014000c7654", "5732833db9988014000c7652", "5732833db9988014000c7656", "56f8e0b29e9bad19000a065d", "56f8e0b29e9bad19000a065e", "56f8e0b29e9bad19000a065f", "56f8e0b29e9bad19000a0660", "56f8e0b29e9bad19000a065c", "570dc90a0b85d914000d7b5b", "570dc90a0b85d914000d7b5d", "570dc90a0b85d914000d7b5e", "570dc90a0b85d914000d7b5c", "5725dbef271a42140099d2af", "5725dbef271a42140099d2ad", "5725dbef271a42140099d2ae", "56d4ceac2ccc5a1400d83243", "56be9eea3aeaaa14008c9185", "56d4ceac2ccc5a1400d83240", "56d4ceac2ccc5a1400d83241", "56be9eea3aeaaa14008c9187", "56bfa761a10cfb1400551202", "56d4ceac2ccc5a1400d83242", "56be9eea3aeaaa14008c9184", "56bfa761a10cfb1400551201", "56bea1f53aeaaa14008c9191", "56bea1f53aeaaa14008c918e", "56bfa761a10cfb1400551205", "56be9eea3aeaaa14008c9186", "56bfa761a10cfb1400551204", "56bea1f53aeaaa14008c918f", "56d4ceac2ccc5a1400d83244", "570d3e82fed7b91900d45d86", "570d3e82fed7b91900d45d83", "570d3e82fed7b91900d45d87", "570d3e82fed7b91900d45d85", "570d3e82fed7b91900d45d84", "56cf8304234ae51400d9bddb", "56d12c3d17492d1400aabb4e", "56d12c3d17492d1400aabb4f", "56d12c3d17492d1400aabb4c", "56cf8304234ae51400d9bdd9", "56d12c3d17492d1400aabb4d", "57303222947a6a140053d274", "57303222947a6a140053d273", "57303222947a6a140053d272", "5730a3018ab72b1400f9c622", "5730a3018ab72b1400f9c61f", "5730a3018ab72b1400f9c620", "5730a3018ab72b1400f9c621", "5730a3018ab72b1400f9c61e", "571de3ebb64a571400c71dd9", "571de3ebb64a571400c71ddb", "571de3ebb64a571400c71dda", "571de3ebb64a571400c71dd8", "57277a21f1498d1400e8f926", "57277a21f1498d1400e8f922", "57277a21f1498d1400e8f924", "57277a21f1498d1400e8f923", "57277a21f1498d1400e8f925", "57269ef4f1498d1400e8e52d", "57269ef4f1498d1400e8e52a", "57269ef4f1498d1400e8e52b", "57269ef4f1498d1400e8e52c", "57269ef4f1498d1400e8e52e", "56f96fee9b226e1400dd1454", "56f96fee9b226e1400dd1455", "56f96fee9b226e1400dd1457", "56f96fee9b226e1400dd1456", "56e7305b37bdd419002c3de7", "56e7305b37bdd419002c3de5", "572ffd1b04bcaa1900d76fbf", "572ffd1b04bcaa1900d76fc0", "572ffd1b04bcaa1900d76fbe", "572ffd1b04bcaa1900d76fc1", "572ffd1b04bcaa1900d76fbd", "5730c888aca1c71400fe5ab8", "5730c888aca1c71400fe5ab9", "570e7c160b85d914000d7f3e", "570e7c160b85d914000d7f3f", "570e7c160b85d914000d7f41", "570e7c160b85d914000d7f3d", "570e7c160b85d914000d7f40", "572e843cc246551400ce42ab", "572e843cc246551400ce42aa" ], "sha256": "8a53ba2cd6ea781f62cd690dfd4b74b060d8453a9aa89946d1e66de435f7d459" }, "validation": { "ids": [ "5706c0ba2eaba6190074ace7", "5706c0ba2eaba6190074ace8", "5706c0ba2eaba6190074ace6", "5706c0ba2eaba6190074ace9", "5710ee7ea58dae1900cd6b54", "5710ee7ea58dae1900cd6b55", "5710ee7ea58dae1900cd6b56", "57262d8f89a1e219009ac4fc", "57262d8f89a1e219009ac4fe", "57262d8f89a1e219009ac4ff", "57262d8f89a1e219009ac4fd", "572eaa1cdfa6aa1500f8d27d", "5730d9eaf6cb411900e244ec", "5730d9eaf6cb411900e244ee", "572eaa1cdfa6aa1500f8d27a", "572eaa1cdfa6aa1500f8d279", "5730d9eaf6cb411900e244ef", "5730d9eaf6cb411900e244ed", "572eaa1cdfa6aa1500f8d27b", "572eaa1cdfa6aa1500f8d27c", "57275373f1498d1400e8f645", "57294fd36aef051400154ca6", "57294fd36aef051400154ca7", "57275373f1498d1400e8f643", "57275373f1498d1400e8f644", "57294fd36aef051400154ca8", "57275373f1498d1400e8f646", "57275373f1498d1400e8f642", "56cfb1a2234ae51400d9be87", "56cfb1a2234ae51400d9be89", "56cfb1a2234ae51400d9be88", "56cfb1a2234ae51400d9be85", "56cfb1a2234ae51400d9be86", "570a89ad6d058f1900182f77", "570a89ad6d058f1900182f78", "570a89ad6d058f1900182f75", "570a89ad6d058f1900182f74", "570a89ad6d058f1900182f76", "570da377df2f5219002ed08e", "570da377df2f5219002ed08c", "570da377df2f5219002ed08d", "570da377df2f5219002ed08a", "570da377df2f5219002ed08b", "573260fde17f3d1400422917", "573260fde17f3d1400422919", "573260fde17f3d1400422916", "573260fde17f3d1400422918", "573260fde17f3d1400422915", "57312a8e05b4da19006bce1b", "57312a8e05b4da19006bce1a", "57312a8e05b4da19006bce18", "57312a8e05b4da19006bce19", "57312a8e05b4da19006bce1c", "57310c4c497a881900248b2b", "57310c4c497a881900248b2d", "57310c4c497a881900248b2f", "57310c4c497a881900248b2c", "57310c4c497a881900248b2e", "56e76ba237bdd419002c3f8b", "56e76ba237bdd419002c3f8a", "56e76ba237bdd419002c3f8d", "56e76ba237bdd419002c3f89", "56e76ba237bdd419002c3f8c", "56df56288bc80c19004e4ac9", "57280b934b864d19001642f2", "57280b934b864d19001642f5", "57280b934b864d19001642f3", "57280b934b864d19001642f6", "57280b934b864d19001642f4", "572782aff1498d1400e8fa36", "572782aff1498d1400e8fa39", "572782aff1498d1400e8fa38", "572782aff1498d1400e8fa37", "572782aff1498d1400e8fa3a", "57268f9cf1498d1400e8e3d6", "57268f9cf1498d1400e8e3d3", "57268f9cf1498d1400e8e3d2", "57268f9cf1498d1400e8e3d4", "57268f9cf1498d1400e8e3d5", "57294a001d0469140077925c", "57294a001d0469140077925b", "57294a001d0469140077925e", "57294a001d0469140077925d", "571a095c4faf5e1900b8a877", "571a095c4faf5e1900b8a878", "571a095c4faf5e1900b8a879", "571a095c4faf5e1900b8a87a", "571a095c4faf5e1900b8a876", "57279cf2708984140094e251", "57279cf2708984140094e24f", "57279cf2708984140094e250", "57279cf2708984140094e252", "570d5502b3d812140066d6d2", "570d5502b3d812140066d6d3", "570d5502b3d812140066d6cf", "570d5502b3d812140066d6d0", "570d5502b3d812140066d6d1", "56de67e0cffd8e1900b4b889", "56de67e0cffd8e1900b4b888", "56de67e0cffd8e1900b4b88a", "572948523f37b3190047820f", "572948523f37b3190047820e", "572948523f37b3190047820d", "5727baba3acd2414000deac1", "5727baba3acd2414000deac3", "5727baba3acd2414000deac5", "5727baba3acd2414000deac4", "5727baba3acd2414000deac2", "5728720dff5b5019007da22a", "5728720dff5b5019007da22d", "5728720dff5b5019007da22c", "5728720dff5b5019007da22b", "5726abd7f1498d1400e8e6ba", "5726abd7f1498d1400e8e6b8", "5726abd7f1498d1400e8e6b7", "5726abd7f1498d1400e8e6b6", "5726abd7f1498d1400e8e6b9", "5735c50be853931400426b6e", "5735c50be853931400426b70", "5735c50be853931400426b6d", "5735c50be853931400426b71", "5735c50be853931400426b6f", "572fd2e504bcaa1900d76d91", "572fd2e504bcaa1900d76d8d", "572fd2e504bcaa1900d76d8f", "572fd2e504bcaa1900d76d8e", "572fd2e504bcaa1900d76d90", "57271883dd62a815002e98ee" ], "sha256": "c87eb694be5603ede27e07e9ce90db909334825ae460a43106882d864f2c513d" }, "heldout": { "ids": [ "57291f153f37b31900478044", "57291f153f37b31900478043", "57291f153f37b31900478045", "57291f153f37b31900478046", "56de3e414396321400ee26d9", "56de3e414396321400ee26d8", "57281bb84b864d190016449a", "57281bb84b864d190016449c", "57281bb84b864d190016449d", "57281bb84b864d190016449b", "5725fb8138643c19005acf42", "5725fb8138643c19005acf43", "5725fb8138643c19005acf3f", "5725fb8138643c19005acf40", "5725fb8138643c19005acf41", "56f88690a6d7ea1400e17723", "56f88690a6d7ea1400e17724", "56f88690a6d7ea1400e17727", "56f88690a6d7ea1400e17725", "56f88690a6d7ea1400e17726", "5730088e947a6a140053cfac", "5730088e947a6a140053cfb0", "5730088e947a6a140053cfae", "5730088e947a6a140053cfad", "5730088e947a6a140053cfaf", "573755afc3c5551400e51eb5", "573755afc3c5551400e51eb4", "573755afc3c5551400e51eb3", "572686ac5951b619008f75ad", "572686ac5951b619008f75aa", "572686ac5951b619008f75ab", "572686ac5951b619008f75ac", "572686ac5951b619008f75a9", "56f88025aef2371900626121", "56f88025aef237190062611f", "56f88025aef2371900626122", "56f88025aef237190062611e", "56f88025aef2371900626120", "56e74d1f00c9c71400d76f6f", "56e74d1f00c9c71400d76f6e", "56e74d1f00c9c71400d76f71", "56e74d1f00c9c71400d76f70", "56d70daa0d65d21400198336", "56d70daa0d65d21400198334", "56bf4e1b3aeaaa14008c95c1", "56bebbbf3aeaaa14008c9319", "56bebbbf3aeaaa14008c9318", "56bf4e1b3aeaaa14008c95bd", "56d9a7addc89441400fdb6a8", "56bf4e1b3aeaaa14008c95be", "56bebbbf3aeaaa14008c9315", "56bf4e1b3aeaaa14008c95c0", "56d70daa0d65d21400198333", "56d70daa0d65d21400198335", "56bebbbf3aeaaa14008c9317", "56d9a7addc89441400fdb6ab", "56bebbbf3aeaaa14008c9316", "56d70daa0d65d21400198332", "56beb6533aeaaa14008c9290", "56bf3e803aeaaa14008c9588", "56d6fea90d65d214001982b0", "56d6fea90d65d214001982b2", "56bf3e803aeaaa14008c9589", "56beb6533aeaaa14008c9291", "56d99da8dc89441400fdb5fd", "56d99da8dc89441400fdb600", "56beb6533aeaaa14008c928e", "56beb6533aeaaa14008c928d", "56bf3e803aeaaa14008c958b", "56d6fea90d65d214001982af", "56d99da8dc89441400fdb5ff", "56d6fea90d65d214001982ae", "5726577f708984140094c301", "5726577f708984140094c304", "5726577f708984140094c303", "5726577f708984140094c302", "571135b8a58dae1900cd6d0e", "571135b8a58dae1900cd6d10", "571135b8a58dae1900cd6d11", "571135b8a58dae1900cd6d0f", "571135b8a58dae1900cd6d12", "56beab833aeaaa14008c91d2", "56beae423aeaaa14008c91f5", "56be4eafacb8001400a50302", "56beae423aeaaa14008c91f4", "56d98b33dc89441400fdb53d", "56d601e41c85041400946ed1", "56d204ade7d4791d00902604", "56d601e41c85041400946ecf", "56beab833aeaaa14008c91d3", "56bf17653aeaaa14008c9515", "56d601e41c85041400946ed0", "56be4eafacb8001400a50304", "56d601e41c85041400946ece", "56beae423aeaaa14008c91f7", "56beae423aeaaa14008c91f6", "56d98b33dc89441400fdb53b", "56d601e41c85041400946ed2", "56d204ade7d4791d00902603", "56bf17653aeaaa14008c9511", "56be4eafacb8001400a50303", "56d98b33dc89441400fdb53c", "56d98b33dc89441400fdb53e", "56bf17653aeaaa14008c9513", "56beab833aeaaa14008c91d4", "56bf17653aeaaa14008c9514", "570d2417fed7b91900d45c3e", "570d2417fed7b91900d45c41", "570d2417fed7b91900d45c3d", "570d2417fed7b91900d45c40", "570d2417fed7b91900d45c3f", "57265bdfdd62a815002e82a0", "57265bdfdd62a815002e829f", "57265bdfdd62a815002e829e", "57265bdfdd62a815002e82a2", "57265bdfdd62a815002e82a1", "56de1645cffd8e1900b4b5d1", "56de1645cffd8e1900b4b5d2", "56de1645cffd8e1900b4b5d0", "57332442d058e614000b5721", "57332442d058e614000b5722", "57332442d058e614000b5720", "57332442d058e614000b5723", "57332442d058e614000b5724", "5726378238643c19005ad315", "572629c6271a42140099d6a6", "5726378238643c19005ad314", "572629c6271a42140099d6a3", "572629c6271a42140099d6a4", "572629c6271a42140099d6a5", "5726378238643c19005ad313", "56e110c3cd28a01900c67535", "56e110c3cd28a01900c67532", "56e110c3cd28a01900c67533", "56e110c3cd28a01900c67531", "56e110c3cd28a01900c67534", "57274a8ff1498d1400e8f5be", "57274a8ff1498d1400e8f5bc", "57274a8ff1498d1400e8f5bd", "5727d3843acd2414000ded69", "5727d3843acd2414000ded6b", "5727d3843acd2414000ded6a", "56f86e91aef2371900626067", "56f86e91aef237190062606a", "56f86e91aef237190062606b", "56f86e91aef2371900626068", "56f86e91aef2371900626069", "56e10b6ee3433e1400422b27", "56e10b6ee3433e1400422b25", "56e10b6ee3433e1400422b24", "56e10b6ee3433e1400422b26", "57097d63ed30961900e841ff", "57097d63ed30961900e841fd", "57097d63ed30961900e841fc", "57097d63ed30961900e841fe", "57097d63ed30961900e84200", "572835854b864d1900164732", "572835854b864d1900164730", "572835854b864d1900164731", "572835854b864d190016472e", "572835854b864d190016472f", "56e11e57e3433e1400422c2a", "56e11e57e3433e1400422c29", "56e11e57e3433e1400422c28", "57283e652ca10214002da169", "57283e652ca10214002da167", "57283e652ca10214002da16a", "57283e652ca10214002da168", "57283e652ca10214002da166", "5730bb522461fd1900a9d011", "5730bb522461fd1900a9d012", "5730bb522461fd1900a9d013", "5730bb522461fd1900a9d015", "56e749dd00c9c71400d76f55", "56e749dd00c9c71400d76f53", "56e749dd00c9c71400d76f51", "56e749dd00c9c71400d76f52", "56e749dd00c9c71400d76f54", "57268d02f1498d1400e8e378", "57268d02f1498d1400e8e37a", "57268d02f1498d1400e8e37b", "57268d02f1498d1400e8e37c", "57268d02f1498d1400e8e379", "56e1a564cd28a01900c67a48", "56e1a564cd28a01900c67a4c", "56e1a564cd28a01900c67a4b", "56e1a564cd28a01900c67a49", "56e1a564cd28a01900c67a4a", "57276166dd62a815002e9bdb", "57276166dd62a815002e9bd8", "57276166dd62a815002e9bda", "57276166dd62a815002e9bd9", "5730eb5b497a881900248a42", "5730eb5b497a881900248a41", "5730eb5b497a881900248a43", "5730eb5b497a881900248a40", "5730eb5b497a881900248a3f", "5728e3c33acd2414000e0132", "5728e3c33acd2414000e0133", "5728e3c33acd2414000e0131", "5728e3c33acd2414000e0135", "572822233acd2414000df556", "572822233acd2414000df557", "5728e3c33acd2414000e0134", "572822233acd2414000df555", "572822233acd2414000df559", "572822233acd2414000df558", "5725d79e89a1e219009abf90", "5725d79e89a1e219009abf93", "5725d79e89a1e219009abf92", "5725d79e89a1e219009abf94", "5725d79e89a1e219009abf91", "5727db85ff5b5019007d96fe", "5727db85ff5b5019007d96ff", "5727db85ff5b5019007d96fd", "5727db85ff5b5019007d96fc", "5729e02f1d0469140077963b", "5729e02f1d0469140077963c", "5727ef664b864d1900164062", "5727ef664b864d1900164063", "5729e02f1d04691400779639", "5729e02f1d0469140077963a", "5727ef664b864d1900164061", "5727ef664b864d1900164060", "5727cb4b2ca10214002d9677", "5727cb4b2ca10214002d9676", "5727cb4b2ca10214002d9678", "5727cb4b2ca10214002d967a", "5727cb4b2ca10214002d9679", "56de40da4396321400ee270a", "56de40da4396321400ee2709", "56de40da4396321400ee2708", "57283d173acd2414000df78f", "5729e6313f37b319004785ad", "57283d173acd2414000df791", "57283d173acd2414000df792", "57283d173acd2414000df793", "5729e6313f37b319004785a9", "5725c071271a42140099d129", "5729e6313f37b319004785ac", "5729e6313f37b319004785ab", "5729e6313f37b319004785aa", "5725c071271a42140099d127", "5725c071271a42140099d12b", "5725c071271a42140099d12a", "57283d173acd2414000df790", "5725c071271a42140099d128", "572a1dbb3f37b319004786f6", "572a1dbb3f37b319004786f8", "572a1dbb3f37b319004786f5", "572a1dbb3f37b319004786f9", "572a1dbb3f37b319004786f7", "572864dd4b864d1900164979", "572864dd4b864d1900164976", "572864dd4b864d190016497a", "572864dd4b864d1900164977" ], "sha256": "fc8d459e0ceb6f26a0b5386d5c5122e694d0fc347b074000c102fe4d9c18d21e" } }, "tokenizer": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "template_kwargs": { "enable_thinking": false }, "limits": { "complete_chat_tokens": 384, "answer_tokens": 32 }, "seed_selection": "none" }, "prediction_csv_sha256": "be7291882e2d5577671c5c132618e5c3a8d82ebfabbc08cd5f69b82bb100a3bf" }, "cmrc": { "rows": [ { "method": "base", "label": "base", "seeds": [ 17 ], "trainable_parameters": 0, "em": 46.875, "em_std": 0, "f1": 71.70898113149433, "f1_std": 0, "adapter_kib": 0.0 }, { "method": "torch", "label": "torch", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 48.307291666666664, "em_std": 0.22552744890219756, "f1": 72.1468682252435, "f1_std": 1.2271050995278894, "adapter_kib": 512.359375 }, { "method": "peft", "label": "peft", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 46.875, "em_std": 1.953125, "f1": 70.77493248953475, "f1_std": 1.4007142440287006, "adapter_kib": 512.546875 }, { "method": "arti", "label": "arti", "seeds": [ 17, 29, 43 ], "trainable_parameters": 131072, "em": 48.307291666666664, "em_std": 0.22552744890219756, "f1": 72.1468682252435, "f1_std": 1.2271050995278894, "adapter_kib": 545.3828125 } ], "sample": { "id": "DEV_479_QUERY_0", "context": "单星是不和任何其他同类星体组成聚星系统的单一天体。恒星、行星、小行星都存在单星系统。长久以来人们一直认为宇宙中大部分的恒星是聚星系统。以双星和三合星较为多见。单星反而较少。然而，根据2006年的一项研究，银河系中的主序星有2/3都是单星，改变了人们的传统认识。单星构成的恒星系统最为简单，围绕恒星运行的小星体(例如行星)的轨道比较简单，不会因为受到多个大天体的影响而走出怪异的路线，从而较容易维持合适的环境、温度和光照，有利于生命的存在。单星也可能因为位置的缘故，被人错认为和另一颗恒星组成双星。实际上两者之间并无相互旋转的运动关系。这样的双星被称为光学双星。太阳一般认为就是一个单星。尽管存在着未经证实的太阳伴星的传闻。可能是因为形成方式的不同，行星和小行星大多是单星。只有少数的小行星或者卫星构成双星系统。但是某些大型的卫星有时也被认为是与自己的母星构成双行星系统。例如:", "question": "单星的定义是什么？", "references": [ "不和任何其他同类星体组成聚星系统的单一天体" ], "predictions": { "base": "单星是不和任何其他同类星体组成聚星系统的单一天体。", "torch": "单星是不和任何其他同类星体组成聚星系统的单一天体。", "peft": "单星是不和任何其他同类星体组成聚星系统的单一天体。", "arti": "单星是不和任何其他同类星体组成聚星系统的单一天体。" } }, "eligibility": { "train": { "source_rows": 10142, "eligible_rows": 4380, "overlength_excluded": 5181, "answer_over_32_excluded": 581, "eligible_contexts": 1168 }, "dev": { "source_rows": 3219, "eligible_rows": 1422, "overlength_excluded": 1762, "answer_over_32_excluded": 35, "eligible_contexts": 403 } }, "split_manifest": { "dataset": "cmrc", "selection": "sha256 context-first then original-ID order; validation contexts removed from train; public dev heldout", "counts": { "train": 512, "validation": 128, "heldout": 256 }, "eligibility": { "train": { "source_rows": 10142, "eligible_rows": 4380, "overlength_excluded": 5181, "answer_over_32_excluded": 581, "eligible_contexts": 1168 }, "dev": { "source_rows": 3219, "eligible_rows": 1422, "overlength_excluded": 1762, "answer_over_32_excluded": 35, "eligible_contexts": 403 } }, "splits": { "train": { "ids": [ "TRAIN_33_QUERY_4", "TRAIN_33_QUERY_3", "TRAIN_33_QUERY_0", "TRAIN_33_QUERY_1", "TRAIN_33_QUERY_2", "TRAIN_633_QUERY_0", "TRAIN_633_QUERY_1", "TRAIN_2735_QUERY_2", "TRAIN_2735_QUERY_3", "TRAIN_2735_QUERY_0", "TRAIN_2735_QUERY_1", "TRAIN_2313_QUERY_1", "TRAIN_2313_QUERY_3", "TRAIN_2313_QUERY_2", "TRAIN_2313_QUERY_4", "TRAIN_2313_QUERY_0", "TRAIN_1827_QUERY_3", "TRAIN_1827_QUERY_1", "TRAIN_1827_QUERY_0", "TRAIN_1827_QUERY_2", "TRAIN_2389_QUERY_1", "TRAIN_2389_QUERY_3", "TRAIN_2389_QUERY_0", "TRAIN_2389_QUERY_4", "TRAIN_2389_QUERY_2", "TRAIN_3696_QUERY_3", "TRAIN_3696_QUERY_0", "TRAIN_3696_QUERY_2", "TRAIN_3696_QUERY_1", "TRAIN_882_QUERY_0", "TRAIN_2507_QUERY_0", "TRAIN_2507_QUERY_2", "TRAIN_1623_QUERY_4", "TRAIN_1623_QUERY_0", "TRAIN_1623_QUERY_2", "TRAIN_1623_QUERY_3", "TRAIN_1623_QUERY_1", "TRAIN_940_QUERY_1", "TRAIN_2265_QUERY_0", "TRAIN_2265_QUERY_2", "TRAIN_2265_QUERY_3", "TRAIN_2265_QUERY_4", "TRAIN_241_QUERY_4", "TRAIN_241_QUERY_0", "TRAIN_241_QUERY_2", "TRAIN_241_QUERY_1", "TRAIN_675_QUERY_0", "TRAIN_675_QUERY_4", "TRAIN_675_QUERY_2", "TRAIN_675_QUERY_3", "TRAIN_675_QUERY_1", "TRAIN_764_QUERY_2", "TRAIN_764_QUERY_4", "TRAIN_764_QUERY_0", "TRAIN_764_QUERY_3", "TRAIN_746_QUERY_3", "TRAIN_746_QUERY_0", "TRAIN_746_QUERY_1", "TRAIN_746_QUERY_2", "TRAIN_1206_QUERY_2", "TRAIN_1206_QUERY_1", "TRAIN_1206_QUERY_3", "TRAIN_1206_QUERY_0", "TRAIN_1206_QUERY_4", "TRAIN_145_QUERY_1", "TRAIN_145_QUERY_4", "TRAIN_145_QUERY_0", "TRAIN_145_QUERY_3", "TRAIN_135_QUERY_3", "TRAIN_135_QUERY_2", "TRAIN_135_QUERY_0", "TRAIN_135_QUERY_4", "TRAIN_2322_QUERY_2", "TRAIN_2322_QUERY_4", "TRAIN_2322_QUERY_3", "TRAIN_2322_QUERY_1", "TRAIN_2322_QUERY_0", "TRAIN_13_QUERY_3", "TRAIN_13_QUERY_0", "TRAIN_2765_QUERY_2", "TRAIN_2765_QUERY_3", "TRAIN_2765_QUERY_0", "TRAIN_2765_QUERY_1", "TRAIN_686_QUERY_2", "TRAIN_686_QUERY_0", "TRAIN_686_QUERY_1", "TRAIN_696_QUERY_2", "TRAIN_696_QUERY_1", "TRAIN_696_QUERY_0", "TRAIN_2290_QUERY_0", "TRAIN_2290_QUERY_4", "TRAIN_2290_QUERY_2", "TRAIN_2290_QUERY_1", "TRAIN_172_QUERY_0", "TRAIN_172_QUERY_1", "TRAIN_172_QUERY_4", "TRAIN_172_QUERY_3", "TRAIN_172_QUERY_2", "TRAIN_1738_QUERY_1", "TRAIN_1738_QUERY_0", "TRAIN_1738_QUERY_2", "TRAIN_1738_QUERY_4", "TRAIN_1337_QUERY_1", "TRAIN_1337_QUERY_0", "TRAIN_1337_QUERY_2", "TRAIN_1337_QUERY_3", "TRAIN_1624_QUERY_3", "TRAIN_1624_QUERY_2", "TRAIN_1624_QUERY_4", "TRAIN_1624_QUERY_1", "TRAIN_1624_QUERY_0", "TRAIN_3092_QUERY_1", "TRAIN_3050_QUERY_1", "TRAIN_3050_QUERY_0", "TRAIN_720_QUERY_2", "TRAIN_720_QUERY_0", "TRAIN_817_QUERY_0", "TRAIN_817_QUERY_1", "TRAIN_817_QUERY_3", "TRAIN_817_QUERY_2", "TRAIN_652_QUERY_3", "TRAIN_652_QUERY_1", "TRAIN_652_QUERY_0", "TRAIN_652_QUERY_2", "TRAIN_2536_QUERY_3", "TRAIN_2536_QUERY_1", "TRAIN_2536_QUERY_2", "TRAIN_2536_QUERY_0", "TRAIN_576_QUERY_2", "TRAIN_576_QUERY_1", "TRAIN_576_QUERY_0", "TRAIN_3684_QUERY_2", "TRAIN_3684_QUERY_0", "TRAIN_3684_QUERY_1", "TRAIN_3684_QUERY_3", "TRAIN_2608_QUERY_3", "TRAIN_2608_QUERY_0", "TRAIN_2608_QUERY_1", "TRAIN_2608_QUERY_4", "TRAIN_2608_QUERY_2", "TRAIN_1314_QUERY_4", "TRAIN_1314_QUERY_2", "TRAIN_1314_QUERY_3", "TRAIN_1314_QUERY_0", "TRAIN_1314_QUERY_1", "TRAIN_1615_QUERY_4", "TRAIN_1615_QUERY_2", "TRAIN_1615_QUERY_0", "TRAIN_1615_QUERY_3", "TRAIN_1615_QUERY_1", "TRAIN_1284_QUERY_4", "TRAIN_1284_QUERY_3", "TRAIN_1284_QUERY_1", "TRAIN_1284_QUERY_0", "TRAIN_1284_QUERY_2", "TRAIN_162_QUERY_2", "TRAIN_162_QUERY_0", "TRAIN_162_QUERY_1", "TRAIN_1228_QUERY_4", "TRAIN_1228_QUERY_0", "TRAIN_1228_QUERY_2", "TRAIN_1228_QUERY_3", "TRAIN_1228_QUERY_1", "TRAIN_1736_QUERY_0", "TRAIN_1736_QUERY_1", "TRAIN_1736_QUERY_4", "TRAIN_1736_QUERY_2", "TRAIN_3551_QUERY_3", "TRAIN_3551_QUERY_1", "TRAIN_3551_QUERY_2", "TRAIN_3551_QUERY_0", "TRAIN_2772_QUERY_1", "TRAIN_2772_QUERY_0", "TRAIN_2772_QUERY_3", "TRAIN_2772_QUERY_2", "TRAIN_120_QUERY_2", "TRAIN_120_QUERY_0", "TRAIN_120_QUERY_4", "TRAIN_120_QUERY_3", "TRAIN_729_QUERY_0", "TRAIN_729_QUERY_3", "TRAIN_729_QUERY_1", "TRAIN_3807_QUERY_1", "TRAIN_3807_QUERY_3", "TRAIN_3807_QUERY_0", "TRAIN_3807_QUERY_4", "TRAIN_3807_QUERY_2", "TRAIN_1277_QUERY_0", "TRAIN_1277_QUERY_1", "TRAIN_1277_QUERY_4", "TRAIN_1277_QUERY_2", "TRAIN_2274_QUERY_4", "TRAIN_2274_QUERY_1", "TRAIN_2274_QUERY_3", "TRAIN_2274_QUERY_2", "TRAIN_2274_QUERY_0", "TRAIN_2504_QUERY_3", "TRAIN_2504_QUERY_0", "TRAIN_2504_QUERY_2", "TRAIN_894_QUERY_2", "TRAIN_894_QUERY_1", "TRAIN_2505_QUERY_1", "TRAIN_2505_QUERY_2", "TRAIN_2505_QUERY_3", "TRAIN_2505_QUERY_0", "TRAIN_3812_QUERY_0", "TRAIN_3812_QUERY_2", "TRAIN_3812_QUERY_1", "TRAIN_3812_QUERY_3", "TRAIN_663_QUERY_2", "TRAIN_663_QUERY_4", "TRAIN_663_QUERY_1", "TRAIN_2452_QUERY_1", "TRAIN_2452_QUERY_0", "TRAIN_2452_QUERY_4", "TRAIN_2452_QUERY_3", "TRAIN_2452_QUERY_2", "TRAIN_1023_QUERY_2", "TRAIN_1023_QUERY_0", "TRAIN_1023_QUERY_4", "TRAIN_2179_QUERY_2", "TRAIN_2179_QUERY_1", "TRAIN_2179_QUERY_3", "TRAIN_446_QUERY_1", "TRAIN_446_QUERY_0", "TRAIN_821_QUERY_1", "TRAIN_821_QUERY_0", "TRAIN_1816_QUERY_3", "TRAIN_1816_QUERY_0", "TRAIN_1816_QUERY_2", "TRAIN_2687_QUERY_0", "TRAIN_2687_QUERY_1", "TRAIN_2687_QUERY_2", "TRAIN_2687_QUERY_3", "TRAIN_931_QUERY_1", "TRAIN_931_QUERY_0", "TRAIN_1294_QUERY_0", "TRAIN_1294_QUERY_4", "TRAIN_1294_QUERY_1", "TRAIN_1294_QUERY_2", "TRAIN_1294_QUERY_3", "TRAIN_3718_QUERY_4", "TRAIN_3718_QUERY_1", "TRAIN_3718_QUERY_0", "TRAIN_3718_QUERY_3", "TRAIN_3718_QUERY_2", "TRAIN_1199_QUERY_3", "TRAIN_1199_QUERY_2", "TRAIN_1199_QUERY_4", "TRAIN_1199_QUERY_0", "TRAIN_1703_QUERY_4", "TRAIN_1703_QUERY_2", "TRAIN_1703_QUERY_0", "TRAIN_1703_QUERY_1", "TRAIN_1703_QUERY_3", "TRAIN_2285_QUERY_0", "TRAIN_2285_QUERY_1", "TRAIN_2285_QUERY_2", "TRAIN_2285_QUERY_3", "TRAIN_2285_QUERY_4", "TRAIN_2100_QUERY_2", "TRAIN_2100_QUERY_4", "TRAIN_2100_QUERY_3", "TRAIN_2100_QUERY_1", "TRAIN_2100_QUERY_0", "TRAIN_1647_QUERY_0", "TRAIN_1647_QUERY_4", "TRAIN_1647_QUERY_1", "TRAIN_1647_QUERY_3", "TRAIN_1647_QUERY_2", "TRAIN_3512_QUERY_2", "TRAIN_3512_QUERY_3", "TRAIN_3512_QUERY_4", "TRAIN_3512_QUERY_1", "TRAIN_3512_QUERY_0", "TRAIN_153_QUERY_4", "TRAIN_153_QUERY_0", "TRAIN_586_QUERY_0", "TRAIN_586_QUERY_2", "TRAIN_586_QUERY_4", "TRAIN_586_QUERY_1", "TRAIN_1024_QUERY_2", "TRAIN_1024_QUERY_1", "TRAIN_1024_QUERY_0", "TRAIN_1024_QUERY_4", "TRAIN_2217_QUERY_0", "TRAIN_2512_QUERY_2", "TRAIN_2512_QUERY_0", "TRAIN_2512_QUERY_1", "TRAIN_469_QUERY_2", "TRAIN_107_QUERY_1", "TRAIN_107_QUERY_0", "TRAIN_107_QUERY_4", "TRAIN_1147_QUERY_3", "TRAIN_1147_QUERY_2", "TRAIN_1147_QUERY_0", "TRAIN_2721_QUERY_3", "TRAIN_2721_QUERY_1", "TRAIN_2721_QUERY_0", "TRAIN_508_QUERY_1", "TRAIN_508_QUERY_0", "TRAIN_693_QUERY_1", "TRAIN_693_QUERY_4", "TRAIN_693_QUERY_2", "TRAIN_693_QUERY_0", "TRAIN_693_QUERY_3", "TRAIN_1750_QUERY_4", "TRAIN_1750_QUERY_3", "TRAIN_1750_QUERY_0", "TRAIN_1750_QUERY_1", "TRAIN_1750_QUERY_2", "TRAIN_2596_QUERY_3", "TRAIN_2596_QUERY_1", "TRAIN_2596_QUERY_0", "TRAIN_2596_QUERY_2", "TRAIN_2677_QUERY_0", "TRAIN_2677_QUERY_2", "TRAIN_2677_QUERY_3", "TRAIN_2677_QUERY_1", "TRAIN_2157_QUERY_0", "TRAIN_2157_QUERY_1", "TRAIN_2157_QUERY_2", "TRAIN_2157_QUERY_3", "TRAIN_2680_QUERY_3", "TRAIN_2680_QUERY_2", "TRAIN_2680_QUERY_0", "TRAIN_2680_QUERY_1", "TRAIN_2241_QUERY_2", "TRAIN_2241_QUERY_3", "TRAIN_2241_QUERY_0", "TRAIN_2241_QUERY_1", "TRAIN_2018_QUERY_0", "TRAIN_2018_QUERY_3", "TRAIN_2018_QUERY_1", "TRAIN_2018_QUERY_2", "TRAIN_952_QUERY_1", "TRAIN_952_QUERY_0", "TRAIN_952_QUERY_3", "TRAIN_1528_QUERY_0", "TRAIN_1528_QUERY_2", "TRAIN_3528_QUERY_3", "TRAIN_3528_QUERY_4", "TRAIN_3528_QUERY_2", "TRAIN_3528_QUERY_1", "TRAIN_3528_QUERY_0", "TRAIN_2461_QUERY_2", "TRAIN_2461_QUERY_1", "TRAIN_2461_QUERY_3", "TRAIN_2461_QUERY_0", "TRAIN_2429_QUERY_0", "TRAIN_2429_QUERY_1", "TRAIN_2429_QUERY_2", "TRAIN_2429_QUERY_4", "TRAIN_10_QUERY_1", "TRAIN_10_QUERY_3", "TRAIN_10_QUERY_0", "TRAIN_3085_QUERY_0", "TRAIN_3085_QUERY_1", "TRAIN_1180_QUERY_4", "TRAIN_1180_QUERY_2", "TRAIN_1180_QUERY_1", "TRAIN_1180_QUERY_3", "TRAIN_1180_QUERY_0", "TRAIN_222_QUERY_3", "TRAIN_222_QUERY_0", "TRAIN_222_QUERY_2", "TRAIN_2282_QUERY_0", "TRAIN_2282_QUERY_1", "TRAIN_2034_QUERY_4", "TRAIN_2034_QUERY_2", "TRAIN_2034_QUERY_0", "TRAIN_2034_QUERY_3", "TRAIN_2034_QUERY_1", "TRAIN_1266_QUERY_3", "TRAIN_1266_QUERY_1", "TRAIN_1266_QUERY_0", "TRAIN_1266_QUERY_2", "TRAIN_1266_QUERY_4", "TRAIN_1205_QUERY_2", "TRAIN_1205_QUERY_0", "TRAIN_1205_QUERY_3", "TRAIN_1205_QUERY_4", "TRAIN_1205_QUERY_1", "TRAIN_2760_QUERY_2", "TRAIN_2760_QUERY_0", "TRAIN_2760_QUERY_1", "TRAIN_1280_QUERY_3", "TRAIN_1280_QUERY_2", "TRAIN_1280_QUERY_0", "TRAIN_1280_QUERY_4", "TRAIN_1714_QUERY_4", "TRAIN_1714_QUERY_2", "TRAIN_1714_QUERY_1", "TRAIN_1714_QUERY_0", "TRAIN_1714_QUERY_3", "TRAIN_2124_QUERY_4", "TRAIN_2124_QUERY_2", "TRAIN_2124_QUERY_0", "TRAIN_2124_QUERY_1", "TRAIN_212_QUERY_0", "TRAIN_212_QUERY_4", "TRAIN_212_QUERY_3", "TRAIN_212_QUERY_2", "TRAIN_40_QUERY_0", "TRAIN_40_QUERY_2", "TRAIN_2720_QUERY_0", "TRAIN_2720_QUERY_2", "TRAIN_2720_QUERY_1", "TRAIN_2720_QUERY_3", "TRAIN_842_QUERY_1", "TRAIN_842_QUERY_2", "TRAIN_842_QUERY_3", "TRAIN_2737_QUERY_0", "TRAIN_2737_QUERY_3", "TRAIN_2737_QUERY_2", "TRAIN_2737_QUERY_1", "TRAIN_1247_QUERY_0", "TRAIN_1247_QUERY_2", "TRAIN_1247_QUERY_1", "TRAIN_670_QUERY_1", "TRAIN_670_QUERY_2", "TRAIN_670_QUERY_0", "TRAIN_726_QUERY_1", "TRAIN_726_QUERY_4", "TRAIN_726_QUERY_0", "TRAIN_726_QUERY_2", "TRAIN_726_QUERY_3", "TRAIN_1021_QUERY_3", "TRAIN_1021_QUERY_2", "TRAIN_3669_QUERY_2", "TRAIN_3669_QUERY_0", "TRAIN_3669_QUERY_1", "TRAIN_313_QUERY_2", "TRAIN_313_QUERY_3", "TRAIN_313_QUERY_4", "TRAIN_313_QUERY_1", "TRAIN_313_QUERY_0", "TRAIN_2159_QUERY_0", "TRAIN_2159_QUERY_2", "TRAIN_2159_QUERY_3", "TRAIN_2159_QUERY_1", "TRAIN_2159_QUERY_4", "TRAIN_183_QUERY_0", "TRAIN_183_QUERY_2", "TRAIN_183_QUERY_1", "TRAIN_183_QUERY_4", "TRAIN_339_QUERY_2", "TRAIN_339_QUERY_1", "TRAIN_339_QUERY_0", "TRAIN_339_QUERY_3", "TRAIN_339_QUERY_4", "TRAIN_517_QUERY_0", "TRAIN_517_QUERY_2", "TRAIN_517_QUERY_3", "TRAIN_517_QUERY_4", "TRAIN_517_QUERY_1", "TRAIN_457_QUERY_0", "TRAIN_1656_QUERY_0", "TRAIN_1656_QUERY_1", "TRAIN_1656_QUERY_3", "TRAIN_2213_QUERY_2", "TRAIN_2213_QUERY_0", "TRAIN_2213_QUERY_3", "TRAIN_2213_QUERY_1", "TRAIN_2746_QUERY_0", "TRAIN_2746_QUERY_3", "TRAIN_2746_QUERY_1", "TRAIN_2746_QUERY_2", "TRAIN_283_QUERY_2", "TRAIN_283_QUERY_0", "TRAIN_283_QUERY_1", "TRAIN_1599_QUERY_3", "TRAIN_1599_QUERY_2", "TRAIN_1599_QUERY_1", "TRAIN_3714_QUERY_2", "TRAIN_3714_QUERY_0", "TRAIN_3714_QUERY_4", "TRAIN_3714_QUERY_3", "TRAIN_3714_QUERY_1", "TRAIN_2065_QUERY_0", "TRAIN_2065_QUERY_3", "TRAIN_2065_QUERY_2", "TRAIN_2065_QUERY_1", "TRAIN_3705_QUERY_2", "TRAIN_2603_QUERY_2", "TRAIN_2603_QUERY_4", "TRAIN_2603_QUERY_0", "TRAIN_2603_QUERY_1", "TRAIN_2603_QUERY_3", "TRAIN_2058_QUERY_3", "TRAIN_2058_QUERY_4", "TRAIN_2058_QUERY_0", "TRAIN_2058_QUERY_2", "TRAIN_2058_QUERY_1", "TRAIN_67_QUERY_3", "TRAIN_67_QUERY_4", "TRAIN_67_QUERY_2", "TRAIN_67_QUERY_0", "TRAIN_67_QUERY_1", "TRAIN_2281_QUERY_2", "TRAIN_2281_QUERY_0", "TRAIN_2281_QUERY_1", "TRAIN_2281_QUERY_3", "TRAIN_2281_QUERY_4", "TRAIN_958_QUERY_2", "TRAIN_958_QUERY_1", "TRAIN_958_QUERY_3", "TRAIN_958_QUERY_0", "TRAIN_3791_QUERY_0", "TRAIN_3791_QUERY_3", "TRAIN_3791_QUERY_1", "TRAIN_2321_QUERY_1" ], "sha256": "23ebaa546c32d8a4034a641cee9ac4329061ad4d0cff834414b2ce91f8ee4ca4" }, "validation": { "ids": [ "TRAIN_3723_QUERY_2", "TRAIN_3723_QUERY_1", "TRAIN_3723_QUERY_0", "TRAIN_3723_QUERY_3", "TRAIN_1022_QUERY_3", "TRAIN_1022_QUERY_4", "TRAIN_1022_QUERY_1", "TRAIN_1022_QUERY_0", "TRAIN_1022_QUERY_2", "TRAIN_1133_QUERY_3", "TRAIN_1133_QUERY_1", "TRAIN_1133_QUERY_4", "TRAIN_1133_QUERY_0", "TRAIN_1060_QUERY_4", "TRAIN_1060_QUERY_3", "TRAIN_1060_QUERY_0", "TRAIN_1060_QUERY_2", "TRAIN_1060_QUERY_1", "TRAIN_118_QUERY_2", "TRAIN_118_QUERY_0", "TRAIN_3007_QUERY_1", "TRAIN_3007_QUERY_0", "TRAIN_3007_QUERY_2", "TRAIN_537_QUERY_3", "TRAIN_537_QUERY_4", "TRAIN_537_QUERY_0", "TRAIN_537_QUERY_2", "TRAIN_537_QUERY_1", "TRAIN_1141_QUERY_1", "TRAIN_1141_QUERY_2", "TRAIN_1141_QUERY_0", "TRAIN_1141_QUERY_3", "TRAIN_2434_QUERY_1", "TRAIN_2434_QUERY_2", "TRAIN_2434_QUERY_3", "TRAIN_2434_QUERY_4", "TRAIN_2434_QUERY_0", "TRAIN_3767_QUERY_3", "TRAIN_3767_QUERY_2", "TRAIN_3767_QUERY_0", "TRAIN_1289_QUERY_2", "TRAIN_1289_QUERY_0", "TRAIN_1289_QUERY_1", "TRAIN_1289_QUERY_4", "TRAIN_1289_QUERY_3", "TRAIN_710_QUERY_3", "TRAIN_710_QUERY_1", "TRAIN_710_QUERY_2", "TRAIN_710_QUERY_4", "TRAIN_710_QUERY_0", "TRAIN_3727_QUERY_0", "TRAIN_3727_QUERY_1", "TRAIN_3727_QUERY_3", "TRAIN_3727_QUERY_4", "TRAIN_3727_QUERY_2", "TRAIN_2042_QUERY_0", "TRAIN_2042_QUERY_4", "TRAIN_2042_QUERY_1", "TRAIN_2042_QUERY_3", "TRAIN_1182_QUERY_0", "TRAIN_180_QUERY_1", "TRAIN_180_QUERY_2", "TRAIN_180_QUERY_4", "TRAIN_2257_QUERY_2", "TRAIN_2257_QUERY_1", "TRAIN_2257_QUERY_3", "TRAIN_2257_QUERY_0", "TRAIN_3783_QUERY_3", "TRAIN_3783_QUERY_0", "TRAIN_3783_QUERY_2", "TRAIN_3783_QUERY_1", "TRAIN_544_QUERY_0", "TRAIN_544_QUERY_3", "TRAIN_544_QUERY_1", "TRAIN_544_QUERY_4", "TRAIN_2295_QUERY_3", "TRAIN_2295_QUERY_1", "TRAIN_2295_QUERY_0", "TRAIN_2295_QUERY_2", "TRAIN_565_QUERY_3", "TRAIN_565_QUERY_1", "TRAIN_565_QUERY_0", "TRAIN_657_QUERY_2", "TRAIN_657_QUERY_1", "TRAIN_657_QUERY_0", "TRAIN_1273_QUERY_1", "TRAIN_1273_QUERY_0", "TRAIN_2021_QUERY_2", "TRAIN_2021_QUERY_0", "TRAIN_2021_QUERY_1", "TRAIN_2021_QUERY_3", "TRAIN_2021_QUERY_4", "TRAIN_2278_QUERY_1", "TRAIN_2278_QUERY_0", "TRAIN_2278_QUERY_2", "TRAIN_2278_QUERY_3", "TRAIN_2278_QUERY_4", "TRAIN_1050_QUERY_1", "TRAIN_1050_QUERY_0", "TRAIN_1050_QUERY_2", "TRAIN_1050_QUERY_3", "TRAIN_1050_QUERY_4", "TRAIN_722_QUERY_2", "TRAIN_722_QUERY_0", "TRAIN_722_QUERY_1", "TRAIN_3079_QUERY_0", "TRAIN_3079_QUERY_1", "TRAIN_3778_QUERY_0", "TRAIN_3778_QUERY_1", "TRAIN_2076_QUERY_3", "TRAIN_2076_QUERY_0", "TRAIN_2076_QUERY_2", "TRAIN_2076_QUERY_4", "TRAIN_2076_QUERY_1", "TRAIN_2020_QUERY_2", "TRAIN_2020_QUERY_0", "TRAIN_2020_QUERY_3", "TRAIN_2426_QUERY_3", "TRAIN_2426_QUERY_4", "TRAIN_2426_QUERY_1", "TRAIN_2426_QUERY_2", "TRAIN_2426_QUERY_0", "TRAIN_714_QUERY_0", "TRAIN_714_QUERY_2", "TRAIN_714_QUERY_1", "TRAIN_714_QUERY_4", "TRAIN_714_QUERY_3", "TRAIN_332_QUERY_0" ], "sha256": "6ac6e21bdfa5de6307d3ad8d016f63d7f0633c2efaea1fc777c7549bfb32630f" }, "heldout": { "ids": [ "DEV_479_QUERY_0", "DEV_430_QUERY_2", "DEV_430_QUERY_1", "DEV_430_QUERY_0", "DEV_1037_QUERY_2", "DEV_1037_QUERY_0", "DEV_1037_QUERY_1", "DEV_398_QUERY_1", "DEV_398_QUERY_2", "DEV_398_QUERY_0", "DEV_1796_QUERY_0", "DEV_1796_QUERY_3", "DEV_1796_QUERY_1", "DEV_1796_QUERY_2", "DEV_1152_QUERY_0", "DEV_1152_QUERY_1", "DEV_1530_QUERY_1", "DEV_1530_QUERY_3", "DEV_1530_QUERY_0", "DEV_1530_QUERY_2", "DEV_247_QUERY_2", "DEV_247_QUERY_1", "DEV_247_QUERY_0", "DEV_271_QUERY_0", "DEV_1089_QUERY_2", "DEV_1089_QUERY_1", "DEV_1089_QUERY_0", "DEV_204_QUERY_1", "DEV_204_QUERY_0", "DEV_1501_QUERY_1", "DEV_1501_QUERY_4", "DEV_1501_QUERY_0", "DEV_1501_QUERY_3", "DEV_1501_QUERY_2", "DEV_634_QUERY_3", "DEV_634_QUERY_2", "DEV_634_QUERY_1", "DEV_634_QUERY_4", "DEV_634_QUERY_0", "DEV_1503_QUERY_1", "DEV_1503_QUERY_3", "DEV_1503_QUERY_2", "DEV_1503_QUERY_0", "DEV_1060_QUERY_3", "DEV_1060_QUERY_2", "DEV_1060_QUERY_1", "DEV_1060_QUERY_0", "DEV_145_QUERY_0", "DEV_145_QUERY_1", "DEV_1118_QUERY_1", "DEV_1118_QUERY_3", "DEV_1118_QUERY_0", "DEV_1118_QUERY_2", "DEV_502_QUERY_1", "DEV_502_QUERY_3", "DEV_502_QUERY_0", "DEV_502_QUERY_2", "DEV_502_QUERY_4", "DEV_393_QUERY_2", "DEV_393_QUERY_3", "DEV_393_QUERY_4", "DEV_393_QUERY_1", "DEV_393_QUERY_0", "DEV_478_QUERY_2", "DEV_478_QUERY_1", "DEV_478_QUERY_0", "DEV_282_QUERY_3", "DEV_282_QUERY_2", "DEV_282_QUERY_1", "DEV_282_QUERY_0", "DEV_300_QUERY_2", "DEV_300_QUERY_0", "DEV_300_QUERY_1", "DEV_1174_QUERY_3", "DEV_1174_QUERY_4", "DEV_1174_QUERY_2", "DEV_1174_QUERY_0", "DEV_1174_QUERY_1", "DEV_61_QUERY_3", "DEV_61_QUERY_1", "DEV_61_QUERY_0", "DEV_61_QUERY_4", "DEV_61_QUERY_2", "DEV_1933_QUERY_2", "DEV_1933_QUERY_0", "DEV_1933_QUERY_3", "DEV_1933_QUERY_4", "DEV_1933_QUERY_1", "DEV_320_QUERY_2", "DEV_320_QUERY_1", "DEV_320_QUERY_0", "DEV_1672_QUERY_1", "DEV_1672_QUERY_3", "DEV_1672_QUERY_2", "DEV_1672_QUERY_0", "DEV_569_QUERY_3", "DEV_569_QUERY_0", "DEV_569_QUERY_2", "DEV_569_QUERY_1", "DEV_303_QUERY_1", "DEV_1162_QUERY_1", "DEV_1162_QUERY_0", "DEV_1162_QUERY_2", "DEV_1078_QUERY_3", "DEV_1078_QUERY_2", "DEV_1078_QUERY_1", "DEV_1078_QUERY_4", "DEV_1078_QUERY_0", "DEV_6_QUERY_3", "DEV_6_QUERY_0", "DEV_6_QUERY_2", "DEV_6_QUERY_1", "DEV_1571_QUERY_3", "DEV_408_QUERY_1", "DEV_330_QUERY_3", "DEV_330_QUERY_0", "DEV_330_QUERY_2", "DEV_330_QUERY_1", "DEV_1167_QUERY_2", "DEV_1167_QUERY_1", "DEV_1167_QUERY_3", "DEV_1167_QUERY_0", "DEV_1167_QUERY_4", "DEV_510_QUERY_1", "DEV_510_QUERY_0", "DEV_510_QUERY_4", "DEV_510_QUERY_2", "DEV_510_QUERY_3", "DEV_380_QUERY_0", "DEV_380_QUERY_1", "DEV_380_QUERY_2", "DEV_380_QUERY_3", "DEV_341_QUERY_4", "DEV_341_QUERY_1", "DEV_341_QUERY_2", "DEV_341_QUERY_3", "DEV_51_QUERY_1", "DEV_51_QUERY_3", "DEV_51_QUERY_2", "DEV_51_QUERY_0", "DEV_163_QUERY_0", "DEV_163_QUERY_4", "DEV_163_QUERY_1", "DEV_163_QUERY_2", "DEV_163_QUERY_3", "DEV_108_QUERY_0", "DEV_108_QUERY_1", "DEV_108_QUERY_3", "DEV_1651_QUERY_0", "DEV_1651_QUERY_2", "DEV_1651_QUERY_4", "DEV_1651_QUERY_3", "DEV_1651_QUERY_1", "DEV_1088_QUERY_3", "DEV_1088_QUERY_0", "DEV_1088_QUERY_1", "DEV_1088_QUERY_2", "DEV_298_QUERY_0", "DEV_298_QUERY_1", "DEV_298_QUERY_3", "DEV_578_QUERY_0", "DEV_578_QUERY_2", "DEV_578_QUERY_1", "DEV_1113_QUERY_1", "DEV_1113_QUERY_0", "DEV_1113_QUERY_2", "DEV_115_QUERY_2", "DEV_115_QUERY_0", "DEV_115_QUERY_3", "DEV_115_QUERY_1", "DEV_1657_QUERY_0", "DEV_1657_QUERY_2", "DEV_1657_QUERY_1", "DEV_1657_QUERY_4", "DEV_1657_QUERY_3", "DEV_216_QUERY_2", "DEV_216_QUERY_0", "DEV_216_QUERY_1", "DEV_319_QUERY_1", "DEV_319_QUERY_0", "DEV_322_QUERY_0", "DEV_322_QUERY_1", "DEV_1771_QUERY_2", "DEV_1771_QUERY_0", "DEV_1771_QUERY_3", "DEV_1771_QUERY_4", "DEV_1771_QUERY_1", "DEV_424_QUERY_0", "DEV_424_QUERY_2", "DEV_424_QUERY_1", "DEV_1889_QUERY_2", "DEV_1889_QUERY_1", "DEV_1889_QUERY_0", "DEV_246_QUERY_2", "DEV_246_QUERY_1", "DEV_246_QUERY_0", "DEV_18_QUERY_0", "DEV_18_QUERY_1", "DEV_18_QUERY_4", "DEV_18_QUERY_3", "DEV_18_QUERY_2", "DEV_1831_QUERY_0", "DEV_1831_QUERY_3", "DEV_1831_QUERY_1", "DEV_1831_QUERY_2", "DEV_156_QUERY_1", "DEV_156_QUERY_3", "DEV_156_QUERY_2", "DEV_156_QUERY_0", "DEV_1148_QUERY_1", "DEV_1148_QUERY_0", "DEV_1148_QUERY_2", "DEV_183_QUERY_0", "DEV_183_QUERY_1", "DEV_183_QUERY_3", "DEV_183_QUERY_2", "DEV_469_QUERY_2", "DEV_469_QUERY_3", "DEV_469_QUERY_0", "DEV_1853_QUERY_1", "DEV_1853_QUERY_0", "DEV_1853_QUERY_3", "DEV_1853_QUERY_2", "DEV_470_QUERY_1", "DEV_470_QUERY_2", "DEV_470_QUERY_0", "DEV_557_QUERY_3", "DEV_557_QUERY_2", "DEV_557_QUERY_1", "DEV_557_QUERY_0", "DEV_557_QUERY_4", "DEV_274_QUERY_2", "DEV_274_QUERY_0", "DEV_205_QUERY_0", "DEV_205_QUERY_1", "DEV_205_QUERY_2", "DEV_15_QUERY_2", "DEV_15_QUERY_0", "DEV_1644_QUERY_1", "DEV_1644_QUERY_2", "DEV_1644_QUERY_0", "DEV_1644_QUERY_3", "DEV_1033_QUERY_0", "DEV_1033_QUERY_2", "DEV_1033_QUERY_3", "DEV_1033_QUERY_1", "DEV_1001_QUERY_2", "DEV_1001_QUERY_0", "DEV_1001_QUERY_1", "DEV_444_QUERY_0", "DEV_444_QUERY_1", "DEV_444_QUERY_2", "DEV_473_QUERY_3", "DEV_473_QUERY_1", "DEV_473_QUERY_2", "DEV_473_QUERY_0" ], "sha256": "59487b576df9f4598a5c6e212a9a466c47c372613c31a9211a027a0811d76183" } }, "tokenizer": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "template_kwargs": { "enable_thinking": false }, "limits": { "complete_chat_tokens": 384, "answer_tokens": 32 }, "seed_selection": "none" }, "prediction_csv_sha256": "e2c772e5978c2f83157c6fa3f11b8131c5fb0604a4e60920db75d90c0842a31f" } }, "runs": [ { "language": "squad", "method": "base", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 0, "steps": 0, "heldout_count": 256, "em": 55.078125, "f1": 62.562165105031, "generation_limit_count": 1, "adapter_payload_bytes": 0, "final_train_loss_diagnostic": null, "data_cursor": null, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "torch", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 54.6875, "f1": 62.25025363969871, "generation_limit_count": 0, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.3116177022457123, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359104, "target_tokens_seen": 9611 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "torch", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 55.859375, "f1": 63.03381040788524, "generation_limit_count": 0, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.47802096605300903, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359395, "target_tokens_seen": 9630 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "torch", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 56.25, "f1": 62.00602565836941, "generation_limit_count": 1, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.4548304080963135, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359502, "target_tokens_seen": 9681 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "peft", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 58.203125, "f1": 65.63052250655048, "generation_limit_count": 0, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.31230539083480835, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359104, "target_tokens_seen": 9611 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "peft", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 57.421875, "f1": 64.38982310178855, "generation_limit_count": 1, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.3682354986667633, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359395, "target_tokens_seen": 9630 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "peft", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 54.6875, "f1": 62.63932425168034, "generation_limit_count": 0, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.4091910719871521, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359502, "target_tokens_seen": 9681 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "arti", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 54.6875, "f1": 62.25025363969871, "generation_limit_count": 0, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.3116177022457123, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359104, "target_tokens_seen": 9611 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "arti", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 55.859375, "f1": 63.03381040788524, "generation_limit_count": 0, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.47802096605300903, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359395, "target_tokens_seen": 9630 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "squad", "method": "arti", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 56.25, "f1": 62.00602565836941, "generation_limit_count": 1, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.4548304080963135, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "1e98d8c65d69d510967d200584fe397b334612223dedd3e20955e898a7264492", "samples_seen": 1600, "input_tokens_seen": 359502, "target_tokens_seen": 9681 }, "data_manifest_sha256": "be7b547a59e9a92ea7bdd68e927861f3710fab53fbc48b3feee6bee71e3ecc65" }, { "language": "cmrc", "method": "base", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 0, "steps": 0, "heldout_count": 256, "em": 46.875, "f1": 71.70898113149433, "generation_limit_count": 2, "adapter_payload_bytes": 0, "final_train_loss_diagnostic": null, "data_cursor": null, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "torch", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.4375, "f1": 70.7328383664144, "generation_limit_count": 8, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.4160727858543396, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498504, "target_tokens_seen": 16632 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "torch", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.046875, "f1": 72.93247446487855, "generation_limit_count": 7, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.49134472012519836, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498978, "target_tokens_seen": 16751 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "torch", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.4375, "f1": 72.77529184443753, "generation_limit_count": 11, "adapter_payload_bytes": 524656, "final_train_loss_diagnostic": 0.5688204765319824, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 499036, "target_tokens_seen": 16781 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "peft", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.828125, "f1": 71.60003757660152, "generation_limit_count": 5, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.38149580359458923, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498504, "target_tokens_seen": 16632 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "peft", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 46.875, "f1": 71.56712124183932, "generation_limit_count": 5, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.48476502299308777, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498978, "target_tokens_seen": 16751 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "peft", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 44.921875, "f1": 69.15763865016342, "generation_limit_count": 9, "adapter_payload_bytes": 524848, "final_train_loss_diagnostic": 0.5222938656806946, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 499036, "target_tokens_seen": 16781 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "arti", "seed": 17, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.4375, "f1": 70.7328383664144, "generation_limit_count": 8, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.4160727858543396, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 17, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498504, "target_tokens_seen": 16632 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "arti", "seed": 29, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.046875, "f1": 72.93247446487855, "generation_limit_count": 7, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.49134472012519836, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 29, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 498978, "target_tokens_seen": 16751 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" }, { "language": "cmrc", "method": "arti", "seed": 43, "base": "Qwen/Qwen3-0.6B", "revision": "c1899de289a04d12100db370d81485cdf75e47ca", "trainable_parameters": 131072, "steps": 200, "heldout_count": 256, "em": 48.4375, "f1": 72.77529184443753, "generation_limit_count": 11, "adapter_payload_bytes": 558472, "final_train_loss_diagnostic": 0.5688204765319824, "data_cursor": { "epoch": 3, "cursor": 64, "seed": 43, "batch_size": 8, "group_by_length": true, "split_sha256": "cd8339cd32a53e1cc660720190682c140132344e6aae5fa669dca312d7e44260", "samples_seen": 1600, "input_tokens_seen": 499036, "target_tokens_seen": 16781 }, "data_manifest_sha256": "94eccf2f086573c8da74ceaab505b97f0a9ef7619f49431a4a3f45622adab35d" } ], "sources": { "squad-train.json": { "url": "https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/eee5fdbf62f8613a7812b03419e6b29617b74fd1/dataset/train-v1.1.json", "sha256": "3527663986b8295af4f7fcdff1ba1ff3f72d07d61a20f487cb238a6ef92fd955", "bytes": 30288272 }, "squad-dev.json": { "url": "https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/eee5fdbf62f8613a7812b03419e6b29617b74fd1/dataset/dev-v1.1.json", "sha256": "95aa6a52d5d6a735563366753ca50492a658031da74f301ac5238b03966972c9", "bytes": 4854279 }, "cmrc-train.json": { "url": "https://raw.githubusercontent.com/ymcui/cmrc2018/c0eb1b6ba219847457e6af3180da722bbeb656af/squad-style-data/cmrc2018_train.json", "sha256": "5497aa2f81908e31d6b0e27d99b1f90ab63a8f58fa92fffe5d17cf62eba0c212", "bytes": 7408757 }, "cmrc-dev.json": { "url": "https://raw.githubusercontent.com/ymcui/cmrc2018/c0eb1b6ba219847457e6af3180da722bbeb656af/squad-style-data/cmrc2018_dev.json", "sha256": "e9ff74231f05c230c6fa88b84441ee334d97234cbb610991cd94b82db00c7f1f", "bytes": 3299139 }, "squad_evaluate.py": { "url": "https://raw.githubusercontent.com/rajpurkar/SQuAD-explorer/eee5fdbf62f8613a7812b03419e6b29617b74fd1/evaluate-v2.0.py", "sha256": "710840ce2c2c61716334c056777032f24660b83eb9869a451036e6a24d8103c4", "bytes": 10547 }, "cmrc_evaluate_v6.py": { "url": "https://raw.githubusercontent.com/ymcui/cmrc2018/c0eb1b6ba219847457e6af3180da722bbeb656af/baseline/cmrc2018_evaluate.py", "sha256": "c14090c77e22531e58b8170952b74d6ebe6e11f4a605e7feba6e56a7caa2735c", "bytes": 4362 } }, "execution_input": { "harness_sha256": "f79a6700e2e576760715639f04215779c120bb2fb4aa5382cccccfc5fbbac34b", "base_revision": "c1899de289a04d12100db370d81485cdf75e47ca", "qualification_sha256": "3bdc0fe2cb17e39487de8dcaf08c8dd38fc9f5fa7c374448742d497f5c534e81", "seeds": [ 17, 29, 43 ], "measurement_harness_sha256": "b1ef45d23c3edbf48a85fa727b2ae4999a0dcf0abb43851c0d961b329344d931", "reproduction_edit": "Public reproduction script removes host metadata and the developer-environment header. Computation, data, optimization and generation code are unchanged." }, "lifecycle": { "cold_reload": true, "resume": true, "reloaded_ids": 512, "resumed_steps": 3, "scope": "both languages, seed 17; separate process; continuation diagnostics excluded from effect scores", "receipts": { "squad": { "cold_reload_answers": 256, "cold_reload_exact": true, "model_manifest_sha256": "089dda57a2a0d86f02960859e119716a03508beddc811fb48a4b7d74a0f8b8c7", "continuation_steps": 3, "checkpoint_step": 200, "final_step": 203, "parameter_max_abs": 0.0, "data_cursor_identical": true }, "cmrc": { "cold_reload_answers": 256, "cold_reload_exact": true, "model_manifest_sha256": "d8c6345f42a6402961ab22a16ee6ae2c804119cfdd00a485df3e0001cc1b966b", "continuation_steps": 3, "checkpoint_step": 200, "final_step": 203, "parameter_max_abs": 0.0, "data_cursor_identical": true } } }, "qualification": { "layer": { "identity": true, "output_max_abs": 0.0, "input_gradient_max_abs": 0.0, "down_gradient_max_abs": 0.0, "up_gradient_max_abs": 0.0 }, "parameters": { "arti": 131072, "torch": 131072, "peft": 131072 }, "peft_shapes": [ { "name": "base_model.model.model.layers.26.self_attn.v_proj.lora_A.default.weight", "shape": [ 32, 1024 ], "dtype": "torch.float32" }, { "name": "base_model.model.model.layers.26.self_attn.v_proj.lora_B.default.weight", "shape": [ 1024, 32 ], "dtype": "torch.float32" }, { "name": "base_model.model.model.layers.27.self_attn.v_proj.lora_A.default.weight", "shape": [ 32, 1024 ], "dtype": "torch.float32" }, { "name": "base_model.model.model.layers.27.self_attn.v_proj.lora_B.default.weight", "shape": [ 1024, 32 ], "dtype": "torch.float32" } ], "training_only_two_step_parameter_max_abs": 0.0, "arti_losses": [ 0.6599963903427124, 1.2166285514831543 ], "torch_losses": [ 0.6599963903427124, 1.2166285514831543 ], "identical_data_cursor": true, "heldout_examined": false }, "publication_scope": "Model/data/evaluation protocol, answers, scores and artifact size. Developer host information and local performance telemetry are excluded.", "nltk_english_tokenizer_files": { "abbrev_types.txt": "92a3e070f43d9b4c5534758ca40ad7343b04e7e29bfe0c2eb658a39445a4f779", "collocations.tab": "8e2da1225e4dd2cc9dba261ee231ccb134859e21b46006e7f472c5ee269af0cf", "ortho_context.tab": "4bbcca25ed3d3f06c02402abf8419b9f033b8adc06e7b482eca4e45f81a5dc4c", "sent_starters.txt": "f3f8535483e1dba487241b764945168123bca3209a9645e59acd1225dc76edac" }, "onboarding_prediction_checks": { "squad": { "text": "10", "token_ids": [ 16, 15, 151645 ], "generated_tokens": 3, "finish_reason": "stop", "id": "my-question-001" }, "cmrc": { "text": "十点", "token_ids": [ 94498, 27442, 151645 ], "generated_tokens": 3, "finish_reason": "stop", "id": "my-question-001" }, "scope": "separate self-authored library-opening inputs; not part of effect benchmark" }, "comparison_scope": "Formula-choice experiment: SiLU residual after MLP versus Linear LoRA on v_proj. Equal parameter count does not align formula or mounting. Only the SiLU ARTI/PyTorch pair is implementation-matched." }