{ "format": "compact-retrieval-1", "methods": { "keyword": { "label": "关键词匹配", "metadata": "原始四字段，按字段加权命中", "algorithm": "不同查询词命中数×字段权重", "short": "关键词" }, "name_description": { "label": "BM25 · name＋description", "metadata": "名称与描述拼成一份文档", "algorithm": "单文档BM25，无字段权重／Schema／别名", "short": "BM25名称＋描述" }, "bm25": { "label": "BM25 · Anan当前版", "metadata": "名称／标识、描述、Schema、已有原生别名", "algorithm": "分字段加权BM25，c586608", "short": "BM25当前版" }, "combined": { "label": "BM25 · 最终候选版", "metadata": "当前版＋直接工具别名＋操作同义词", "algorithm": "相同分字段BM25，仅扩充检索词表；未移植Anan", "short": "BM25候选版" } }, "rows": [ { "query_id": "N001", "query": "ListFiles", "category": "exact", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ListFiles", "apple/reminders.list", "builtin/EditFile", "builtin/WriteFile", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.list", "apple/photos.albums.list", "apple/reminders.lists", "builtin/ReadFile", "apple/health.workouts.list", "builtin/Memory", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/photos.search", "apple/reminders.create", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/alarms.create", "apple/calendar.events.get", "apple/clipboard.read", "apple/calendar.events.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get" ], "scores": [ 7, 4.5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 4, 3, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ListFiles", "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/reminders.lists", "apple/calendar.list", "apple/clipboard.read", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/photos.search", "apple/alarms.list", "apple/calendar.events.get", "apple/alarms.create", "apple/photos.albums.list", "apple/health.workouts.list", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "apple/shortcuts.run", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 5.043838265675319, 4.639219693206515, 3.714454817908332, 3.472413797389085, 3.374072336740228, 2.1668991632120185, 1.970426045916842, 1.931435403678754, 1.614897263876673, 1.4924909939710067, 1.4924909939710067, 1.4705301552404113, 1.4702032485297019, 1.457997372644377, 1.3776271994526728, 1.340122705366102, 1.3390180568934824, 1.2865248727729224, 1.1248369805109821, 1.0415379820610877, 0.9793722303182939, 0.9242094098357349, 0.899392122924025 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/EditFile", "builtin/ReadFile", "builtin/Memory", "apple/reminders.lists", "apple/reminders.list", "apple/calendar.list", "apple/contacts.list", "apple/alarms.list", "apple/calendar.events.list", "apple/photos.albums.list", "apple/health.workouts.list", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/alarms.create", "apple/calendar.events.update", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 9.643468258788317, 10.321912047288809, 9.259243366443126, 8.549670258073231, 5.45328318424094, 4.1228942449914925, 3.8296680656769464, 3.8153957247318555, 3.344633169936297, 3.3265336457761996, 3.1661756288646488, 2.9379072248550164, 2.2499293125845314, 1.8024824369174723, 1.6515435525390731, 1.4901550213895047, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.0204469850336544, 0.9781948581756706, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "apple/reminders.lists", "apple/reminders.list", "apple/calendar.list", "apple/contacts.list", "apple/alarms.list", "apple/calendar.events.list", "apple/photos.albums.list", "apple/health.workouts.list", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/alarms.create", "apple/calendar.events.update", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 10.843591632532075, 12.982913464988664, 11.138827250019764, 9.259243366443126, 5.45328318424094, 4.1228942449914925, 3.8296680656769464, 3.8153957247318555, 3.344633169936297, 3.3265336457761996, 3.1661756288646488, 2.9379072248550164, 2.2499293125845314, 1.8024824369174723, 1.6515435525390731, 1.4901550213895047, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.0204469850336544, 0.9781948581756706, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N002", "query": "ReadFile", "category": "exact", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "builtin/EditFile", "apple/clipboard.read", "builtin/WriteFile", "builtin/ListFiles", "builtin/Memory", "apple/photos.search", "builtin/ScheduledTasks", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.list", "apple/contacts.get", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.eta", "apple/photos.get", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/shortcuts.run" ], "scores": [ 8, 5.5, 5, 4.5, 3, 2.5, 2, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ReadFile", "builtin/Memory", "builtin/EditFile", "builtin/WriteFile", "apple/clipboard.read", "builtin/ListFiles", "apple/photos.search", "apple/contacts.get", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/calendar.events.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "builtin/ScheduledTasks", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 5.20810680629957, 5.095309287670941, 4.7373151115820535, 3.714454817908332, 3.2754104614901403, 3.053494701017844, 2.2055776060257926, 1.596129569605437, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.0956951902365162, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 1.033343757933971, 0.9551091781400716, 0.8975292682836623, 0.8464971647744733, 0.7868454382417407, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ReadFile", "builtin/EditFile", "builtin/WriteFile", "builtin/Memory", "apple/clipboard.read", "builtin/ListFiles", "apple/photos.search", "apple/contacts.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/weather.current", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/health.sleep.query", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/shortcuts.run", "apple/health.statistics" ], "scores": [ 13.10696022217413, 10.649898971965822, 10.321912047288809, 5.955715380754048, 5.857157575876017, 5.804260981819989, 2.0745172484576795, 1.60914410629741, 1.3042089742613945, 1.2461128550311649, 1.231374918430577, 1.1490595548480602, 1.113807122507814, 1.0992184654999453, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 0.9499145762410579, 0.8895050372184048, 0.8363195411534453, 0.7745686203791382, 0.7660997030790287, 0.7031782969374535 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/contacts.get", "apple/photos.search", "apple/health.workouts.get", "apple/device.get", "apple/battery.get", "apple/health.workouts.list", "apple/calendar.events.list", "apple/alarms.list", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.list", "apple/reminders.list", "apple/weather.current", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/shortcuts.run", "apple/maps.eta", "apple/weather.hourly", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 15.340605499914341, 12.982913464988664, 10.24338125913383, 5.804260981819989, 5.484745123318538, 4.67189124737578, 2.1425381623194113, 1.8636633604573114, 1.8264601231288218, 1.8002653409399807, 1.7904649384226818, 1.7223681387479477, 1.6917167140453433, 1.648914120016547, 1.6332646411623757, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.8714182810782765, 0.8131654195283325, 0.7778939227999769, 0.7660997030790287, 0.7580363862218745, 0.5918458513942811, 0.5481460160064278, 0.4976245769158341 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N003", "query": "WriteFile", "category": "exact", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/WriteFile" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/EditFile", "builtin/ReadFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks", "apple/shortcuts.run" ], "scores": [ 7.5, 4.5, 4.5, 4, 3, 2, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/Memory", "builtin/ReadFile", "builtin/EditFile", "builtin/ListFiles", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 7.594051058282012, 3.7936376826106004, 3.4744596028470913, 3.472413797389085, 3.374072336740228, 3.053494701017844, 1.931435403678754, 1.4705301552404113, 1.3804900280474186 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/EditFile", "builtin/ReadFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks", "apple/shortcuts.run" ], "scores": [ 18.084202584958113, 11.754253707105612, 9.259243366443126, 8.549670258073231, 5.804260981819989, 5.105460677068182, 1.8024824369174723, 1.3532076178115147, 1.214637928883098, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/ReadFile", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks", "apple/shortcuts.run" ], "scores": [ 24.303876879365774, 11.754253707105612, 11.138827250019764, 9.259243366443126, 5.804260981819989, 5.105460677068182, 1.8024824369174723, 1.3532076178115147, 1.214637928883098, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N004", "query": "EditFile", "category": "exact", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/EditFile" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 7.5, 4.5, 4, 3, 3, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/EditFile", "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "apple/clipboard.read", "apple/photos.search" ], "scores": [ 7.591697808911764, 6.744026576792688, 3.714454817908332, 3.472413797389085, 3.053494701017844, 1.931435403678754, 1.4705301552404113 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/Memory", "builtin/ReadFile", "builtin/ListFiles", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 18.831162819962692, 10.321912047288809, 9.393585288475634, 8.549670258073231, 5.804260981819989, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "builtin/Memory", "builtin/ListFiles", "apple/contacts.update", "apple/calendar.events.update", "apple/reminders.update", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 19.59072667344976, 12.982913464988664, 11.138827250019764, 8.113156886048474, 5.804260981819989, 2.9482051569982852, 2.876364970293838, 2.807942621296027, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N005", "query": "Memory", "category": "exact", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/Memory" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 4.5, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 4.4242782079522565, 1.5480217516112227 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 15.187094551046085, 1.4978074273357158 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 20.995734892602883, 1.4978074273357158 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N006", "query": "GetCurrentTime", "category": "exact", "domain": "clock", "intent": "capability:builtin/GetCurrentTime", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/GetCurrentTime" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/GetCurrentTime" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/GetCurrentTime", "apple/location.get", "apple/calendar.events.get", "apple/reminders.get", "apple/weather.current", "apple/battery.get", "apple/contacts.get", "apple/device.get", "apple/health.workouts.get", "apple/photos.get", "builtin/ScheduledTasks", "apple/calendar.events.update", "apple/maps.eta", "apple/maps.route", "apple/music.status", "apple/weather.hourly", "apple/alarms.create", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/health.statistics", "apple/reminders.list", "builtin/EditFile", "apple/alarms.cancel", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/maps.show", "apple/music.playback", "apple/weather.daily", "apple/calendar.events.create", "apple/health.sleep.query", "apple/health.workouts.list", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 12, 5, 4, 4, 4, 3, 3, 3, 3, 3, 2.5, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "apple/location.get", "apple/maps.eta", "apple/music.status", "apple/maps.route", "apple/reminders.get", "apple/weather.hourly", "apple/calendar.events.get", "apple/health.workouts.get", "apple/contacts.get", "builtin/EditFile", "apple/calendar.events.list", "apple/weather.current", "apple/location.reverseGeocode", "apple/reminders.list", "apple/device.get", "apple/battery.get", "apple/alarms.cancel", "apple/weather.daily", "apple/calendar.events.delete", "apple/location.geocode", "apple/alarms.create", "apple/photos.get", "apple/maps.show", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/maps.search", "apple/health.statistics", "apple/music.playback" ], "scores": [ 7.641927157355829, 4.589593331797433, 3.8393070504842095, 3.107819213045363, 2.875211078836437, 2.6948504816329724, 2.548251113137415, 2.4701324863933865, 1.8901006600625418, 1.863675018861272, 1.8329870145700418, 1.8251756430963588, 1.668971403398956, 1.6335467459150792, 1.6257579312913266, 1.615261053678907, 1.5959224584056921, 1.5741603564255942, 1.5605468552419268, 1.5229870523435276, 1.5102993475823525, 1.4513293217173118, 1.3393568262008189, 1.2842193286009138, 1.251344851939756, 1.2177431769453415, 1.2067986160068875, 0.9645187186498915, 0.9418286545359099 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/GetCurrentTime", "apple/location.get", "apple/reminders.get", "apple/calendar.events.get", "apple/weather.current", "apple/battery.get", "apple/contacts.get", "apple/device.get", "apple/photos.get", "apple/health.workouts.get", "apple/maps.eta", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.status", "apple/calendar.events.update", "builtin/EditFile", "apple/maps.route", "apple/weather.hourly", "apple/calendar.events.list", "apple/location.reverseGeocode", "apple/alarms.create", "apple/health.statistics", "apple/reminders.list", "apple/weather.daily", "apple/location.geocode", "apple/maps.show", "apple/maps.search", "apple/alarms.cancel", "apple/music.playback", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/health.workouts.list", "apple/calendar.events.create" ], "scores": [ 16.16532883992373, 7.840370058970693, 6.964452568757729, 6.122855657631305, 5.045382794218522, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.081448291306185, 2.9597062323693777, 2.875549410042651, 2.571500048869087, 2.455716218027152, 2.363997637693785, 2.3540016180843235, 2.2611434407596116, 1.9907457914176865, 1.8701253527179627, 1.592799090577726, 1.578316724633953, 1.534210625705793, 1.5142936950371024, 1.4663815769404227, 1.4428330874025772, 1.2141942578109772, 1.1369779299085616, 1.0352314574515493, 0.8767732783360794, 0.8668821733731853, 0.8668821733731853, 0.6727985360982812, 0.6215769746195708, 0.6148894289223729 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/GetCurrentTime", "apple/location.get", "apple/reminders.get", "apple/calendar.events.get", "apple/weather.current", "apple/battery.get", "apple/contacts.get", "apple/device.get", "apple/photos.get", "apple/health.workouts.get", "apple/maps.eta", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.status", "apple/calendar.events.update", "builtin/EditFile", "apple/maps.route", "apple/weather.hourly", "apple/calendar.events.list", "apple/location.reverseGeocode", "apple/alarms.create", "apple/health.statistics", "apple/reminders.list", "apple/weather.daily", "apple/location.geocode", "apple/maps.show", "apple/maps.search", "apple/alarms.cancel", "apple/music.playback", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/health.workouts.list", "apple/calendar.events.create" ], "scores": [ 19.638028345025088, 7.840370058970693, 6.964452568757729, 6.122855657631305, 5.045382794218522, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.081448291306185, 2.9597062323693777, 2.875549410042651, 2.571500048869087, 2.455716218027152, 2.363997637693785, 2.3540016180843235, 2.2611434407596116, 1.9907457914176865, 1.8701253527179627, 1.592799090577726, 1.578316724633953, 1.534210625705793, 1.5142936950371024, 1.4663815769404227, 1.4428330874025772, 1.2141942578109772, 1.1369779299085616, 1.0352314574515493, 0.8767732783360794, 0.8668821733731853, 0.8668821733731853, 0.6727985360982812, 0.6215769746195708, 0.6148894289223729 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N007", "query": "AskUserQuestions", "category": "exact", "domain": "questions", "intent": "capability:builtin/AskUserQuestions", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/device.get", "apple/maps.show", "apple/shortcuts.run", "builtin/Memory" ], "scores": [ 12.5, 1.5, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/AskUserQuestions", "apple/shortcuts.run", "apple/device.get", "builtin/Memory", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel", "builtin/ScheduledTasks" ], "scores": [ 13.013628670499601, 2.7491696581568283, 2.165477147820978, 2.1142385633003573, 2.041771298060864, 1.9107839181010244, 1.8905693842378057, 1.001015519842559 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/AskUserQuestions", "apple/shortcuts.run", "apple/device.get", "builtin/Memory", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel", "builtin/ScheduledTasks" ], "scores": [ 37.14831232603789, 2.7231044752292686, 2.1706565447392974, 2.089608949221399, 2.0639215021837605, 1.9003916383525448, 1.8791219646015662, 1.2466794473988847 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/AskUserQuestions", "apple/shortcuts.run", "apple/device.get", "builtin/Memory", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel", "builtin/ScheduledTasks" ], "scores": [ 47.20399338409034, 2.7231044752292686, 2.1706565447392974, 2.089608949221399, 2.0639215021837605, 1.9003916383525448, 1.8791219646015662, 1.2466794473988847 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N008", "query": "ScheduledTasks", "category": "exact", "domain": "scheduling", "intent": "capability:builtin/ScheduledTasks", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ScheduledTasks", "apple/alarms.list", "apple/links.open", "apple/location.get", "apple/music.playback", "apple/photos.albums.list", "apple/photos.get", "apple/photos.search", "apple/shortcuts.run", "apple/music.search", "apple/music.status" ], "scores": [ 8.5, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ScheduledTasks", "apple/location.get", "apple/alarms.list", "apple/photos.get", "apple/shortcuts.run", "apple/photos.albums.list", "apple/photos.search", "apple/music.playback", "apple/links.open", "apple/music.status", "apple/music.search" ], "scores": [ 3.603520181101086, 3.6159324379378806, 3.1906434839564177, 2.9982916896328513, 2.968465435887508, 2.8012675913097365, 2.455503949626172, 2.3399737043922757, 1.8943358780347852, 1.626248069953341, 1.41382735889783 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ScheduledTasks", "apple/location.get", "apple/alarms.list", "apple/photos.get", "apple/shortcuts.run", "apple/photos.albums.list", "apple/photos.search", "apple/music.playback", "apple/links.open", "apple/music.status", "apple/music.search" ], "scores": [ 13.74962010852568, 3.6245810396304687, 3.1732903655065345, 2.9714858949152445, 2.9403210853302486, 2.79381410582082, 2.409611304770311, 2.2914423939818986, 1.8402155394941357, 1.625108561440322, 1.4011872391657196 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ScheduledTasks", "apple/location.get", "apple/alarms.list", "apple/photos.get", "apple/shortcuts.run", "apple/photos.albums.list", "apple/photos.search", "apple/music.playback", "apple/links.open", "apple/music.status", "apple/music.search" ], "scores": [ 18.360044906243935, 3.6245810396304687, 3.1732903655065345, 2.9714858949152445, 2.9403210853302486, 2.79381410582082, 2.409611304770311, 2.2914423939818986, 1.8402155394941357, 1.625108561440322, 1.4011872391657196 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N009", "query": "calendar.events.create", "category": "exact", "domain": "calendar", "intent": "capability:apple/calendar.events.create", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "apple/calendar.events.create" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "apple/calendar.events.create" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/alarms.create", "apple/calendar.list", "apple/contacts.create", "apple/reminders.create", "builtin/ScheduledTasks", "apple/links.open", "builtin/Memory", "apple/health.statistics", "apple/weather.daily", "builtin/WriteFile" ], "scores": [ 13, 8.5, 8.5, 8.5, 7.5, 4, 4, 4, 4, 2.5, 2, 1.5, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.list", "apple/reminders.create", "apple/contacts.create", "builtin/WriteFile", "apple/alarms.create", "builtin/Memory", "builtin/ScheduledTasks", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 8.856548838247154, 7.4770854631840855, 6.689897641695674, 5.590861843841759, 5.5844602644808585, 3.353990223330057, 3.05666003761185, 3.030740607742593, 2.8131576240574634, 2.5006287346568747, 2.1142385633003573, 2.0672994037973105, 1.4761555867694751, 1.3457586502029633 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/reminders.create", "apple/contacts.create", "apple/alarms.create", "apple/calendar.list", "builtin/ScheduledTasks", "builtin/Memory", "builtin/WriteFile", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 24.65544433191423, 18.391891088889523, 16.547688866034505, 15.557210645961044, 15.255336227778646, 8.542801847641005, 8.510275502348785, 7.914298923247929, 7.725956296832728, 4.017130557062427, 3.0418207860050117, 2.8774990349848864, 1.290252793313428, 1.1713329690232368, 0.6658726147629157 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/reminders.create", "apple/contacts.create", "apple/alarms.create", "apple/calendar.list", "builtin/WriteFile", "builtin/ScheduledTasks", "builtin/Memory", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 24.65544433191423, 18.391891088889523, 16.547688866034505, 15.557210645961044, 15.255336227778646, 8.542801847641005, 8.510275502348785, 7.914298923247929, 7.725956296832728, 5.715162977290321, 4.017130557062427, 3.0418207860050117, 1.290252793313428, 1.1713329690232368, 1.142400969046904 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N010", "query": "contacts.get", "category": "exact", "domain": "contacts", "intent": "capability:apple/contacts.get", "target_capability": "Exact declared name identifies this tool.", "acceptable": [ "apple/contacts.get" ], "rationale": "Exact declared name identifies this tool.", "basis": [ "apple/contacts.get" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.get", "apple/contacts.delete", "apple/contacts.update", "apple/calendar.events.get", "apple/contacts.create", "apple/contacts.list", "apple/location.get", "apple/reminders.get", "builtin/GetCurrentTime", "apple/battery.get", "apple/device.get", "apple/health.workouts.get", "apple/photos.get", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/calendar.events.update" ], "scores": [ 7.5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 2, 1.5, 1.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.list", "apple/contacts.update", "apple/reminders.get", "apple/calendar.events.get", "builtin/GetCurrentTime", "apple/location.get", "apple/health.workouts.get", "apple/device.get", "apple/battery.get", "apple/calendar.events.delete", "apple/photos.get", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 5.9260007930349685, 3.908385007994482, 3.6211209622787783, 3.5908759744630157, 3.057540300459825, 2.6948504816329724, 2.4701324863933865, 2.447456010488996, 2.168723551118598, 1.8901006600625418, 1.615261053678907, 1.5959224584056921, 1.5229870523435276, 1.3393568262008189, 1.251344851939756, 1.2177431769453415 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/contacts.get", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.create", "apple/contacts.list", "apple/reminders.get", "apple/location.get", "apple/calendar.events.get", "builtin/GetCurrentTime", "apple/battery.get", "apple/device.get", "apple/photos.get", "apple/health.workouts.get", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/calendar.events.update" ], "scores": [ 16.580653360394834, 11.146694165037609, 10.06869788632105, 9.890981982747522, 9.52373522475088, 6.964452568757729, 6.236039709838479, 6.122855657631305, 6.090113235029695, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.081448291306185, 3.1528867180674975, 2.571500048869087, 1.7744787403473539 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/contacts.get", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.create", "apple/contacts.list", "apple/reminders.get", "apple/location.get", "apple/calendar.events.get", "builtin/GetCurrentTime", "apple/battery.get", "apple/device.get", "apple/photos.get", "apple/health.workouts.get", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/calendar.events.update" ], "scores": [ 16.580653360394834, 11.146694165037609, 10.06869788632105, 9.890981982747522, 9.52373522475088, 6.964452568757729, 6.236039709838479, 6.122855657631305, 6.090113235029695, 4.616915268299708, 4.616915268299708, 4.616915268299708, 4.081448291306185, 3.1528867180674975, 2.571500048869087, 1.7744787403473539 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N011", "query": "list directory files", "category": "phrase", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Lists one directory level, not text contents.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Lists one directory level, not text contents.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ListFiles", "apple/reminders.list", "builtin/EditFile", "builtin/WriteFile", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.list", "apple/photos.albums.list", "apple/reminders.lists", "builtin/Memory", "builtin/ReadFile", "apple/health.workouts.list", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/photos.search", "apple/reminders.create", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/alarms.create", "apple/calendar.events.get", "apple/clipboard.read", "apple/calendar.events.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get" ], "scores": [ 8, 4.5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ListFiles", "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/reminders.lists", "apple/calendar.list", "apple/clipboard.read", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/photos.search", "apple/alarms.list", "apple/calendar.events.get", "apple/alarms.create", "apple/photos.albums.list", "apple/health.workouts.list", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "apple/shortcuts.run", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 9.76591911448042, 7.908786667152112, 3.714454817908332, 3.472413797389085, 3.374072336740228, 2.1668991632120185, 1.970426045916842, 1.931435403678754, 1.614897263876673, 1.4924909939710067, 1.4924909939710067, 1.4705301552404113, 1.4702032485297019, 1.457997372644377, 1.3776271994526728, 1.340122705366102, 1.3390180568934824, 1.2865248727729224, 1.1248369805109821, 1.0415379820610877, 0.9793722303182939, 0.9242094098357349, 0.899392122924025 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/EditFile", "builtin/Memory", "builtin/ReadFile", "apple/reminders.lists", "apple/reminders.list", "apple/calendar.list", "apple/contacts.list", "apple/alarms.list", "apple/calendar.events.list", "apple/photos.albums.list", "apple/health.workouts.list", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/alarms.create", "apple/calendar.events.update", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 14.510449056205509, 10.321912047288809, 9.259243366443126, 8.684761659484723, 8.549670258073231, 4.1228942449914925, 3.8296680656769464, 3.8153957247318555, 3.344633169936297, 3.3265336457761996, 3.1661756288646488, 2.9379072248550164, 2.2499293125845314, 1.8024824369174723, 1.6515435525390731, 1.4901550213895047, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.0204469850336544, 0.9781948581756706, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "apple/reminders.lists", "apple/reminders.list", "apple/calendar.list", "apple/contacts.list", "apple/alarms.list", "apple/calendar.events.list", "apple/photos.albums.list", "apple/health.workouts.list", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/alarms.create", "apple/calendar.events.update", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 20.224124429670262, 12.982913464988664, 11.138827250019764, 9.259243366443126, 8.684761659484723, 4.1228942449914925, 3.8296680656769464, 3.8153957247318555, 3.344633169936297, 3.3265336457761996, 3.1661756288646488, 2.9379072248550164, 2.2499293125845314, 1.8024824369174723, 1.6515435525390731, 1.4901550213895047, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.0204469850336544, 0.9781948581756706, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N012", "query": "read UTF-8 file line range", "category": "phrase", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Reads UTF-8 with line offset and limit.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Reads UTF-8 with line offset and limit.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "apple/clipboard.read", "builtin/ListFiles", "builtin/Memory", "apple/calendar.events.list", "apple/photos.search", "apple/weather.daily", "apple/weather.hourly", "builtin/ScheduledTasks", "apple/alarms.list", "apple/battery.get", "apple/contacts.get", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.eta", "apple/photos.get", "apple/weather.current", "apple/shortcuts.run" ], "scores": [ 12.5, 6.5, 5.5, 5, 3, 2.5, 2, 2, 2, 2, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/Memory", "apple/calendar.events.list", "builtin/EditFile", "apple/clipboard.read", "builtin/ListFiles", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search", "apple/contacts.get", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "builtin/ScheduledTasks", "apple/alarms.list", "apple/photos.get", "apple/health.statistics" ], "scores": [ 22.778654236010244, 12.41527734548879, 5.095309287670941, 4.7718137863492, 4.7373151115820535, 3.2754104614901403, 3.053494701017844, 2.9702852175013907, 2.760972476842871, 2.2055776060257926, 1.596129569605437, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 1.033343757933971, 0.9551091781400716, 0.8975292682836623, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/Memory", "apple/clipboard.read", "builtin/ListFiles", "apple/calendar.events.list", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search", "apple/contacts.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/weather.current", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/photos.get", "apple/shortcuts.run", "apple/health.statistics" ], "scores": [ 33.56439581059017, 19.221736291966863, 10.649898971965822, 5.955715380754048, 5.857157575876017, 5.804260981819989, 4.830334544679512, 2.93457281792249, 2.717894186527557, 2.0745172484576795, 1.60914410629741, 1.3042089742613945, 1.2461128550311649, 1.231374918430577, 1.1490595548480602, 1.0992184654999453, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 0.9499145762410579, 0.8895050372184048, 0.7660997030790287, 0.7031782969374535 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/calendar.events.list", "apple/clipboard.read", "apple/weather.hourly", "apple/weather.daily", "apple/contacts.get", "apple/photos.search", "apple/health.workouts.get", "apple/device.get", "apple/battery.get", "apple/health.workouts.list", "apple/alarms.list", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.list", "apple/reminders.list", "apple/weather.current", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/shortcuts.run", "apple/maps.eta", "apple/health.statistics" ], "scores": [ 40.76941254376644, 21.88273770966672, 10.24338125913383, 5.804260981819989, 5.484745123318538, 5.408244136217041, 4.67189124737578, 2.6900991281633257, 2.4914715821548468, 2.1425381623194113, 1.8636633604573114, 1.8264601231288218, 1.8002653409399807, 1.7904649384226818, 1.7223681387479477, 1.648914120016547, 1.6332646411623757, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.8714182810782765, 0.8131654195283325, 0.7778939227999769, 0.7660997030790287, 0.7580363862218745, 0.4976245769158341 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N013", "query": "create UTF-8 text file", "category": "phrase", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "Creates a text file; full replacement requires explicit overwrite.", "acceptable": [ "builtin/WriteFile" ], "rationale": "Creates a text file; full replacement requires explicit overwrite.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/alarms.create", "apple/calendar.events.create", "apple/contacts.create", "apple/reminders.create", "builtin/Memory", "builtin/ListFiles", "apple/clipboard.read", "apple/shortcuts.run", "apple/clipboard.write", "builtin/ScheduledTasks", "apple/contacts.update", "apple/photos.search", "builtin/AskUserQuestions" ], "scores": [ 7.5, 7, 6, 4, 4, 4, 4, 3.5, 3, 2, 2, 1.5, 1.5, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "apple/clipboard.read", "apple/clipboard.write", "apple/reminders.create", "builtin/ListFiles", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.create", "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/contacts.update", "apple/shortcuts.run", "apple/photos.search" ], "scores": [ 15.228434969546253, 15.170543227704252, 6.1013651219564125, 5.588698166147449, 5.024433357821653, 3.147473456270661, 3.05666003761185, 3.053494701017844, 3.030740607742593, 2.9403271227683874, 2.5006287346568747, 2.165477147820978, 2.0672994037973105, 1.8905693842378057, 1.7777279238039982, 1.4705301552404113 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.create", "builtin/ListFiles", "apple/clipboard.read", "apple/clipboard.write", "apple/shortcuts.run", "builtin/ScheduledTasks", "builtin/AskUserQuestions", "apple/contacts.update", "apple/photos.search" ], "scores": [ 22.09923532695175, 20.350094993332057, 13.389913275678673, 8.742722384005322, 8.542801847641005, 8.510275502348785, 7.990654771663212, 7.914298923247929, 5.804260981819989, 4.695460391741362, 4.561302494397273, 3.7390963986769874, 2.298799517998139, 2.062180702926565, 1.762134739525646, 1.3532076178115147 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.create", "builtin/ListFiles", "apple/clipboard.read", "apple/clipboard.write", "apple/shortcuts.run", "builtin/ScheduledTasks", "builtin/AskUserQuestions", "apple/contacts.update", "apple/photos.search" ], "scores": [ 30.101911787574082, 25.21368714191401, 15.582622618653229, 8.662703535789237, 8.542801847641005, 8.510275502348785, 7.990654771663212, 7.914298923247929, 5.804260981819989, 4.524784223205542, 4.2922006742543, 3.563699388054, 2.298799517998139, 1.9405188352247646, 1.6581745951752167, 1.3532076178115147 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N014", "query": "replace exact text segment", "category": "phrase", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "Replaces one unique nonempty text segment.", "acceptable": [ "builtin/EditFile" ], "rationale": "Replaces one unique nonempty text segment.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "apple/shortcuts.run", "builtin/Memory", "apple/contacts.update", "apple/links.open", "apple/clipboard.read", "apple/music.playback", "apple/photos.albums.list", "builtin/AskUserQuestions", "builtin/ReadFile", "apple/maps.search" ], "scores": [ 4, 3, 3, 3, 2, 1.5, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "builtin/Memory", "apple/contacts.update", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ReadFile", "builtin/AskUserQuestions", "apple/photos.albums.list", "apple/music.playback", "apple/links.open" ], "scores": [ 10.78591982296686, 5.451581734141717, 5.091840995890211, 4.072799739738038, 4.007150531002361, 3.0929979541428994, 2.8581499673178334, 2.165477147820978, 2.1038511081773286, 1.6175296369719743, 1.4227132918780445 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "builtin/Memory", "apple/shortcuts.run", "apple/contacts.update", "apple/clipboard.read", "builtin/ReadFile", "builtin/AskUserQuestions", "apple/photos.albums.list", "apple/music.playback", "apple/links.open", "apple/maps.search" ], "scores": [ 13.253566444282058, 8.0515489880312, 6.464638616293724, 5.5508304544361575, 3.9311516648124067, 2.89297795482389, 2.745409276253241, 2.062180702926565, 1.6731343036002873, 1.5839818954898712, 1.3124837602688564, 1.0073540476707994 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "apple/contacts.update", "builtin/Memory", "apple/shortcuts.run", "builtin/ReadFile", "apple/clipboard.read", "builtin/WriteFile", "apple/calendar.events.update", "apple/reminders.update", "builtin/AskUserQuestions", "apple/photos.albums.list", "apple/music.playback", "apple/links.open", "apple/maps.search" ], "scores": [ 17.778121504118538, 7.475367640530878, 6.091466147888526, 6.0622526649341175, 5.37543344381317, 5.019844432888661, 2.72230178628807, 2.5040111006170407, 2.4919310360729865, 2.432653518515385, 1.9405188352247646, 1.6731343036002873, 1.3722784440479752, 1.3124837602688564, 1.0073540476707994 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N015", "query": "lasting user memories Markdown", "category": "phrase", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Maintains persistent Markdown user memories.", "acceptable": [ "builtin/Memory" ], "rationale": "Maintains persistent Markdown user memories.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/device.get", "apple/maps.show" ], "scores": [ 8, 4, 2.5, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/device.get", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel" ], "scores": [ 14.152044956510137, 3.282365175825118, 2.5490372714537815, 2.165477147820978, 2.041771298060864, 1.9107839181010244, 1.8905693842378057 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory", "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/device.get", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel" ], "scores": [ 25.893971445660384, 8.409346679470302, 2.7444868747346005, 2.1706565447392974, 2.0639215021837605, 1.9003916383525448, 1.8791219646015662 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory", "builtin/AskUserQuestions", "builtin/ScheduledTasks", "apple/device.get", "apple/calendar.events.delete", "apple/maps.show", "apple/alarms.cancel" ], "scores": [ 36.41929178431741, 8.409346679470302, 2.7444868747346005, 2.1706565447392974, 2.0639215021837605, 1.9003916383525448, 1.8791219646015662 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N016", "query": "read events overlapping time range", "category": "phrase", "domain": "calendar", "intent": "capability:apple/calendar.events.list", "target_capability": "Returns events overlapping start/end; no get ID required.", "acceptable": [ "apple/calendar.events.list" ], "rationale": "Returns events overlapping start/end; no get ID required.", "basis": [ "apple/calendar.events.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ReadFile", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/clipboard.read", "builtin/GetCurrentTime", "apple/weather.hourly", "apple/health.sleep.query", "builtin/ScheduledTasks", "apple/maps.eta", "apple/weather.daily", "apple/alarms.create", "apple/health.statistics", "apple/health.workouts.list", "apple/reminders.list", "apple/alarms.cancel", "apple/alarms.list", "apple/battery.get", "apple/contacts.get", "apple/device.get", "apple/health.workouts.get", "apple/location.get", "apple/maps.route", "apple/music.status", "apple/photos.get", "apple/photos.search", "apple/weather.current", "builtin/EditFile", "builtin/Memory", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 8.5, 5, 5, 5, 4.5, 4.5, 4, 4, 3, 2.5, 2.5, 2, 2, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.events.list", "builtin/ReadFile", "apple/health.sleep.query", "apple/weather.hourly", "apple/calendar.events.get", "apple/calendar.events.delete", "apple/maps.eta", "apple/calendar.events.update", "builtin/GetCurrentTime", "apple/calendar.events.create", "apple/weather.daily", "apple/location.get", "apple/music.status", "apple/reminders.list", "builtin/Memory", "apple/contacts.get", "apple/maps.route", "apple/alarms.cancel", "apple/alarms.create", "builtin/EditFile", "apple/clipboard.read", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/device.get", "apple/battery.get", "builtin/ScheduledTasks", "apple/alarms.list", "apple/photos.get", "apple/photos.search", "apple/health.statistics" ], "scores": [ 13.370403992009997, 5.320057110176738, 4.431223049984784, 4.380354387160692, 4.371701721545876, 3.6762037203908497, 3.4707944270573217, 3.3033503181448025, 2.989243631377985, 2.9067413944589244, 2.760972476842871, 2.4208697806788355, 1.7197049516029317, 1.6257579312913266, 1.6208496848238496, 1.596129569605437, 1.5909917502355233, 1.5741603564255942, 1.4513293217173118, 1.3632427748418257, 1.3439750578113865, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.082418100416025, 1.0694589285764813, 1.033343757933971, 0.9551091781400716, 0.8975292682836623, 0.7350474507853814, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "builtin/ReadFile", "apple/health.sleep.query", "builtin/GetCurrentTime", "apple/clipboard.read", "apple/weather.hourly", "apple/weather.daily", "apple/maps.eta", "builtin/ScheduledTasks", "apple/health.workouts.list", "builtin/Memory", "apple/contacts.get", "apple/location.get", "apple/alarms.create", "apple/reminders.list", "builtin/EditFile", "apple/health.statistics", "apple/health.workouts.get", "apple/weather.current", "apple/music.status", "apple/device.get", "apple/battery.get", "apple/maps.route", "apple/alarms.cancel", "apple/alarms.list", "apple/photos.get", "apple/reminders.create", "apple/reminders.update", "apple/photos.search" ], "scores": [ 18.394462803098374, 11.557162787362683, 10.699359130222755, 10.19763685123182, 9.161857596411817, 8.228845139486534, 5.172799228455651, 4.858690344610446, 4.054675138958545, 3.8563231333659926, 2.717894186527557, 2.661694808656941, 2.2865400716465247, 1.8676898296507356, 1.6111411824454678, 1.60914410629741, 1.6043303491322136, 1.578316724633953, 1.5142936950371024, 1.3906556055226948, 1.3385764696989595, 1.3042089742613945, 1.231374918430577, 1.1370398010457616, 1.0850070323653966, 1.0711583788400274, 1.0469491829486346, 1.0352314574515493, 0.9499145762410579, 0.8895050372184048, 0.8668821733731853, 0.8668821733731853, 0.721309630646165 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "builtin/ReadFile", "builtin/GetCurrentTime", "apple/health.sleep.query", "apple/weather.hourly", "apple/clipboard.read", "apple/location.get", "apple/weather.daily", "apple/reminders.list", "apple/maps.eta", "apple/health.workouts.list", "apple/contacts.get", "builtin/ScheduledTasks", "apple/health.workouts.get", "apple/device.get", "apple/battery.get", "apple/alarms.list", "apple/photos.get", "apple/alarms.create", "builtin/Memory", "apple/music.status", "apple/health.statistics", "apple/maps.route", "apple/alarms.cancel", "builtin/EditFile", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.list", "apple/weather.current", "apple/reminders.create", "apple/reminders.update", "apple/photos.search" ], "scores": [ 18.9723723946359, 12.533842224985177, 10.699359130222755, 10.19763685123182, 9.161857596411817, 7.873333425280212, 6.466611294672154, 4.851474685755682, 3.611849443606829, 2.8694088104583075, 2.5810097867547075, 2.4914715821548468, 2.395776456156075, 2.3485728160387884, 2.3439451133675187, 2.1425381623194113, 1.9506459363267974, 1.8264601231288218, 1.8002653409399807, 1.7904649384226818, 1.648914120016547, 1.6332646411623757, 1.578316724633953, 1.1401709250099572, 1.1370398010457616, 1.1330227496773402, 1.0469491829486346, 1.0352314574515493, 0.9841378926907047, 0.9510032826687982, 0.9266425589614367, 0.9266425589614367, 0.9034987037373633, 0.8714182810782765, 0.8668821733731853, 0.8668821733731853, 0.5104557426457969 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N017", "query": "mark reminder completion", "category": "phrase", "domain": "reminders", "intent": "capability:apple/reminders.complete", "target_capability": "Sets completed on an existing reminder.", "acceptable": [ "apple/reminders.complete" ], "rationale": "Sets completed on an existing reminder.", "basis": [ "apple/reminders.complete" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.update", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/links.open", "apple/shortcuts.run", "builtin/ScheduledTasks" ], "scores": [ 5.5, 4.5, 4.5, 4.5, 4, 4, 4, 2, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/reminders.complete", "apple/reminders.get", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.list", "apple/reminders.create", "apple/shortcuts.run" ], "scores": [ 7.68921743625183, 3.612813620113822, 3.5412226376495974, 3.2512998801244857, 3.1371483815404533, 3.110945717900015, 3.05666003761185, 2.7491696581568283 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/reminders.complete", "apple/reminders.get", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.lists", "apple/reminders.list", "apple/reminders.create", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 13.660175832119426, 9.605636229394815, 9.130891830020635, 8.373833017913912, 8.138621340785264, 7.845501845651658, 7.538338257026199, 2.7231044752292686, 0.9040728988797129, 0.70568370941594 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.complete", "apple/reminders.get", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.lists", "apple/reminders.list", "apple/reminders.create", "apple/shortcuts.run", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 13.660175832119426, 9.605636229394815, 9.130891830020635, 8.373833017913912, 8.138621340785264, 7.845501845651658, 7.538338257026199, 2.7231044752292686, 1.2107026713576772, 0.9040728988797129 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N018", "query": "find contacts by phone email", "category": "phrase", "domain": "contacts", "intent": "capability:apple/contacts.list", "target_capability": "List query matches name, organization, phone and email.", "acceptable": [ "apple/contacts.list" ], "rationale": "List query matches name, organization, phone and email.", "basis": [ "apple/contacts.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "builtin/EditFile", "builtin/ListFiles", "builtin/ReadFile", "builtin/ScheduledTasks", "builtin/WriteFile" ], "scores": [ 7.5, 6, 5, 4.5, 4.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.update", "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete" ], "scores": [ 9.62988633490375, 8.904887263053702, 4.062325774173696, 3.908385007994482, 3.6211209622787783 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "apple/contacts.get", "apple/contacts.delete", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/ScheduledTasks" ], "scores": [ 18.190254824016865, 14.14965819813494, 13.731304536422714, 11.963738092095124, 11.146694165037609, 1.0922001006905449, 0.9881630284044697, 0.9518941151163509, 0.9022222210637898, 0.8652613463947955 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "apple/contacts.get", "apple/contacts.delete", "apple/music.search", "apple/photos.search", "apple/battery.get", "apple/device.get", "apple/maps.search", "apple/photos.get", "apple/alarms.list", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.get", "apple/reminders.list", "apple/health.workouts.list", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/ScheduledTasks" ], "scores": [ 18.190254824016865, 15.592826328846504, 13.731304536422714, 13.527042142651224, 11.146694165037609, 1.7052574879460844, 1.7052574879460844, 1.607920950061708, 1.607920950061708, 1.607920950061708, 1.5633040505560998, 1.5210963760156138, 1.5210963760156138, 1.5210963760156138, 1.4811079163986542, 1.4431681307115642, 1.407123515710695, 1.407123515710695, 1.407123515710695, 1.3728355300713926, 1.30903966193533, 1.0922001006905449, 0.9881630284044697, 0.9518941151163509, 0.9022222210637898, 0.8652613463947955 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N019", "query": "route distance travel time ordered steps", "category": "phrase", "domain": "maps", "intent": "capability:apple/maps.route", "target_capability": "Route returns ordered steps; eta does not.", "acceptable": [ "apple/maps.route" ], "rationale": "Route returns ordered steps; eta does not.", "basis": [ "apple/maps.route" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/GetCurrentTime", "apple/health.statistics", "apple/weather.hourly", "apple/alarms.create", "apple/calendar.events.list", "apple/reminders.list", "apple/alarms.cancel", "apple/health.workouts.get", "apple/location.get", "apple/music.status", "apple/weather.daily", "builtin/ScheduledTasks", "apple/calendar.events.create", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.workouts.list", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 9, 4, 4, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/maps.route", "apple/maps.eta", "apple/weather.hourly", "apple/health.workouts.get", "builtin/GetCurrentTime", "apple/location.get", "apple/health.statistics", "apple/weather.daily", "apple/calendar.events.list", "apple/music.status", "apple/reminders.list", "apple/alarms.cancel", "apple/alarms.create" ], "scores": [ 18.698912329219787, 11.96939132526323, 3.745551198539369, 3.4945260320409735, 2.989243631377985, 2.4208697806788355, 2.219316713009146, 2.1709031724984498, 1.8251756430963588, 1.7197049516029317, 1.6257579312913266, 1.5741603564255942, 1.4513293217173118 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/GetCurrentTime", "apple/health.statistics", "apple/health.workouts.get", "apple/weather.hourly", "apple/weather.daily", "apple/calendar.events.list", "apple/location.get", "apple/alarms.create", "apple/reminders.list", "apple/music.status", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/health.workouts.list", "apple/calendar.events.create", "apple/calendar.events.update" ], "scores": [ 25.541051923943513, 10.815148546708565, 4.858690344610446, 4.017291136515695, 3.5983069383381916, 3.22915232462268, 2.1370314849333996, 1.8701253527179627, 1.6043303491322136, 1.578316724633953, 1.5142936950371024, 1.1370398010457616, 1.0352314574515493, 0.9536608257008461, 0.8668821733731853, 0.8668821733731853, 0.6727985360982812, 0.6215769746195708, 0.6148894289223729, 0.589518897346431 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/GetCurrentTime", "apple/health.statistics", "apple/health.workouts.get", "apple/weather.hourly", "apple/weather.daily", "apple/calendar.events.list", "apple/location.get", "apple/alarms.create", "apple/reminders.list", "apple/music.status", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/health.workouts.list", "apple/calendar.events.create", "apple/calendar.events.update" ], "scores": [ 25.541051923943513, 10.815148546708565, 6.466611294672154, 4.017291136515695, 3.5983069383381916, 3.22915232462268, 2.1370314849333996, 1.8701253527179627, 1.6043303491322136, 1.578316724633953, 1.5142936950371024, 1.1370398010457616, 1.0352314574515493, 0.9536608257008461, 0.8668821733731853, 0.8668821733731853, 0.6727985360982812, 0.6215769746195708, 0.6148894289223729, 0.589518897346431 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N020", "query": "hourly forecast explicit coordinates", "category": "phrase", "domain": "weather", "intent": "capability:apple/weather.hourly", "target_capability": "Hourly forecast with explicit coordinates.", "acceptable": [ "apple/weather.hourly" ], "rationale": "Hourly forecast with explicit coordinates.", "basis": [ "apple/weather.hourly" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/weather.hourly", "apple/weather.current", "apple/weather.daily", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/maps.search", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/location.geocode", "apple/maps.route", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 7, 3, 3, 2, 2, 2, 1.5, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/weather.hourly", "apple/weather.daily", "apple/weather.current", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 9.77232932304213, 6.015779997346616, 5.466013271599939, 4.344331537974681, 3.8242096099791643, 3.415313669393855, 1.769394254695836, 1.5045297512928304, 1.41382735889783, 1.3859756210546532, 1.3209217681794347, 1.1344628597433402 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/weather.hourly", "apple/weather.daily", "apple/weather.current", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/maps.search", "apple/location.geocode", "apple/maps.route", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 17.387891814990255, 5.870612840394097, 5.334879775308666, 4.35250156347687, 3.7414095359091446, 3.31792153621212, 2.078782606482171, 1.6844578417802005, 1.4175298978595754, 1.3134609719514998, 1.2480153304315296, 1.1020517573918405, 0.8505056366425909, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/weather.hourly", "apple/weather.daily", "apple/weather.current", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/maps.search", "apple/location.geocode", "apple/maps.route", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 17.387891814990255, 5.870612840394097, 5.334879775308666, 4.35250156347687, 3.7414095359091446, 3.31792153621212, 2.078782606482171, 1.6844578417802005, 1.4175298978595754, 1.3134609719514998, 1.2480153304315296, 1.1020517573918405, 0.8505056366425909, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N021", "query": "sleep stage intervals source", "category": "phrase", "domain": "health", "intent": "capability:apple/health.sleep.query", "target_capability": "Returns sleep intervals and source, not a summed night.", "acceptable": [ "apple/health.sleep.query" ], "rationale": "Returns sleep intervals and source, not a summed night.", "basis": [ "apple/health.sleep.query" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/health.sleep.query", "apple/weather.current", "builtin/ScheduledTasks" ], "scores": [ 7, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/health.sleep.query", "apple/weather.current" ], "scores": [ 20.201272345000685, 2.669503018633696 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/health.sleep.query", "apple/weather.current", "builtin/ScheduledTasks" ], "scores": [ 26.983976800120658, 2.640038657355815, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/health.sleep.query", "apple/weather.current", "builtin/ScheduledTasks" ], "scores": [ 26.983976800120658, 2.640038657355815, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N022", "query": "list photo albums metadata", "category": "phrase", "domain": "photos", "intent": "capability:apple/photos.albums.list", "target_capability": "Lists regular and smart albums metadata.", "acceptable": [ "apple/photos.albums.list" ], "rationale": "Lists regular and smart albums metadata.", "basis": [ "apple/photos.albums.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/photos.albums.list", "apple/photos.search", "apple/photos.get", "apple/alarms.list", "apple/reminders.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.list", "apple/reminders.lists", "builtin/ListFiles", "apple/health.workouts.list", "apple/links.open", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/contacts.get", "apple/reminders.create", "builtin/Memory", "builtin/ScheduledTasks", "apple/alarms.create", "apple/calendar.events.get", "apple/shortcuts.run", "apple/calendar.events.create", "apple/contacts.delete", "apple/contacts.update", "apple/health.workouts.get" ], "scores": [ 13, 6.5, 5.5, 5, 4.5, 4, 4, 4, 4, 4, 3, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/photos.albums.list", "apple/photos.search", "apple/photos.get", "apple/alarms.list", "apple/contacts.get", "apple/reminders.lists", "builtin/ListFiles", "apple/calendar.list", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/calendar.events.get", "apple/alarms.create", "apple/health.workouts.list", "apple/reminders.create", "builtin/Memory", "apple/calendar.events.delete", "apple/alarms.cancel", "apple/shortcuts.run", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 10.92031086068259, 5.96244402058457, 5.455059126413018, 3.8664894335213966, 3.133344695217922, 2.1668991632120185, 1.9903435646574756, 1.970426045916842, 1.614897263876673, 1.4924909939710067, 1.4924909939710067, 1.457997372644377, 1.3776271994526728, 1.3390180568934824, 1.2865248727729224, 1.164760090359424, 1.1248369805109821, 1.0415379820610877, 0.9793722303182939, 0.9242094098357349, 0.899392122924025 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/photos.albums.list", "apple/photos.search", "apple/photos.get", "apple/alarms.list", "apple/reminders.lists", "builtin/ListFiles", "apple/reminders.list", "apple/calendar.list", "apple/contacts.get", "apple/contacts.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/links.open", "builtin/Memory", "apple/alarms.create", "apple/calendar.events.update", "apple/shortcuts.run", "apple/contacts.delete", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 25.499616026705496, 14.354037177321125, 13.027204308446525, 5.709787024621713, 4.1228942449914925, 3.8392072769683265, 3.8296680656769464, 3.8153957247318555, 3.521879101101916, 3.344633169936297, 3.1661756288646488, 2.2499293125845314, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.1237743813923031, 1.1087089859323587, 1.0204469850336544, 0.9781948581756706, 0.724055318310476, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/photos.albums.list", "apple/photos.search", "apple/photos.get", "apple/alarms.list", "builtin/ListFiles", "apple/reminders.lists", "apple/reminders.list", "apple/calendar.list", "apple/contacts.get", "apple/contacts.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/links.open", "apple/reminders.create", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/ScheduledTasks", "apple/calendar.events.get", "builtin/Memory", "apple/alarms.create", "apple/calendar.events.update", "apple/shortcuts.run", "apple/contacts.delete", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create" ], "scores": [ 25.499616026705496, 14.354037177321125, 13.027204308446525, 5.709787024621713, 5.039330650712083, 4.1228942449914925, 3.8296680656769464, 3.8153957247318555, 3.521879101101916, 3.344633169936297, 3.1661756288646488, 2.2499293125845314, 1.7843771184199018, 1.4697791978069161, 1.4175588968227548, 1.3736294807719625, 1.2610708289265875, 1.1253412868673076, 1.1087089859323587, 1.0204469850336544, 0.9781948581756706, 0.724055318310476, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N023", "query": "看看这个目录里有哪些文件", "category": "chinese", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Directory enumeration is ListFiles.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Directory enumeration is ListFiles.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/ReadFile" ], "scores": [ 10.845833860260033, 3.9133926474172007, 3.8077348880372774 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N024", "query": "读取文件第十行开始的二十行", "category": "chinese", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Line offset/limit belong to ReadFile.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Line offset/limit belong to ReadFile.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/battery.get", "apple/calendar.events.get", "apple/contacts.get", "apple/device.get", "apple/health.workouts.get", "apple/location.get", "apple/photos.get", "apple/reminders.get" ], "scores": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/health.workouts.get" ], "scores": [ 3.061358970857833, 3.061358970857833, 2.855577779850958, 2.855577779850958, 2.6757188918211545, 2.6757188918211545, 2.3763673602466326, 2.250478892425348 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/ListFiles", "builtin/WriteFile", "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/health.workouts.get" ], "scores": [ 11.215511602767954, 5.5906952380295865, 3.9133926474172007, 3.5772446563552007, 3.5772446563552007, 3.5077195040172278, 3.5077195040172278, 3.4408453261367575, 3.4408453261367575, 3.376473337912044, 3.2546944532702384 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N025", "query": "帮我新建一个文本文件", "category": "chinese", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "Creates UTF-8 file.", "acceptable": [ "builtin/WriteFile" ], "rationale": "Creates UTF-8 file.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/alarms.create", "apple/calendar.events.create", "apple/contacts.create", "apple/reminders.create" ], "scores": [ 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/alarms.create", "apple/contacts.create", "apple/calendar.events.create", "apple/reminders.create" ], "scores": [ 4.412032236400579, 3.818888304565086, 3.578355194658009, 3.17801937787516 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "apple/alarms.create", "apple/contacts.create", "apple/calendar.events.create", "apple/reminders.create", "builtin/WriteFile" ], "scores": [ 4.87245480374825, 4.679304717430048, 4.588360595747694, 4.500884147785318, 3.275434609930434 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } } } }, { "query_id": "N026", "query": "把文件里唯一的一段文字替换掉", "category": "chinese", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "Exact one-segment edit is EditFile.", "acceptable": [ "builtin/EditFile" ], "rationale": "Exact one-segment edit is EditFile.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ListFiles", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/contacts.update", "apple/calendar.events.update", "apple/reminders.update" ], "scores": [ 5.5906952380295865, 3.9133926474172007, 3.8077348880372774, 3.4625922028793776, 3.203146750471291, 3.1250942919952016, 3.050755223585493 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } } } }, { "query_id": "N027", "query": "记住我长期偏好使用中文交流", "category": "chinese", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Lasting user preference belongs in Memory.", "acceptable": [ "builtin/Memory" ], "rationale": "Lasting user preference belongs in Memory.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/Memory" ], "scores": [ 9.433359994200462 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N028", "query": "查一下以前记下来的我的偏好", "category": "chinese", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Views lasting user memories.", "acceptable": [ "builtin/Memory" ], "rationale": "Views lasting user memories.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "N029", "query": "现在几点，今天星期几", "category": "chinese", "domain": "clock", "intent": "capability:builtin/GetCurrentTime", "target_capability": "Device current time/date/weekday.", "acceptable": [ "builtin/GetCurrentTime" ], "rationale": "Device current time/date/weekday.", "basis": [ "builtin/GetCurrentTime" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/GetCurrentTime" ], "scores": [ 15.765415866691338 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N030", "query": "用一个问题卡片让我选择方案", "category": "chinese", "domain": "questions", "intent": "capability:builtin/AskUserQuestions", "target_capability": "Question card with choices.", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Question card with choices.", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/AskUserQuestions" ], "scores": [ 10.819054455927933 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N031", "query": "每天早上定时帮我做个只读检查", "category": "chinese", "domain": "scheduling", "intent": "capability:builtin/ScheduledTasks", "target_capability": "Daily best-effort read-only scheduled task.", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Daily best-effort read-only scheduled task.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ScheduledTasks" ], "scores": [ 4.971371455436067 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N032", "query": "查一下下周日历有哪些安排", "category": "chinese", "domain": "calendar", "intent": "capability:apple/calendar.events.list", "target_capability": "Reads events over a range.", "acceptable": [ "apple/calendar.events.list" ], "rationale": "Reads events over a range.", "basis": [ "apple/calendar.events.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.list", "apple/links.open" ], "scores": [ 3, 3, 3, 3, 3, 3, 2 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.list", "apple/links.open" ], "scores": [ 6.307066233654392, 6.307066233654392, 5.909814956690538, 5.909814956690538, 5.559640381210311, 5.559640381210311, 0.799714193115399 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "combined": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.list", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 6.824089447774135, 5.877308721094332, 5.718627155863521, 5.290141292477006, 5.290141292477006, 5.161234137816374, 1.3720255930646441 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } } } }, { "query_id": "N033", "query": "把这条待办标记为完成", "category": "chinese", "domain": "reminders", "intent": "capability:apple/reminders.complete", "target_capability": "Updates completion, not delete.", "acceptable": [ "apple/reminders.complete" ], "rationale": "Updates completion, not delete.", "basis": [ "apple/reminders.complete" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "apple/links.open" ], "scores": [ 3, 3, 3, 3, 3, 3, 3, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/reminders.complete", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/links.open" ], "scores": [ 5.900735909580919, 5.900735909580919, 5.201462681129266, 5.201462681129266, 4.910500535445122, 4.910500535445122, 4.650365928740785, 0.6140403197686277 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.complete", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/reminders.update", "apple/links.open" ], "scores": [ 13.165146488179994, 7.610419067199501, 5.997963272418636, 5.350206793778464, 5.209553951730541, 4.8287236072233135, 4.71385897242224, 0.6140403197686277 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N034", "query": "查找名字里包含小王的联系人", "category": "chinese", "domain": "contacts", "intent": "capability:apple/contacts.list", "target_capability": "Substring contact search.", "acceptable": [ "apple/contacts.list" ], "rationale": "Substring contact search.", "basis": [ "apple/contacts.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.list", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.list", "apple/maps.search", "apple/music.search", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.list" ], "scores": [ 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/contacts.list", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.create", "apple/contacts.get", "apple/alarms.list", "apple/music.search", "apple/photos.search", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/maps.search", "apple/reminders.list", "apple/health.workouts.list" ], "scores": [ 5.669365781612276, 3.76823592045894, 3.76823592045894, 3.514939236506309, 3.514939236506309, 2.7182316407131206, 2.535515092349053, 2.535515092349053, 2.375815388733735, 2.235041074941157, 2.235041074941157, 2.235041074941157, 1.9982377076261157, 1.8977063868338677 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/contacts.list", "apple/contacts.delete", "apple/music.search", "apple/photos.search", "apple/maps.search", "apple/contacts.create", "apple/contacts.get", "apple/alarms.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/contacts.update", "apple/health.workouts.list" ], "scores": [ 6.151206179340887, 3.9296844445859853, 3.5147642278936893, 3.5147642278936893, 3.375434661720279, 3.3689159367244588, 3.275434609930434, 3.246730289701211, 3.1274803458411458, 3.071081104659632, 3.071081104659632, 3.016679975730112, 2.9482051569982852, 2.9134618525541662 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N035", "query": "按给定起终点坐标计算步行路线", "category": "chinese", "domain": "maps", "intent": "capability:apple/maps.route", "target_capability": "Explicit-coordinate route calculation.", "acceptable": [ "apple/maps.route" ], "rationale": "Explicit-coordinate route calculation.", "basis": [ "apple/maps.route" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/maps.show" ], "scores": [ 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.route", "apple/maps.show", "apple/location.get", "apple/maps.search" ], "scores": [ 4.851812771864217, 4.851812771864217, 4.094088437154942, 4.094088437154942, 4.094088437154942, 3.9350368504723905, 3.366326726695869 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "combined": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.route", "apple/maps.show", "apple/location.get", "apple/maps.search" ], "scores": [ 5.866964571583784, 5.866964571583784, 5.1220885466831145, 5.1220885466831145, 4.131942557119925, 3.8077348880372774, 3.6602378602162426 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } } } }, { "query_id": "N036", "query": "查未来七天每天的天气", "category": "chinese", "domain": "weather", "intent": "capability:apple/weather.daily", "target_capability": "Daily forecast, not hourly/current.", "acceptable": [ "apple/weather.daily" ], "rationale": "Daily forecast, not hourly/current.", "basis": [ "apple/weather.daily" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 5.26032566896794, 5.26032566896794, 5.26032566896794 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/ScheduledTasks" ], "scores": [ 6.121624994499776, 6.121624994499776, 6.121624994499776, 4.971371455436067 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N037", "query": "暂停正在播放的歌曲", "category": "chinese", "domain": "music", "intent": "capability:apple/music.playback", "target_capability": "pause action controls Anan player.", "acceptable": [ "apple/music.playback" ], "rationale": "pause action controls Anan player.", "basis": [ "apple/music.playback" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/music.playback", "apple/music.search", "apple/music.status" ], "scores": [ 3, 3, 3 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/music.playback", "apple/music.status", "apple/music.search" ], "scores": [ 10.52065133793588, 10.52065133793588, 8.399091419843161 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/music.playback", "apple/music.status", "apple/music.search" ], "scores": [ 12.243249988999551, 12.243249988999551, 8.537484583254917 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N038", "query": "I forgot the filename; show names in the current folder", "category": "intent", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Only directory enumeration requested.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Only directory enumeration requested.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/maps.show", "apple/weather.current", "builtin/GetCurrentTime", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.update", "apple/shortcuts.run", "builtin/EditFile", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/calendar.events.list", "apple/contacts.list", "apple/device.get", "apple/health.statistics", "apple/links.open", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/music.playback", "apple/music.search", "apple/music.status", "apple/photos.search", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 6.5, 4, 4, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/maps.show", "apple/clipboard.read", "apple/shortcuts.run", "apple/device.get", "apple/contacts.create", "builtin/GetCurrentTime", "apple/contacts.list", "apple/contacts.update", "apple/calendar.events.list", "builtin/EditFile", "apple/alarms.list", "apple/weather.current", "apple/alarms.cancel", "apple/location.reverseGeocode", "apple/music.search", "apple/weather.daily", "apple/alarms.create", "apple/location.geocode", "apple/maps.eta", "apple/music.status", "apple/photos.search", "apple/maps.route", "apple/maps.search", "apple/weather.hourly", "apple/links.open", "apple/health.statistics", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 7.4579784107371605, 4.691207804444204, 3.1669845791669973, 3.114418360186576, 2.4692163806821252, 2.205227515488847, 2.0915170203879607, 2.0251373927906937, 1.934298977912546, 1.8329870145700418, 1.6861137326635875, 1.668971403398956, 1.6682760259384652, 1.6335467459150792, 1.584464330735021, 1.5605468552419268, 1.5381011872642523, 1.5102993475823525, 1.4379715520033696, 1.3881142614424309, 1.297625056165918, 1.2842193286009138, 1.2067986160068875, 1.1381819434781144, 1.0010725906208502, 0.9645187186498915, 0.9418286545359099, 0.8833160037757247 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/maps.show", "builtin/GetCurrentTime", "apple/weather.current", "apple/clipboard.read", "apple/shortcuts.run", "apple/contacts.create", "apple/contacts.update", "apple/device.get", "builtin/EditFile", "apple/contacts.list", "apple/calendar.events.list", "apple/alarms.list", "apple/alarms.cancel", "apple/location.reverseGeocode", "apple/music.search", "apple/alarms.create", "apple/weather.daily", "apple/location.geocode", "apple/maps.eta", "apple/music.status", "apple/photos.search", "apple/maps.route", "builtin/ScheduledTasks", "apple/maps.search", "apple/weather.hourly", "apple/links.open", "apple/health.statistics", "apple/music.playback" ], "scores": [ 17.58395706701395, 5.216525260283588, 5.045382794218522, 4.668650686638992, 4.428347172254761, 4.00922799620642, 3.4385279545186087, 3.119415169082174, 2.3540016180843235, 2.0828091760528666, 1.9662731002711833, 1.6769433783228374, 1.6581745951752167, 1.592799090577726, 1.570298655749525, 1.5219060705338434, 1.4663815769404227, 1.4428330874025772, 1.369169802552464, 1.3186764169813905, 1.2733728264483664, 1.2141942578109772, 1.1703680984814393, 1.1369779299085616, 1.0689954759741835, 0.9724723892857147, 0.898812452944287, 0.8767732783360794 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "apple/maps.show", "builtin/ListFiles", "builtin/GetCurrentTime", "apple/weather.current", "apple/shortcuts.run", "apple/contacts.list", "apple/calendar.events.list", "apple/alarms.list", "apple/contacts.create", "apple/contacts.update", "apple/clipboard.read", "apple/device.get", "builtin/EditFile", "apple/photos.albums.list", "apple/calendar.list", "apple/reminders.list", "apple/health.workouts.list", "apple/alarms.cancel", "apple/location.reverseGeocode", "apple/music.search", "apple/alarms.create", "apple/weather.daily", "apple/location.geocode", "apple/maps.eta", "apple/music.status", "apple/photos.search", "apple/maps.route", "builtin/ScheduledTasks", "apple/maps.search", "apple/weather.hourly", "apple/links.open", "apple/health.statistics", "apple/music.playback" ], "scores": [ 12.211946578674754, 10.510277244460893, 7.081303815323238, 5.045382794218522, 4.428347172254761, 4.394393424857844, 4.2201231443656075, 4.113348637617448, 4.00922799620642, 3.4385279545186087, 3.3922838226895626, 3.119415169082174, 2.3540016180843235, 2.311584248804978, 2.253850044094424, 2.19892950792101, 2.09674493165168, 1.6581745951752167, 1.592799090577726, 1.570298655749525, 1.5219060705338434, 1.4663815769404227, 1.4428330874025772, 1.369169802552464, 1.3186764169813905, 1.2733728264483664, 1.2141942578109772, 1.1703680984814393, 1.1369779299085616, 1.0689954759741835, 0.9724723892857147, 0.898812452944287, 0.8767732783360794 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N039", "query": "I need the contents of notes.txt, not just its name", "category": "intent", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Read an existing text file.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Read an existing text file.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open", "builtin/ScheduledTasks", "apple/clipboard.write", "apple/contacts.list", "apple/maps.search", "apple/maps.show", "apple/alarms.create", "apple/alarms.list", "apple/clipboard.read", "apple/music.search", "apple/calendar.events.list", "apple/calendar.events.update", "apple/contacts.create", "apple/contacts.update", "apple/health.sleep.query", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.create", "apple/reminders.list", "apple/reminders.update", "apple/shortcuts.run", "apple/alarms.cancel", "apple/battery.get", "apple/calendar.events.get", "apple/contacts.get", "apple/device.get", "apple/health.statistics", "apple/health.workouts.get", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/music.playback", "apple/music.status", "apple/photos.get", "apple/weather.current", "apple/weather.daily", "apple/calendar.events.create", "apple/health.workouts.list", "builtin/Memory", "builtin/WriteFile" ], "scores": [ 4.5, 3, 2.5, 2.5, 2.5, 2.5, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "builtin/ScheduledTasks", "apple/clipboard.write", "apple/clipboard.read", "apple/calendar.events.get", "apple/health.workouts.get", "apple/shortcuts.run", "apple/contacts.get", "apple/device.get", "apple/maps.search", "apple/reminders.update", "apple/contacts.list", "apple/alarms.list", "apple/maps.show", "apple/music.search", "apple/contacts.create", "apple/reminders.list", "apple/alarms.create", "apple/music.playback", "apple/calendar.events.update", "apple/contacts.update", "apple/calendar.events.list", "apple/links.open", "apple/alarms.cancel", "apple/health.sleep.query", "apple/weather.current", "apple/photos.albums.list", "apple/photos.search", "apple/health.statistics", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/weather.daily" ], "scores": [ 6.089871907450824, 4.3968000490354155, 4.126521925498937, 3.3189549419353934, 3.1777733868287177, 3.1669845791669973, 3.133344695217922, 3.114418360186576, 3.1068144561180953, 3.048112898315442, 3.0212410077285945, 2.9568323463287935, 2.956632980994578, 2.800696056612951, 2.4692163806821252, 2.448649573649498, 2.368074492613364, 2.167107836671426, 2.1038511081773286, 2.0251373927906937, 1.934298977912546, 1.8609071452820887, 1.6682760259384652, 1.5629367438392368, 1.3848806590980578, 1.3627028549488005, 1.297625056165918, 1.2432451371448472, 1.2162317258779303, 1.1119626517018855, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9834495738143223, 0.7495532564546209 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "builtin/ScheduledTasks", "apple/clipboard.write", "apple/maps.show", "apple/shortcuts.run", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.update", "apple/calendar.events.get", "apple/contacts.list", "apple/maps.search", "apple/reminders.update", "apple/links.open", "apple/health.workouts.get", "apple/device.get", "apple/calendar.events.update", "apple/alarms.list", "apple/music.search", "apple/contacts.get", "apple/calendar.events.list", "apple/reminders.list", "apple/alarms.create", "apple/reminders.create", "apple/music.playback", "builtin/WriteFile", "apple/alarms.cancel", "apple/health.sleep.query", "apple/photos.albums.list", "apple/photos.search", "builtin/Memory", "apple/weather.current", "apple/calendar.events.create", "apple/health.statistics", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/weather.daily", "apple/health.workouts.list" ], "scores": [ 5.881770087964311, 5.066242127384126, 4.602588997745121, 4.428347172254761, 4.106680032946002, 4.00922799620642, 3.4385279545186087, 3.432161121074587, 3.33281305718288, 3.32218021823558, 3.300021167757637, 3.2799844103704494, 3.2721473302672357, 3.119415169082174, 2.7432118216061774, 2.71720115433891, 2.563381729591779, 2.5305379431348114, 2.276181297253949, 2.259122597561351, 2.196766475540141, 1.7643212335994698, 1.720954340406063, 1.6690871113261696, 1.6581745951752167, 1.6244277159640166, 1.6193758131939588, 1.5572714341624867, 1.1941549476394007, 1.1309556352520849, 1.116129367372341, 1.0093267431030897, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.6063450614565641, 0.29633377370106745 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ScheduledTasks", "builtin/WriteFile", "apple/clipboard.write", "apple/maps.show", "builtin/ReadFile", "apple/shortcuts.run", "apple/links.open", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.update", "apple/calendar.events.get", "apple/contacts.list", "apple/maps.search", "apple/reminders.update", "apple/health.workouts.get", "apple/device.get", "apple/calendar.events.update", "apple/alarms.list", "apple/music.search", "apple/contacts.get", "apple/calendar.events.list", "apple/reminders.list", "apple/alarms.create", "apple/reminders.create", "apple/alarms.cancel", "apple/health.sleep.query", "apple/photos.albums.list", "apple/music.playback", "apple/photos.search", "apple/weather.current", "apple/calendar.events.create", "builtin/Memory", "apple/health.statistics", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/weather.daily", "apple/health.workouts.list" ], "scores": [ 5.881770087964311, 4.811677440005942, 4.748517860972697, 4.602588997745121, 4.500884147785318, 4.428347172254761, 4.174600716577611, 4.106680032946002, 4.00922799620642, 3.4385279545186087, 3.432161121074587, 3.33281305718288, 3.32218021823558, 3.300021167757637, 3.2721473302672357, 3.119415169082174, 2.7432118216061774, 2.71720115433891, 2.563381729591779, 2.5305379431348114, 2.276181297253949, 2.259122597561351, 2.196766475540141, 1.7643212335994698, 1.6581745951752167, 1.6244277159640166, 1.6193758131939588, 1.5839818954898712, 1.5572714341624867, 1.1309556352520849, 1.116129367372341, 1.0991109834001507, 1.0093267431030897, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.6063450614565641, 0.29633377370106745 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } } } }, { "query_id": "N040", "query": "Save this entire draft as a new text document", "category": "intent", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "Whole new file, not patch.", "acceptable": [ "builtin/WriteFile" ], "rationale": "Whole new file, not patch.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/links.open", "builtin/EditFile", "apple/clipboard.write", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/clipboard.read", "builtin/AskUserQuestions", "builtin/Memory", "builtin/ReadFile", "apple/calendar.events.update" ], "scores": [ 2, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/contacts.update", "apple/clipboard.write", "apple/clipboard.read", "builtin/ScheduledTasks", "builtin/ReadFile", "builtin/EditFile", "builtin/AskUserQuestions", "apple/shortcuts.run" ], "scores": [ 5.294608914812445, 3.147473456270661, 3.0929979541428994, 2.939466819887189, 2.8581499673178334, 2.7272927852161843, 2.165477147820978, 1.7777279238039982 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "builtin/EditFile", "apple/contacts.update", "apple/clipboard.write", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ScheduledTasks", "builtin/ReadFile", "builtin/Memory", "builtin/AskUserQuestions", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 5.37491513278338, 5.145562771104417, 4.561302494397273, 2.972996695597959, 2.89297795482389, 2.887598644067425, 2.745409276253241, 2.669512779150425, 2.062180702926565, 1.7823784642970022, 1.5055923631327193 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "builtin/ReadFile", "apple/contacts.update", "apple/clipboard.write", "apple/alarms.create", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/contacts.create", "apple/clipboard.read", "apple/calendar.events.create", "apple/reminders.create", "apple/links.open", "builtin/Memory", "builtin/AskUserQuestions", "apple/calendar.events.update" ], "scores": [ 14.18772319163044, 10.977167322774886, 8.827579320925938, 5.0416026267539875, 4.2922006742543, 2.9026295885011453, 2.887598644067425, 2.797599684974972, 2.7369467418327798, 2.72230178628807, 2.6610014176998558, 2.5891569919465325, 2.390644071377807, 2.169239180031041, 1.9405188352247646, 1.2119692675107079 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N041", "query": "Fix a single typo in an existing document without replacing everything", "category": "intent", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "One exact substring replacement.", "acceptable": [ "builtin/EditFile" ], "rationale": "One exact substring replacement.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/WriteFile", "apple/battery.get", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/photos.search", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.write" ], "scores": [ 3, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "builtin/WriteFile", "apple/location.reverseGeocode", "apple/location.geocode", "apple/battery.get", "apple/maps.search", "apple/photos.search" ], "scores": [ 10.130387001786444, 2.8055106012874256, 2.5938411871930867, 2.4696228887134084, 2.072598362607447, 1.6973910453637415 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "builtin/WriteFile", "apple/location.reverseGeocode", "apple/location.geocode", "apple/battery.get", "apple/maps.search", "apple/calendar.events.delete", "apple/photos.search", "apple/clipboard.write", "apple/calendar.events.update" ], "scores": [ 11.697348827220424, 2.877556815390035, 2.606627671177044, 2.4735473042818215, 2.054068595663089, 1.7350573342035134, 1.6656673071721346, 1.4956890564914902, 0.8851282095813073 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "apple/calendar.events.update", "builtin/ReadFile", "apple/contacts.update", "apple/reminders.update", "apple/location.reverseGeocode", "apple/location.geocode", "apple/battery.get", "apple/maps.search", "apple/calendar.events.delete", "apple/photos.search", "apple/clipboard.write" ], "scores": [ 15.610741474637624, 12.241698546352723, 4.010222501576509, 3.8077348880372774, 3.203146750471291, 3.050755223585493, 2.877556815390035, 2.606627671177044, 2.4735473042818215, 2.054068595663089, 1.7350573342035134, 1.6656673071721346, 1.4956890564914902 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N042", "query": "Remember that I prefer quiet restaurants for future conversations", "category": "intent", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Persist future-facing preference.", "acceptable": [ "builtin/Memory" ], "rationale": "Persist future-facing preference.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/alarms.create", "apple/links.open", "apple/alarms.cancel", "apple/alarms.list", "apple/calendar.events.list", "apple/clipboard.read", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/music.search", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 2.5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/alarms.create", "apple/health.workouts.get", "apple/health.workouts.list", "apple/links.open", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/clipboard.read", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 4.67652502244824, 3.1777733868287177, 3.048112898315442, 3.0437145865886266, 2.7156932419630486, 1.934298977912546, 1.7997415466286588, 1.7043370142720697, 1.6861137326635875, 1.6682760259384652, 1.584464330735021, 1.297625056165918, 0.8833160037757247 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/alarms.create", "apple/health.workouts.get", "apple/health.workouts.list", "apple/links.open", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/clipboard.read", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 6.680289744443825, 3.2721473302672357, 3.1263892000981492, 2.9567570064903284, 2.757843902510372, 1.9662731002711833, 1.7642134293155032, 1.6961419113447813, 1.6769433783228374, 1.6581745951752167, 1.570298655749525, 1.2733728264483664, 0.8546632305151596 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "apple/alarms.create", "builtin/Memory", "apple/health.workouts.get", "apple/health.workouts.list", "apple/links.open", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/clipboard.read", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 6.680289744443825, 4.716679997100231, 3.2721473302672357, 3.1263892000981492, 2.9567570064903284, 2.757843902510372, 1.9662731002711833, 1.7642134293155032, 1.6961419113447813, 1.6769433783228374, 1.6581745951752167, 1.570298655749525, 1.2733728264483664, 0.8546632305151596 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N043", "query": "Remove the outdated preference stored in my lasting memories", "category": "intent", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Memory delete is supported, ordinary file deletion is not.", "acceptable": [ "builtin/Memory" ], "rationale": "Memory delete is supported, ordinary file deletion is not.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 6.5, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 12.03780639320978, 1.5480217516112227 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 22.71192968896393, 1.4978074273357158 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory", "apple/contacts.delete", "apple/calendar.events.delete", "apple/reminders.delete", "builtin/ScheduledTasks" ], "scores": [ 36.812060498698806, 4.269497978854699, 4.131942557119925, 3.881812733740401, 1.4978074273357158 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N044", "query": "I have not told you which option I want; ask me to choose", "category": "intent", "domain": "questions", "intent": "capability:builtin/AskUserQuestions", "target_capability": "Collect missing choice through question card.", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Collect missing choice through question card.", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/AskUserQuestions", "apple/links.open", "apple/alarms.create", "apple/alarms.list", "apple/music.search", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/contacts.list", "apple/health.sleep.query", "apple/maps.search", "apple/photos.albums.list", "apple/photos.search", "apple/alarms.cancel", "apple/battery.get", "apple/clipboard.read", "apple/clipboard.write", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/maps.show", "apple/music.status", "apple/photos.get", "apple/reminders.create", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "apple/health.workouts.list", "apple/reminders.list" ], "scores": [ 5.5, 3, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/AskUserQuestions", "apple/alarms.list", "apple/music.search", "apple/shortcuts.run", "apple/alarms.create", "apple/calendar.events.list", "apple/links.open", "apple/clipboard.read", "apple/alarms.cancel", "apple/health.sleep.query", "apple/weather.current", "apple/photos.albums.list", "builtin/ScheduledTasks", "apple/clipboard.write", "apple/photos.search", "apple/health.statistics", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/contacts.list", "apple/maps.show", "apple/maps.search", "apple/weather.daily" ], "scores": [ 8.39528011322221, 2.9568323463287935, 2.800696056612951, 2.7491696581568283, 2.368074492613364, 1.934298977912546, 1.8609071452820887, 1.7043370142720697, 1.6682760259384652, 1.5629367438392368, 1.3848806590980578, 1.3627028549488005, 1.359961304742879, 1.3098389254907072, 1.297625056165918, 1.2432451371448472, 1.2162317258779303, 1.1119626517018855, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9834495738143223, 0.9297239873406339, 0.9098422129056576, 0.8549912766967913, 0.7495532564546209 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/AskUserQuestions", "apple/shortcuts.run", "apple/alarms.list", "apple/music.search", "apple/calendar.events.list", "apple/alarms.create", "apple/links.open", "apple/clipboard.read", "apple/alarms.cancel", "apple/health.sleep.query", "apple/photos.albums.list", "apple/photos.search", "apple/contacts.list", "builtin/ScheduledTasks", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/health.statistics", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/weather.daily", "apple/reminders.list", "apple/health.workouts.list" ], "scores": [ 18.31318055453427, 2.7231044752292686, 2.71720115433891, 2.563381729591779, 2.276181297253949, 2.196766475540141, 2.1921558552058418, 1.6961419113447813, 1.6581745951752167, 1.6244277159640166, 1.6193758131939588, 1.5572714341624867, 1.2500038811300134, 1.2336474431789102, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0093267431030897, 0.9930830738422539, 0.9207888081104977, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6063450614565641, 0.3147136525322586, 0.29633377370106745 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/AskUserQuestions", "apple/shortcuts.run", "apple/alarms.list", "apple/music.search", "apple/calendar.events.list", "apple/alarms.create", "apple/links.open", "apple/clipboard.read", "apple/alarms.cancel", "apple/health.sleep.query", "apple/photos.albums.list", "apple/photos.search", "apple/contacts.list", "builtin/ScheduledTasks", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/health.statistics", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/weather.daily", "apple/reminders.list", "apple/health.workouts.list" ], "scores": [ 33.01501544267522, 2.7231044752292686, 2.71720115433891, 2.563381729591779, 2.276181297253949, 2.196766475540141, 2.138688158803692, 1.6961419113447813, 1.6581745951752167, 1.6244277159640166, 1.6193758131939588, 1.5572714341624867, 1.2500038811300134, 1.2336474431789102, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0093267431030897, 0.9930830738422539, 0.9207888081104977, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6063450614565641, 0.3147136525322586, 0.29633377370106745 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N045", "query": "Tomorrow run a read-only weather check while I am away", "category": "intent", "domain": "scheduling", "intent": "capability:builtin/ScheduledTasks", "target_capability": "Once scheduled read task; best effort, not alarm.", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Once scheduled read task; best effort, not alarm.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ScheduledTasks", "apple/clipboard.read", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/shortcuts.run", "builtin/ReadFile", "apple/alarms.list", "apple/photos.search", "apple/links.open", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.list", "apple/music.search", "apple/photos.get", "apple/music.playback", "builtin/Memory", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/contacts.get", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.eta", "apple/maps.show", "apple/music.status", "apple/photos.albums.list", "builtin/EditFile" ], "scores": [ 7, 5, 5, 5, 5, 4, 4, 3, 3, 2.5, 2, 2, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ScheduledTasks", "apple/weather.current", "apple/shortcuts.run", "apple/weather.hourly", "apple/weather.daily", "apple/alarms.list", "apple/photos.search", "apple/music.search", "apple/clipboard.read", "apple/calendar.events.list", "apple/alarms.cancel", "apple/calendar.events.create", "apple/alarms.create", "apple/links.open", "apple/photos.get", "apple/music.status", "builtin/ReadFile", "apple/photos.albums.list", "builtin/Memory", "apple/contacts.get", "builtin/EditFile", "apple/maps.show", "apple/health.workouts.get", "apple/health.workouts.list", "apple/calendar.events.update", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "apple/music.playback", "apple/health.statistics" ], "scores": [ 9.518772838797723, 5.6277546708809, 4.453668907514033, 4.224192292580948, 3.98899528478155, 3.925442239404573, 3.5161216696380766, 3.301144726824801, 3.0483120720834562, 3.029994168149062, 2.9389093624971165, 2.722090979800692, 2.709587567906209, 2.51290816131581, 2.1043278842905497, 1.892501474377823, 1.7356930089104854, 1.634894371444502, 1.6208496848238496, 1.596129569605437, 1.3632427748418257, 1.2842193286009138, 1.2665935090803384, 1.2149135706002092, 1.1274973225408556, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 0.9418286545359099, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/weather.current", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/weather.hourly", "apple/weather.daily", "apple/clipboard.read", "builtin/ReadFile", "apple/alarms.list", "apple/photos.search", "apple/links.open", "apple/music.search", "apple/calendar.events.list", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.create", "apple/photos.get", "builtin/Memory", "apple/music.status", "apple/music.playback", "apple/contacts.get", "apple/photos.albums.list", "builtin/EditFile", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.show", "apple/health.sleep.query", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/calendar.events.update", "apple/health.statistics" ], "scores": [ 12.618358951393477, 11.18130140095132, 10.960898197569428, 10.696863355591796, 10.480184724196864, 5.750817050303326, 4.557289964100901, 3.8410522123748727, 3.385039957096879, 3.285320266792436, 3.191844593653075, 3.0800802227789976, 2.8587792774414713, 2.6238452508644228, 2.4284016484087565, 2.0264829671269666, 1.9987894289795634, 1.798233682123699, 1.621653562440418, 1.60914410629741, 1.5512004698252941, 1.3906556055226948, 1.3042089742613945, 1.2461128550311649, 1.2141942578109772, 1.0992184654999453, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 1.0689954759741835, 0.7031782969374535 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/shortcuts.run", "apple/weather.current", "builtin/ScheduledTasks", "apple/weather.hourly", "apple/weather.daily", "apple/clipboard.read", "apple/alarms.list", "builtin/ReadFile", "apple/calendar.events.list", "apple/links.open", "apple/music.search", "apple/photos.search", "apple/alarms.cancel", "apple/photos.get", "apple/alarms.create", "apple/photos.albums.list", "apple/calendar.events.create", "apple/contacts.get", "apple/health.workouts.get", "apple/device.get", "apple/music.status", "apple/battery.get", "apple/health.workouts.list", "apple/music.playback", "builtin/Memory", "apple/maps.show", "apple/calendar.events.update", "builtin/EditFile", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/calendar.list", "apple/reminders.list", "apple/health.sleep.query", "apple/maps.eta", "apple/health.statistics" ], "scores": [ 17.25493363643592, 12.258402314041176, 10.6250040622497, 10.452389665832632, 10.253762119824152, 4.565550721803088, 4.540051756150362, 4.201778249894578, 3.657989814316527, 3.285320266792436, 3.191844593653075, 3.1741860690965105, 2.8587792774414713, 2.7702425710709373, 2.6238452508644228, 2.477843028786731, 2.4284016484087565, 2.1425381623194113, 1.8264601231288218, 1.8002653409399807, 1.798233682123699, 1.7904649384226818, 1.7223681387479477, 1.621653562440418, 1.5278191715440528, 1.2141942578109772, 1.0689954759741835, 0.9841378926907047, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.7778939227999769, 0.7580363862218745, 0.4976245769158341 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } } } }, { "query_id": "N046", "query": "Put a thirty minute meeting on my calendar", "category": "intent", "domain": "calendar", "intent": "capability:apple/calendar.events.create", "target_capability": "Create timed non-recurring event.", "acceptable": [ "apple/calendar.events.create" ], "rationale": "Create timed non-recurring event.", "basis": [ "apple/calendar.events.create" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.list", "apple/calendar.events.delete", "apple/links.open", "builtin/ScheduledTasks", "apple/health.statistics", "apple/weather.daily" ], "scores": [ 4.5, 4.5, 4, 4, 4, 3, 2, 2, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.list", "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 3.353990223330057, 3.3063649324302986, 3.105383741638209, 3.0094803210198413, 2.2811099463360556, 1.9146581234509097, 1.802365165051345, 1.4761555867694751, 1.3457586502029633 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "bm25": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.list", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 8.29960284151429, 8.117821392761574, 7.725956296832728, 6.834728301526837, 5.949092692075655, 4.55597709755589, 3.486659142444121, 1.290252793313428, 1.1713329690232368, 0.6658726147629157 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.list", "apple/calendar.list", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/alarms.create", "apple/contacts.create", "builtin/ScheduledTasks", "apple/reminders.create", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 11.676494269469377, 8.29960284151429, 7.725956296832728, 6.834728301526837, 5.949092692075655, 4.55597709755589, 3.881812733740401, 3.6602378602162426, 3.486659142444121, 3.4625922028793776, 1.290252793313428, 1.1713329690232368, 1.142400969046904 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N047", "query": "Add a new item to my shopping reminder list", "category": "intent", "domain": "reminders", "intent": "capability:apple/reminders.create", "target_capability": "Creates reminder, not calendar event.", "acceptable": [ "apple/reminders.create" ], "rationale": "Creates reminder, not calendar event.", "basis": [ "apple/reminders.create" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/reminders.list", "apple/reminders.lists", "apple/reminders.create", "apple/contacts.list", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.update", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/photos.albums.list", "builtin/ListFiles", "apple/health.workouts.list", "builtin/Memory", "apple/links.open", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/clipboard.write", "apple/alarms.create", "apple/calendar.events.get", "apple/health.sleep.query", "apple/location.geocode", "apple/shortcuts.run", "apple/calendar.events.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get", "apple/photos.search", "builtin/EditFile" ], "scores": [ 9.5, 8, 5.5, 5, 4.5, 4.5, 4.5, 4.5, 4, 4, 4, 4, 4, 3, 3, 2.5, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "name_description": { "ranking": [ "apple/reminders.list", "apple/reminders.lists", "builtin/Memory", "apple/reminders.create", "apple/clipboard.write", "apple/contacts.list", "apple/reminders.get", "apple/reminders.complete", "apple/health.sleep.query", "apple/reminders.delete", "apple/reminders.update", "apple/location.geocode", "builtin/ListFiles", "apple/calendar.list", "apple/calendar.events.list", "apple/alarms.list", "apple/calendar.events.get", "apple/alarms.create", "apple/photos.albums.list", "apple/health.workouts.list", "apple/calendar.events.delete", "apple/alarms.cancel", "apple/shortcuts.run", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 7.05208628552052, 5.7081218008616155, 4.4343270643050205, 4.343184910384773, 3.947200460209589, 3.941140567620505, 3.612813620113822, 3.4062280126487074, 3.348804949568758, 3.2512998801244857, 3.1371483815404533, 2.8181396909485716, 1.9903435646574756, 1.970426045916842, 1.614897263876673, 1.4702032485297019, 1.457997372644377, 1.3776271994526728, 1.340122705366102, 1.3390180568934824, 1.1248369805109821, 1.0415379820610877, 0.9793722303182939, 0.9242094098357349, 0.899392122924025 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "bm25": { "ranking": [ "apple/reminders.list", "apple/reminders.lists", "apple/reminders.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.update", "apple/contacts.list", "apple/clipboard.write", "builtin/Memory", "builtin/ListFiles", "apple/calendar.list", "apple/alarms.list", "apple/calendar.events.list", "apple/health.sleep.query", "apple/photos.albums.list", "apple/location.geocode", "apple/calendar.events.update", "apple/health.workouts.list", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/alarms.cancel", "builtin/EditFile", "apple/calendar.events.get", "apple/alarms.create", "apple/links.open", "apple/shortcuts.run", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/calendar.events.create", "apple/photos.search" ], "scores": [ 14.113624751150606, 12.261515585776756, 9.605636229394815, 9.285244357925471, 9.130891830020635, 9.008117454833116, 8.373833017913912, 5.783088009758299, 5.437015511024541, 5.3036522000472965, 3.8392072769683265, 3.8153957247318555, 3.3265336457761996, 3.1661756288646488, 3.032739008254341, 2.9379072248550164, 2.8320318668461497, 2.760573322472673, 2.2499293125845314, 2.1651437278063, 1.4175588968227548, 1.3736294807719625, 1.2442452235478334, 1.1253412868673076, 1.0204469850336544, 0.995888282467371, 0.724055318310476, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.30692226461244376, 0.29833593472755854 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "combined": { "ranking": [ "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/alarms.create", "apple/calendar.events.create", "apple/contacts.list", "apple/contacts.create", "apple/clipboard.write", "builtin/ListFiles", "builtin/Memory", "apple/calendar.list", "apple/alarms.list", "apple/calendar.events.list", "apple/photos.albums.list", "apple/location.geocode", "builtin/WriteFile", "apple/health.workouts.list", "apple/health.sleep.query", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/links.open", "apple/calendar.events.delete", "apple/alarms.cancel", "apple/calendar.events.get", "builtin/EditFile", "apple/shortcuts.run", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/contacts.update", "apple/photos.search" ], "scores": [ 14.358324248611579, 14.113624751150606, 12.261515585776756, 9.605636229394815, 9.285244357925471, 9.130891830020635, 8.373833017913912, 7.01841025745229, 5.805587624617734, 5.783088009758299, 5.655597979539574, 5.437015511024541, 5.039330650712083, 4.091248563709154, 3.8153957247318555, 3.3265336457761996, 3.1661756288646488, 2.9379072248550164, 2.8320318668461497, 2.6610014176998558, 2.2499293125845314, 2.1990878262180744, 2.1901641256863784, 2.1651437278063, 1.421134674234693, 1.4175588968227548, 1.3736294807719625, 1.1253412868673076, 0.8460531825275042, 0.724055318310476, 0.6009512783548976, 0.6009512783548976, 0.5844827580213056, 0.3914103754917451, 0.29833593472755854 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N048", "query": "Add a new person and their email to my contacts", "category": "intent", "domain": "contacts", "intent": "capability:apple/contacts.create", "target_capability": "Creates contact with name/email.", "acceptable": [ "apple/contacts.create" ], "rationale": "Creates contact with name/email.", "basis": [ "apple/contacts.create" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "builtin/Memory", "apple/calendar.list", "apple/health.sleep.query", "builtin/AskUserQuestions", "apple/calendar.events.update", "apple/links.open", "builtin/EditFile", "builtin/ScheduledTasks" ], "scores": [ 6, 5, 4.5, 4.5, 4.5, 1.5, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.update", "apple/calendar.list", "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete", "apple/health.sleep.query", "builtin/AskUserQuestions", "builtin/Memory" ], "scores": [ 6.610381154683383, 5.981213781756763, 4.642751813673299, 4.062325774173696, 3.908385007994482, 3.6211209622787783, 3.348804949568758, 3.348804949568758, 3.269566973945597 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "bm25": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "apple/contacts.get", "apple/contacts.delete", "apple/calendar.list", "builtin/Memory", "builtin/AskUserQuestions", "apple/health.sleep.query", "apple/calendar.events.update", "builtin/EditFile", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 14.776530028321321, 12.205249253105848, 12.117106122803115, 11.963738092095124, 11.146694165037609, 4.777427771816066, 4.194943214114938, 3.400782226857424, 3.032739008254341, 1.7823784642970022, 1.2442452235478334, 0.8652613463947955, 0.290204573051431 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/contacts.create", "apple/contacts.update", "apple/contacts.list", "apple/contacts.get", "apple/contacts.delete", "apple/alarms.create", "apple/calendar.events.create", "apple/reminders.create", "apple/calendar.list", "builtin/AskUserQuestions", "builtin/Memory", "builtin/WriteFile", "apple/health.sleep.query", "apple/calendar.events.update", "builtin/ScheduledTasks", "builtin/EditFile", "apple/links.open" ], "scores": [ 17.772704102342686, 14.776530028321321, 12.205249253105848, 11.963738092095124, 11.146694165037609, 5.997963272418636, 5.49866536000529, 5.350206793778464, 4.777427771816066, 3.400782226857424, 2.982539577776796, 2.6610014176998558, 2.1990878262180744, 1.2119692675107079, 0.8652613463947955, 0.8460531825275042, 0.21043200287701586 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N049", "query": "How long would driving between these two coordinates take?", "category": "intent", "domain": "maps", "intent": "capability:apple/maps.eta|apple/maps.route", "target_capability": "Both return travel-time estimate; detailed steps not required.", "acceptable": [ "apple/maps.eta", "apple/maps.route" ], "rationale": "Both return travel-time estimate; detailed steps not required.", "basis": [ "apple/maps.eta", "apple/maps.route" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/maps.eta", "apple/maps.route", "apple/health.statistics", "apple/links.open", "apple/location.reverseGeocode", "apple/maps.show", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 3.5, 2.5, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/maps.eta", "apple/maps.route", "apple/health.statistics", "apple/location.reverseGeocode", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "apple/links.open" ], "scores": [ 8.75697438307545, 5.909868546922292, 2.583955380200963, 2.430546544538726, 1.9107839181010244, 1.726212147301921, 1.6934955774573934, 1.5741568020017251, 1.1344628597433402 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.eta", "apple/maps.route", "apple/health.statistics", "apple/location.reverseGeocode", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "apple/links.open" ], "scores": [ 10.802085773322952, 7.908922140557247, 2.5329463505701764, 2.49296358785834, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.5495958966369607, 1.1020517573918405 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.eta", "apple/maps.route", "builtin/Memory", "apple/health.statistics", "apple/location.reverseGeocode", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "apple/links.open" ], "scores": [ 10.802085773322952, 7.908922140557247, 4.716679997100231, 2.5329463505701764, 2.49296358785834, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.5495958966369607, 1.1020517573918405 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N050", "query": "Open the supported app using its registered URL", "category": "intent", "domain": "links", "intent": "capability:apple/links.open", "target_capability": "Hands off supported URL/app; not arbitrary UI operation.", "acceptable": [ "apple/links.open" ], "rationale": "Hands off supported URL/app; not arbitrary UI operation.", "basis": [ "apple/links.open" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/alarms.create", "apple/calendar.events.get", "apple/clipboard.read", "apple/contacts.list", "apple/health.workouts.get", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/music.status", "apple/shortcuts.run", "builtin/ScheduledTasks" ], "scores": [ 10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/links.open", "apple/maps.show", "apple/contacts.list", "apple/calendar.events.get", "apple/health.workouts.get", "apple/maps.eta", "apple/maps.route", "apple/clipboard.read", "apple/music.status", "apple/maps.search", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 13.461476709232656, 4.126968314047327, 3.5156165905658834, 3.3189549419353934, 3.1777733868287177, 2.950632574157707, 2.6351421055956448, 2.4221849112268674, 2.38399622539939, 2.251823179421304, 2.0519806739118946, 2.01195188835184, 1.1554436837598263 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/links.open", "apple/maps.show", "apple/contacts.list", "apple/calendar.events.get", "apple/health.workouts.get", "apple/maps.eta", "apple/maps.route", "apple/clipboard.read", "apple/music.status", "apple/maps.search", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 25.203807987684797, 4.11126841487463, 3.50097963484709, 3.432161121074587, 3.2721473302672357, 2.955321350109538, 2.620810221347025, 2.410538121601221, 2.382325764389142, 2.2316910788332835, 2.0325256171929906, 1.9907674591623792, 1.117963704064442 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/links.open", "apple/maps.show", "builtin/ReadFile", "apple/contacts.list", "apple/calendar.events.get", "apple/health.workouts.get", "apple/maps.eta", "apple/maps.route", "apple/clipboard.read", "apple/music.status", "apple/maps.search", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 24.53320437986201, 8.210138614943588, 3.8077348880372774, 3.50097963484709, 3.432161121074587, 3.2721473302672357, 2.955321350109538, 2.620810221347025, 2.410538121601221, 2.382325764389142, 2.2316910788332835, 2.0325256171929906, 1.9907674591623792, 1.117963704064442 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N051", "query": "Run my saved shortcut by its exact name", "category": "intent", "domain": "shortcuts", "intent": "capability:apple/shortcuts.run", "target_capability": "Executes named shortcut handoff.", "acceptable": [ "apple/shortcuts.run" ], "rationale": "Executes named shortcut handoff.", "basis": [ "apple/shortcuts.run" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/shortcuts.run", "apple/health.workouts.get", "apple/contacts.create", "apple/contacts.update", "apple/links.open", "apple/maps.search", "apple/maps.show", "builtin/Memory", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.update", "apple/clipboard.read", "apple/contacts.delete", "apple/contacts.list", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.list", "apple/music.playback", "apple/photos.albums.list", "builtin/ScheduledTasks", "apple/clipboard.write", "builtin/EditFile" ], "scores": [ 12, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/shortcuts.run", "apple/health.workouts.get", "apple/calendar.events.get", "apple/device.get", "builtin/Memory", "builtin/ScheduledTasks", "apple/contacts.create", "apple/clipboard.read", "apple/maps.search", "apple/photos.albums.list", "apple/contacts.list", "apple/maps.show", "apple/contacts.update", "apple/contacts.delete", "apple/health.workouts.list", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/calendar.events.create", "apple/links.open", "apple/calendar.events.update", "apple/health.statistics", "apple/music.playback" ], "scores": [ 15.430263944866173, 5.2876262392249815, 3.3189549419353934, 3.114418360186576, 2.6514356819835214, 2.5246596325529898, 2.4692163806821252, 2.4221849112268674, 2.251823179421304, 2.1038511081773286, 2.0915170203879607, 2.04679076808892, 2.0251373927906937, 2.0237659864584026, 2.0237659864584026, 1.8030590716561463, 1.7000568511749237, 1.4656245762044289, 1.4227132918780445, 1.3968322222103842, 1.1949215294801032, 1.1668112962717396 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/shortcuts.run", "apple/health.workouts.get", "apple/contacts.create", "apple/maps.show", "apple/contacts.update", "apple/calendar.events.get", "apple/maps.search", "apple/device.get", "builtin/Memory", "builtin/ScheduledTasks", "apple/clipboard.read", "apple/contacts.list", "apple/health.workouts.list", "apple/contacts.delete", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/photos.albums.list", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/clipboard.write", "apple/health.statistics", "apple/music.playback", "builtin/EditFile" ], "scores": [ 37.31268876402051, 5.444658877767731, 4.00922799620642, 3.858980236745431, 3.4385279545186087, 3.432161121074587, 3.2390451265040827, 3.119415169082174, 2.696335164237074, 2.4574770197500073, 2.410538121601221, 2.0828091760528666, 2.0757368032812122, 2.0453664522045925, 1.8310446076149913, 1.7184999580546296, 1.6731343036002873, 1.4661702610544265, 1.393115594519765, 1.3124837602688564, 1.1926532872089062, 1.1713329690232368, 1.1426115024435417, 0.9022222210637898 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/shortcuts.run", "apple/health.workouts.get", "apple/contacts.create", "apple/maps.show", "apple/contacts.update", "apple/calendar.events.get", "apple/maps.search", "apple/device.get", "builtin/Memory", "builtin/ScheduledTasks", "apple/clipboard.read", "apple/contacts.list", "apple/health.workouts.list", "apple/contacts.delete", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/photos.albums.list", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/clipboard.write", "apple/health.statistics", "apple/music.playback", "builtin/EditFile" ], "scores": [ 43.3863209995051, 5.444658877767731, 4.00922799620642, 3.858980236745431, 3.4385279545186087, 3.432161121074587, 3.2390451265040827, 3.119415169082174, 2.696335164237074, 2.4574770197500073, 2.410538121601221, 2.0828091760528666, 2.0757368032812122, 2.0453664522045925, 1.8310446076149913, 1.7184999580546296, 1.6731343036002873, 1.4661702610544265, 1.393115594519765, 1.3124837602688564, 1.1926532872089062, 1.1713329690232368, 1.1426115024435417, 0.9022222210637898 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N052", "query": "Show recorded running workouts", "category": "intent", "domain": "health", "intent": "capability:apple/health.workouts.list", "target_capability": "Lists workouts with running activityType.", "acceptable": [ "apple/health.workouts.list" ], "rationale": "Lists workouts with running activityType.", "basis": [ "apple/health.workouts.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/health.workouts.list", "apple/health.workouts.get", "apple/maps.show", "apple/clipboard.read", "builtin/ScheduledTasks" ], "scores": [ 6.5, 5.5, 4, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/health.workouts.list", "apple/health.workouts.get", "apple/maps.show", "apple/clipboard.read" ], "scores": [ 12.98645727077551, 8.90126116611652, 4.126968314047327, 2.986870790172134 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/health.workouts.list", "apple/health.workouts.get", "apple/maps.show", "apple/clipboard.read", "builtin/ScheduledTasks" ], "scores": [ 20.6915373398184, 17.56590315724968, 12.510782572457543, 2.9725087752942105, 1.1139812831763996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/health.workouts.list", "apple/health.workouts.get", "apple/maps.show", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/clipboard.read", "builtin/ScheduledTasks" ], "scores": [ 22.78828227147008, 17.56590315724968, 7.1387720841183455, 2.4364052592946104, 2.311584248804978, 2.311584248804978, 2.253850044094424, 2.253850044094424, 2.19892950792101, 1.6961419113447813, 1.1139812831763996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N053", "query": "env path offset limit UTF-8", "category": "schema", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "offset/limit plus text identify line reader.", "acceptable": [ "builtin/ReadFile" ], "rationale": "offset/limit plus text identify line reader.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "apple/photos.search", "builtin/WriteFile", "apple/maps.search", "apple/music.search", "apple/weather.daily", "apple/weather.hourly", "builtin/ListFiles", "apple/calendar.events.list", "apple/contacts.list", "apple/health.sleep.query", "apple/health.workouts.list", "apple/photos.albums.list", "apple/reminders.list", "builtin/EditFile", "builtin/GetCurrentTime", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.update", "apple/health.statistics", "apple/links.open", "builtin/Memory", "builtin/ScheduledTasks" ], "scores": [ 4, 3, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "apple/photos.search", "builtin/GetCurrentTime", "apple/maps.search", "apple/music.search", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 8.839979462997334, 8.700822527580458, 4.921846759628811, 4.03298001739044, 2.251823179421304, 2.251823179421304, 2.1237880527269173, 1.9741270386011305 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "apple/photos.search", "builtin/ListFiles", "builtin/EditFile", "apple/weather.hourly", "apple/music.search", "apple/weather.daily", "apple/photos.albums.list", "apple/maps.search", "apple/contacts.list", "apple/health.sleep.query", "builtin/GetCurrentTime", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.list", "builtin/Memory", "apple/health.statistics", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 13.023618087746556, 11.424500575669835, 4.069381215516833, 3.6048283689702902, 2.4130569909379647, 2.2799263547847666, 2.265119076735111, 2.192506367203581, 2.00895928936296, 1.9720989474975124, 1.9705323619926771, 1.8289829736906624, 1.8098642771568842, 1.7557848473388846, 1.7046673453637797, 1.3752720568765606, 1.1752925005508255, 0.7751595373426047, 0.7663297824208364, 0.7501397165950441, 0.7191887155352337, 0.21043200287701586, 0.186931570207364 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "apple/photos.search", "builtin/ListFiles", "builtin/EditFile", "apple/weather.hourly", "apple/music.search", "apple/weather.daily", "apple/photos.albums.list", "apple/maps.search", "apple/contacts.list", "apple/health.sleep.query", "builtin/GetCurrentTime", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.list", "builtin/Memory", "apple/health.statistics", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 13.023618087746556, 11.424500575669835, 4.069381215516833, 3.6048283689702902, 2.4130569909379647, 2.2799263547847666, 2.265119076735111, 2.192506367203581, 2.00895928936296, 1.9720989474975124, 1.9705323619926771, 1.8289829736906624, 1.8098642771568842, 1.7557848473388846, 1.7046673453637797, 1.3752720568765606, 1.1752925005508255, 0.7751595373426047, 0.7663297824208364, 0.7501397165950441, 0.7191887155352337, 0.21043200287701586, 0.186931570207364 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N054", "query": "oldText newText unique nonempty text segment", "category": "schema", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "Exact edit schema fields.", "acceptable": [ "builtin/EditFile" ], "rationale": "Exact edit schema fields.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "apple/shortcuts.run", "builtin/Memory", "apple/clipboard.read", "apple/contacts.update", "builtin/AskUserQuestions", "builtin/ReadFile", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "apple/clipboard.read", "builtin/ReadFile", "builtin/AskUserQuestions", "apple/contacts.update", "apple/shortcuts.run" ], "scores": [ 12.548474202548531, 3.147473456270661, 3.0929979541428994, 2.8581499673178334, 2.165477147820978, 1.8905693842378057, 1.7777279238039982 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/EditFile", "apple/clipboard.write", "builtin/Memory", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ReadFile", "builtin/AskUserQuestions", "apple/calendar.events.update", "apple/contacts.update", "apple/links.open" ], "scores": [ 17.706413405590315, 4.561302494397273, 3.6077853228815506, 2.972996695597959, 2.89297795482389, 2.745409276253241, 2.062180702926565, 1.7823784642970022, 1.762134739525646, 0.3254228443330937 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.write", "builtin/Memory", "apple/alarms.create", "apple/shortcuts.run", "apple/contacts.create", "apple/clipboard.read", "apple/calendar.events.create", "apple/reminders.create", "builtin/AskUserQuestions", "apple/contacts.update", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 19.50093070754454, 5.165012518316896, 5.019844432888661, 4.2922006742543, 3.1075117237621663, 2.9026295885011453, 2.797599684974972, 2.7369467418327798, 2.72230178628807, 2.6610014176998558, 2.5891569919465325, 1.9405188352247646, 1.6581745951752167, 1.2119692675107079, 0.3254228443330937 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N055", "query": "overwrite content path UTF-8", "category": "schema", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "overwrite is explicit full-file write option.", "acceptable": [ "builtin/WriteFile" ], "rationale": "overwrite is explicit full-file write option.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.write", "apple/music.playback", "apple/photos.search", "builtin/Memory", "apple/links.open", "builtin/EditFile", "builtin/ListFiles" ], "scores": [ 3.5, 2.5, 2, 1, 1, 1, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "apple/clipboard.write", "apple/photos.search", "apple/music.playback" ], "scores": [ 8.839979462997334, 8.700822527580458, 3.0869611235447083, 2.647743489910853, 2.167107836671426 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.write", "builtin/Memory", "apple/music.playback", "builtin/ListFiles", "apple/photos.search", "builtin/EditFile", "apple/links.open" ], "scores": [ 13.3718890808886, 10.420984653644659, 5.836343395449581, 2.3694474481902263, 1.720954340406063, 1.4622381665831385, 1.44304638019922, 1.2815932897590763, 0.21043200287701586 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.write", "builtin/Memory", "apple/music.playback", "builtin/ListFiles", "apple/photos.search", "builtin/EditFile", "apple/links.open" ], "scores": [ 16.514479409568374, 14.921868801429977, 5.518619129038152, 2.274403483950976, 1.5839818954898712, 1.4622381665831385, 1.44304638019922, 1.2815932897590763, 0.21043200287701586 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N056", "query": "directory level limit 200", "category": "schema", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "One level directory listing with bounded item limit.", "acceptable": [ "builtin/ListFiles" ], "rationale": "One level directory listing with bounded item limit.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ListFiles", "apple/maps.search", "apple/music.search", "apple/weather.daily", "apple/weather.hourly", "apple/battery.get", "builtin/Memory", "apple/calendar.events.list", "apple/contacts.list", "apple/health.sleep.query", "apple/health.workouts.list", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.list", "builtin/ReadFile" ], "scores": [ 2.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ListFiles", "apple/battery.get", "builtin/Memory", "apple/maps.search", "apple/music.search", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 9.4441616976102, 4.459978227724632, 3.269566973945597, 2.251823179421304, 2.251823179421304, 2.1237880527269173, 1.9741270386011305 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ListFiles", "apple/battery.get", "builtin/Memory", "apple/weather.hourly", "apple/music.search", "apple/weather.daily", "apple/maps.search", "apple/photos.albums.list", "apple/contacts.list", "apple/health.sleep.query", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/photos.search", "builtin/ReadFile" ], "scores": [ 10.506839613788287, 4.46475237675939, 3.2314784752437844, 2.2799263547847666, 2.265119076735111, 2.192506367203581, 1.9720989474975124, 1.1479996505587697, 1.130323284158789, 0.8641940862618136, 0.8347146518534616, 0.8253300840178237, 0.7984011807425733, 0.7731740273870537, 0.6992578404944915 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ListFiles", "apple/battery.get", "builtin/Memory", "apple/weather.hourly", "apple/music.search", "apple/weather.daily", "apple/maps.search", "apple/photos.albums.list", "apple/contacts.list", "apple/health.sleep.query", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/photos.search", "builtin/ReadFile" ], "scores": [ 15.020391613509284, 4.46475237675939, 3.2314784752437844, 2.2799263547847666, 2.265119076735111, 2.192506367203581, 1.9720989474975124, 1.1479996505587697, 1.130323284158789, 0.8641940862618136, 0.8347146518534616, 0.8253300840178237, 0.7984011807425733, 0.7731740273870537, 0.6992578404944915 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N057", "query": "command view create edit delete Markdown", "category": "schema", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Memory command enum and Markdown scope.", "acceptable": [ "builtin/Memory" ], "rationale": "Memory command enum and Markdown scope.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/contacts.create", "apple/contacts.delete", "apple/reminders.create", "apple/reminders.delete", "builtin/EditFile", "builtin/ScheduledTasks", "builtin/WriteFile" ], "scores": [ 8, 4, 4, 4, 4, 4, 4, 4, 3, 3, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/EditFile", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "builtin/WriteFile", "apple/alarms.create" ], "scores": [ 15.437739036410166, 4.217625472171536, 3.951705758579185, 3.752882783565409, 3.6211209622787783, 3.224841952905041, 3.05666003761185, 3.030740607742593, 2.9403271227683874, 2.8131576240574634, 2.5006287346568747 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory", "apple/reminders.delete", "apple/contacts.delete", "builtin/EditFile", "apple/calendar.events.delete", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.create", "builtin/ScheduledTasks", "builtin/WriteFile" ], "scores": [ 23.309764558320083, 10.156196128710354, 9.97993715650646, 9.571919453519563, 8.698195352050329, 8.542801847641005, 8.510275502348785, 7.990654771663212, 7.914298923247929, 4.454103609202588, 2.8774990349848864 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory", "builtin/EditFile", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.create", "builtin/WriteFile", "builtin/ScheduledTasks", "builtin/ReadFile", "apple/contacts.update", "apple/calendar.events.update", "apple/reminders.update" ], "scores": [ 21.160150883865164, 10.33148330700663, 10.156196128710354, 9.97993715650646, 8.698195352050329, 8.542801847641005, 8.510275502348785, 7.990654771663212, 7.914298923247929, 5.715162977290321, 4.454103609202588, 4.26982905438862, 2.9482051569982852, 2.876364970293838, 2.807942621296027 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N058", "query": "draft kind weekly weekdays timeZone requiredCapabilities", "category": "schema", "domain": "scheduling", "intent": "capability:builtin/ScheduledTasks", "target_capability": "Scheduled task draft weekly/requiredCapabilities schema.", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Scheduled task draft weekly/requiredCapabilities schema.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ScheduledTasks", "builtin/GetCurrentTime", "apple/links.open", "apple/reminders.list", "apple/alarms.cancel", "apple/calendar.events.list", "apple/calendar.events.update", "apple/maps.eta", "apple/maps.route", "apple/weather.current", "apple/weather.hourly", "apple/alarms.create", "builtin/Memory", "apple/calendar.events.create", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.list", "apple/location.get", "apple/music.status", "apple/reminders.create", "apple/reminders.update", "apple/music.playback" ], "scores": [ 7, 6, 4.5, 3.5, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "builtin/ScheduledTasks", "apple/reminders.list", "apple/maps.eta", "apple/weather.current", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/links.open", "apple/location.get", "builtin/Memory", "apple/calendar.events.list", "apple/music.status", "apple/calendar.events.update", "apple/alarms.create" ], "scores": [ 11.717831244157203, 7.111693082571059, 4.645263111511694, 4.289317969242983, 3.9038450391689175, 3.2771054828991106, 3.242436382364059, 2.904443348919582, 2.56559844863368, 2.4208697806788355, 1.865646158070287, 1.8251756430963588, 1.7197049516029317, 1.4803458231303015, 1.4513293217173118 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "builtin/GetCurrentTime", "builtin/ScheduledTasks", "apple/reminders.list", "apple/links.open", "apple/weather.current", "apple/maps.eta", "apple/calendar.events.list", "apple/calendar.events.update", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "builtin/Memory", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/location.get", "apple/alarms.create", "apple/health.statistics", "apple/health.workouts.list", "apple/calendar.events.create", "apple/music.status", "apple/music.playback" ], "scores": [ 11.14346153121252, 10.42714231666909, 4.6073825986826185, 4.183368132326216, 3.69852345925528, 3.277871957729798, 2.78847515074097, 2.7494930109062903, 2.543296170132332, 2.5148309429133135, 2.239157435018262, 2.123063961142903, 1.8284761827757008, 1.8284761827757008, 1.6343925455007966, 1.6043303491322136, 1.578316724633953, 1.5435378962158324, 1.5099628183863403, 1.4937171307863306, 1.1370398010457616, 0.9179745018243874 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/GetCurrentTime", "builtin/ScheduledTasks", "apple/links.open", "apple/reminders.list", "apple/weather.current", "apple/maps.eta", "apple/calendar.events.list", "apple/calendar.events.update", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "builtin/Memory", "apple/reminders.create", "apple/reminders.update", "apple/health.sleep.query", "apple/location.get", "apple/alarms.create", "apple/health.statistics", "apple/health.workouts.list", "apple/calendar.events.create", "apple/music.status", "apple/music.playback" ], "scores": [ 17.264934480995226, 15.398513772105158, 5.068419840571305, 4.6073825986826185, 3.69852345925528, 3.277871957729798, 2.78847515074097, 2.7494930109062903, 2.543296170132332, 2.5148309429133135, 2.239157435018262, 2.123063961142903, 1.8284761827757008, 1.8284761827757008, 1.6343925455007966, 1.6043303491322136, 1.578316724633953, 1.5435378962158324, 1.5099628183863403, 1.4937171307863306, 1.1370398010457616, 0.9179745018243874 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N059", "query": "questions options free text question card", "category": "schema", "domain": "questions", "intent": "capability:builtin/AskUserQuestions", "target_capability": "Question card schema allows free text.", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Question card schema allows free text.", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/AskUserQuestions", "apple/clipboard.write", "apple/shortcuts.run", "builtin/EditFile", "apple/clipboard.read", "apple/contacts.update", "builtin/ReadFile", "builtin/Memory" ], "scores": [ 9, 1.5, 1.5, 1.5, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/AskUserQuestions", "apple/clipboard.write", "apple/clipboard.read", "builtin/ReadFile", "builtin/EditFile", "apple/contacts.update", "apple/shortcuts.run" ], "scores": [ 19.09754470411657, 3.147473456270661, 3.0929979541428994, 2.8581499673178334, 2.7272927852161843, 1.8905693842378057, 1.7777279238039982 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/AskUserQuestions", "apple/clipboard.write", "builtin/EditFile", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ReadFile", "apple/contacts.update", "builtin/Memory" ], "scores": [ 33.30829074446245, 4.561302494397273, 4.1306699092355466, 2.972996695597959, 2.89297795482389, 2.745409276253241, 1.762134739525646, 1.3563273996917304 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/AskUserQuestions", "builtin/EditFile", "builtin/ReadFile", "apple/clipboard.write", "apple/shortcuts.run", "apple/clipboard.read", "builtin/WriteFile", "apple/contacts.update", "builtin/Memory" ], "scores": [ 49.41521056065254, 6.323379252210103, 5.019844432888661, 4.2922006742543, 2.797599684974972, 2.72230178628807, 2.5040111006170407, 1.6581745951752167, 1.276308551475644 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N060", "query": "calendarIds start end timeZone offset", "category": "schema", "domain": "calendar", "intent": "capability:apple/calendar.events.list", "target_capability": "Calendar list range/filter/pagination fields.", "acceptable": [ "apple/calendar.events.list" ], "rationale": "Calendar list range/filter/pagination fields.", "basis": [ "apple/calendar.events.list" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.create", "builtin/GetCurrentTime", "apple/calendar.events.get", "apple/health.statistics", "apple/photos.search", "apple/calendar.list", "apple/reminders.list", "apple/calendar.events.delete", "apple/health.sleep.query", "apple/health.workouts.list", "apple/alarms.create", "apple/links.open", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/location.get", "apple/maps.eta", "apple/maps.route", "apple/maps.show", "apple/music.status", "apple/reminders.create", "apple/reminders.update", "apple/weather.daily", "apple/weather.hourly", "apple/contacts.list", "apple/photos.albums.list", "builtin/Memory", "builtin/ReadFile" ], "scores": [ 8.5, 7.5, 7, 6, 5.5, 4.5, 4.5, 4, 4, 3, 2.5, 2.5, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "apple/photos.search", "apple/calendar.events.get", "apple/calendar.events.list", "apple/reminders.list", "apple/calendar.events.update", "apple/calendar.list", "apple/health.statistics", "apple/calendar.events.create", "apple/maps.show", "apple/location.get", "apple/maps.eta", "apple/calendar.events.delete", "apple/music.status", "apple/maps.route", "apple/alarms.cancel", "apple/weather.daily", "apple/alarms.create", "apple/weather.hourly" ], "scores": [ 11.055203666158867, 8.36200315262235, 6.755163959395887, 5.131540575526658, 4.645263111511694, 4.594667765007597, 3.353990223330057, 3.3248940132553706, 3.0094803210198413, 2.6351421055956448, 2.4208697806788355, 2.40133549848084, 1.9146581234509097, 1.7197049516029317, 1.5909917502355233, 1.5741603564255942, 1.4761555867694751, 1.4513293217173118, 1.4100691696593006 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "bm25": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.get", "builtin/GetCurrentTime", "apple/calendar.list", "apple/photos.search", "apple/health.statistics", "apple/reminders.list", "apple/calendar.events.delete", "apple/health.sleep.query", "apple/health.workouts.list", "apple/alarms.create", "apple/reminders.create", "apple/reminders.update", "builtin/ScheduledTasks", "apple/maps.show", "apple/location.get", "apple/maps.eta", "apple/weather.daily", "apple/music.status", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/photos.albums.list", "apple/contacts.list", "apple/links.open", "builtin/ReadFile", "builtin/Memory" ], "scores": [ 14.684931603634698, 11.720289216093375, 10.954002229381262, 10.70401981344924, 8.78890477546824, 7.725956296832728, 7.13464523136536, 6.229132941972527, 5.758448572823133, 4.55597709755589, 4.134548419943995, 3.7946763283971214, 2.3446465070547893, 1.8284761827757008, 1.8284761827757008, 1.7012883480573988, 1.6769433783228374, 1.6043303491322136, 1.590536429816914, 1.290252793313428, 1.1370398010457616, 1.0469491829486346, 1.0352314574515493, 0.9217503154435029, 0.8609596388041906, 0.8402090778338882, 0.6658726147629157, 0.6641347819940883, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.get", "builtin/GetCurrentTime", "apple/calendar.list", "apple/photos.search", "apple/health.statistics", "apple/reminders.list", "apple/calendar.events.delete", "apple/health.sleep.query", "apple/health.workouts.list", "apple/alarms.create", "apple/reminders.create", "apple/reminders.update", "builtin/ScheduledTasks", "apple/maps.show", "apple/location.get", "apple/maps.eta", "apple/weather.daily", "apple/links.open", "apple/music.status", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/photos.albums.list", "apple/contacts.list", "builtin/ReadFile", "builtin/Memory" ], "scores": [ 14.684931603634698, 11.720289216093375, 10.954002229381262, 10.70401981344924, 10.396825725529947, 7.725956296832728, 7.13464523136536, 6.229132941972527, 5.758448572823133, 4.55597709755589, 4.134548419943995, 3.7946763283971214, 2.3446465070547893, 1.8284761827757008, 1.8284761827757008, 1.7012883480573988, 1.6769433783228374, 1.6043303491322136, 1.590536429816914, 1.290252793313428, 1.142400969046904, 1.1370398010457616, 1.0469491829486346, 1.0352314574515493, 0.9217503154435029, 0.8609596388041906, 0.8402090778338882, 0.6641347819940883, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N061", "query": "completed reminderId", "category": "schema", "domain": "reminders", "intent": "capability:apple/reminders.complete", "target_capability": "Completion setter fields.", "acceptable": [ "apple/reminders.complete" ], "rationale": "Completion setter fields.", "basis": [ "apple/reminders.complete" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/reminders.get", "apple/reminders.list", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.lists", "apple/links.open", "builtin/ScheduledTasks", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/music.playback", "apple/photos.get", "apple/alarms.cancel", "apple/calendar.events.get", "apple/calendar.events.list", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get", "apple/photos.search", "builtin/AskUserQuestions" ], "scores": [ 6, 6, 5.5, 5.5, 5, 5, 4, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "name_description": { "ranking": [ "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/photos.get", "apple/music.playback", "apple/calendar.events.delete", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 6.307278074848403, 5.117342066483571, 4.8339042269754104, 3.5412226376495974, 3.4062280126487074, 3.2512998801244857, 3.1371483815404533, 2.253912805996125, 1.8781025023437183, 1.8016996019310052, 1.5532511243390867, 1.4803458231303015, 1.4405949110207767 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "bm25": { "ranking": [ "apple/reminders.complete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.update", "apple/reminders.lists", "apple/photos.get", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.playback", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.get", "apple/links.open", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 12.463361551346706, 11.971646560145278, 10.398128180433039, 9.998329208180644, 9.240351400598522, 8.955342835635903, 8.138621340785264, 2.0189831893083445, 1.8960280939425802, 1.793265803487735, 1.7244033122771167, 1.3699185217230823, 1.3062125395735957, 0.7847527060650342, 0.70568370941594, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.complete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.update", "apple/reminders.lists", "apple/photos.get", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.playback", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/calendar.events.get", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 12.463361551346706, 11.971646560145278, 10.398128180433039, 9.998329208180644, 9.240351400598522, 8.955342835635903, 8.138621340785264, 2.0189831893083445, 1.8960280939425802, 1.793265803487735, 1.7244033122771167, 1.3699185217230823, 1.3062125395735957, 1.2107026713576772, 0.7847527060650342, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N062", "query": "assetId opaque image video metadata", "category": "schema", "domain": "photos", "intent": "capability:apple/photos.get", "target_capability": "Fetch one metadata item by opaque asset ID.", "acceptable": [ "apple/photos.get" ], "rationale": "Fetch one metadata item by opaque asset ID.", "basis": [ "apple/photos.get" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/photos.get", "apple/photos.search", "apple/photos.albums.list", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/music.playback", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "builtin/ScheduledTasks", "apple/alarms.list", "apple/clipboard.read", "apple/alarms.cancel", "apple/calendar.events.get", "apple/calendar.events.list", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "builtin/AskUserQuestions" ], "scores": [ 7.5, 6, 4, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.get", "apple/clipboard.read", "apple/alarms.list", "apple/reminders.create", "apple/music.playback", "apple/calendar.events.delete", "apple/reminders.list", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 18.043304278192572, 9.875966587440619, 8.668194520658389, 2.6944644547345806, 2.4221849112268674, 2.3962861849916948, 2.0606820288717214, 1.8781025023437183, 1.8016996019310052, 1.7229585090753954, 1.5532511243390867, 1.4803458231303015, 1.4405949110207767 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/photos.get", "apple/photos.search", "apple/photos.albums.list", "apple/clipboard.read", "apple/alarms.list", "apple/reminders.get", "apple/calendar.events.delete", "apple/music.playback", "apple/reminders.create", "apple/calendar.events.create", "apple/reminders.list", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/reminders.complete", "apple/calendar.events.get", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 21.73325289868253, 12.071667756739544, 9.852138897792141, 2.410538121601221, 2.383253378845514, 2.3660103307504636, 1.793265803487735, 1.7244033122771167, 1.7020131435723234, 1.3699185217230823, 1.3088266447267578, 1.3062125395735957, 0.9919551950628672, 0.8674373781600091, 0.8524674830238934, 0.7847527060650342, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/photos.get", "apple/photos.search", "apple/photos.albums.list", "apple/clipboard.read", "apple/alarms.list", "apple/reminders.get", "apple/calendar.events.delete", "apple/music.playback", "apple/reminders.create", "apple/calendar.events.create", "apple/reminders.list", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/reminders.complete", "apple/calendar.events.get", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 21.73325289868253, 12.071667756739544, 9.852138897792141, 2.410538121601221, 2.383253378845514, 2.3660103307504636, 1.793265803487735, 1.7244033122771167, 1.7020131435723234, 1.3699185217230823, 1.3088266447267578, 1.3062125395735957, 0.9919551950628672, 0.8674373781600091, 0.8524674830238934, 0.7847527060650342, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N063", "query": "metric heartRateVariabilitySDNN start end", "category": "schema", "domain": "health", "intent": "capability:apple/health.statistics", "target_capability": "Statistics metric enum includes HRV.", "acceptable": [ "apple/health.statistics" ], "rationale": "Statistics metric enum includes HRV.", "basis": [ "apple/health.statistics" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/health.statistics", "apple/photos.search", "apple/calendar.events.update", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.list", "apple/health.sleep.query", "apple/health.workouts.list", "apple/maps.show", "builtin/Memory" ], "scores": [ 8.5, 3, 2, 1.5, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/health.statistics", "apple/photos.search", "apple/calendar.events.get", "apple/maps.show", "apple/calendar.events.update" ], "scores": [ 18.940150583824813, 6.0878998829043915, 3.649780217757678, 2.6351421055956448, 2.3135578186715415 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/health.statistics", "apple/photos.search", "apple/calendar.events.get", "apple/calendar.events.update", "apple/maps.show", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.workouts.list", "apple/calendar.events.create", "builtin/Memory" ], "scores": [ 28.64008190220582, 5.2814844234348, 3.8692915119224027, 2.8536349304388415, 1.6769433783228374, 1.5353669870143491, 1.4415911021883758, 1.378447345389575, 1.3586109759504261, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/health.statistics", "apple/photos.search", "apple/calendar.events.get", "apple/calendar.events.update", "apple/maps.show", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.workouts.list", "apple/calendar.events.create", "builtin/Memory" ], "scores": [ 28.64008190220582, 5.2814844234348, 3.8692915119224027, 2.8536349304388415, 1.6769433783228374, 1.5353669870143491, 1.4415911021883758, 1.378447345389575, 1.3586109759504261, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N064", "query": "action pause resume previous next songId", "category": "schema", "domain": "music", "intent": "capability:apple/music.playback", "target_capability": "Player action enum; songId for play.", "acceptable": [ "apple/music.playback" ], "rationale": "Player action enum; songId for play.", "basis": [ "apple/music.playback" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/music.playback", "builtin/ScheduledTasks", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/music.search", "apple/photos.get", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/shortcuts.run", "apple/alarms.cancel", "apple/calendar.events.get", "apple/calendar.events.list", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get", "apple/photos.search", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "builtin/AskUserQuestions" ], "scores": [ 10.5, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/music.playback", "builtin/ScheduledTasks", "apple/music.search", "apple/reminders.get", "apple/shortcuts.run", "apple/photos.get", "apple/reminders.create", "apple/calendar.events.delete", "apple/reminders.list", "apple/calendar.events.create", "apple/calendar.events.update" ], "scores": [ 20.194578290931105, 2.8210849390681956, 2.776792493451411, 2.6944644547345806, 2.4516459762570415, 2.253912805996125, 2.0606820288717214, 1.8016996019310052, 1.7229585090753954, 1.5532511243390867, 1.4803458231303015 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/music.playback", "apple/music.search", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/reminders.get", "apple/photos.get", "apple/calendar.events.delete", "apple/reminders.create", "apple/calendar.events.create", "apple/reminders.list", "apple/calendar.events.update", "apple/reminders.delete", "apple/reminders.complete", "apple/calendar.events.get", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/reminders.update", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 32.09136126074502, 4.95015208282663, 2.711237774794011, 2.4284016484087565, 2.3660103307504636, 2.0189831893083445, 1.793265803487735, 1.7020131435723234, 1.3699185217230823, 1.3088266447267578, 1.3062125395735957, 0.8674373781600091, 0.8524674830238934, 0.7847527060650342, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5815098177219924, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/music.playback", "apple/music.search", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/reminders.get", "apple/photos.get", "apple/calendar.events.delete", "apple/reminders.create", "apple/calendar.events.create", "apple/reminders.list", "apple/calendar.events.update", "apple/reminders.delete", "apple/reminders.complete", "apple/calendar.events.get", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/reminders.update", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 32.09136126074502, 4.95015208282663, 2.711237774794011, 2.4284016484087565, 2.3660103307504636, 2.0189831893083445, 1.793265803487735, 1.7020131435723234, 1.3699185217230823, 1.3088266447267578, 1.3062125395735957, 0.8674373781600091, 0.8524674830238934, 0.7847527060650342, 0.6959051917000892, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5815098177219924, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N065", "query": "List file names only; do not read their contents", "category": "interference", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Listing names differs from reading text.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Listing names differs from reading text.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "apple/photos.albums.list", "apple/alarms.list", "builtin/ListFiles", "apple/contacts.list", "apple/calendar.events.list", "builtin/EditFile", "builtin/ScheduledTasks", "apple/calendar.list", "apple/clipboard.read", "apple/reminders.list", "builtin/Memory", "builtin/WriteFile", "apple/health.workouts.list", "apple/links.open", "apple/photos.search", "apple/reminders.lists", "apple/shortcuts.run", "apple/health.sleep.query", "apple/maps.show", "apple/alarms.create", "apple/photos.get", "apple/alarms.cancel", "apple/calendar.events.update", "apple/clipboard.write", "apple/music.playback", "apple/reminders.create", "apple/battery.get", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/maps.eta", "apple/music.search", "apple/music.status", "apple/weather.current", "apple/weather.daily", "apple/calendar.events.delete", "apple/contacts.create", "apple/contacts.get", "apple/health.workouts.get", "apple/maps.search", "apple/calendar.events.get", "apple/location.geocode", "apple/location.get", "apple/weather.hourly", "builtin/AskUserQuestions", "apple/calendar.events.create", "apple/contacts.delete" ], "scores": [ 8, 7.5, 7, 7, 6.5, 5.5, 5.5, 5.5, 5, 5, 5, 5, 5, 4.5, 4, 4, 4, 4, 3.5, 3.5, 3, 3, 2.5, 2.5, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "apple/photos.albums.list", "apple/calendar.list", "builtin/Memory", "apple/shortcuts.run", "apple/health.sleep.query", "builtin/ReadFile", "builtin/ListFiles", "apple/alarms.list", "builtin/EditFile", "apple/contacts.list", "apple/clipboard.write", "apple/maps.show", "apple/device.get", "apple/links.open", "builtin/ScheduledTasks", "builtin/WriteFile", "apple/photos.search", "apple/alarms.create", "builtin/AskUserQuestions", "apple/photos.get", "apple/clipboard.read", "apple/music.playback", "apple/music.search", "apple/music.status", "apple/calendar.events.list", "apple/weather.current", "apple/health.workouts.list", "apple/contacts.create", "apple/reminders.create", "apple/alarms.cancel", "apple/reminders.lists", "apple/battery.get", "apple/maps.eta", "apple/calendar.events.update", "apple/contacts.update", "apple/health.statistics", "apple/contacts.get", "apple/weather.daily", "apple/reminders.list", "apple/calendar.events.get", "apple/health.workouts.get", "apple/calendar.events.delete", "apple/location.geocode", "apple/location.get", "apple/maps.search", "apple/weather.hourly" ], "scores": [ 6.651277750430946, 6.613177859590141, 6.260069378030365, 5.8508560588424965, 5.631741717659189, 5.20810680629957, 5.043838265675319, 4.980250368935893, 4.7373151115820535, 4.513732001699601, 4.3968000490354155, 4.240852309595492, 4.196836460602601, 3.9361959833690285, 3.798793049494683, 3.714454817908332, 3.6890267687125697, 3.3790868854437415, 3.348804949568758, 3.3205596101684804, 3.2754104614901403, 3.108936491207336, 2.9329121219677106, 2.875951048192145, 2.710592454113189, 2.6261405201969312, 2.5539316274936916, 2.4692163806821252, 2.398487524474808, 2.3121713186197392, 2.1668991632120185, 2.0882313374885, 2.0882313374885, 2.0517067323765903, 2.0251373927906937, 1.9605841929851233, 1.596129569605437, 1.5363986946963615, 1.4924909939710067, 1.457997372644377, 1.2665935090803384, 1.1248369805109821, 1.0700151212109088, 1.031117386694429, 0.8549912766967913, 0.8464971647744733 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.14285714285714285 } }, "bm25": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/calendar.list", "apple/photos.albums.list", "apple/shortcuts.run", "apple/contacts.list", "apple/alarms.list", "apple/clipboard.read", "apple/maps.show", "apple/health.sleep.query", "apple/clipboard.write", "apple/calendar.events.list", "builtin/ScheduledTasks", "apple/device.get", "apple/reminders.list", "apple/reminders.lists", "apple/links.open", "apple/photos.search", "apple/contacts.create", "apple/contacts.update", "apple/health.workouts.list", "builtin/AskUserQuestions", "apple/music.playback", "apple/photos.get", "apple/alarms.create", "apple/music.search", "apple/music.status", "apple/alarms.cancel", "apple/reminders.create", "apple/weather.current", "apple/contacts.get", "apple/calendar.events.update", "apple/battery.get", "apple/maps.eta", "apple/health.workouts.get", "apple/health.statistics", "apple/calendar.events.delete", "apple/weather.daily", "apple/calendar.events.get", "apple/maps.search", "apple/location.geocode", "apple/location.get", "apple/weather.hourly", "apple/contacts.delete", "apple/calendar.events.create" ], "scores": [ 13.10696022217413, 11.99099915861498, 10.649898971965822, 9.643468258788317, 8.646227560859904, 8.592823496547922, 8.415885517053447, 7.527884141076022, 6.677446227119177, 6.5309002558443074, 5.857157575876017, 5.816783255556098, 5.756385189718303, 5.066242127384126, 4.589890948355228, 4.395858470521765, 4.204422201447571, 4.144381718209205, 4.1228942449914925, 4.060637122403543, 4.047109290901706, 4.00922799620642, 3.8299383300103536, 3.792375941316764, 3.400782226857424, 3.3426079028464812, 3.01956604096922, 2.797246570370531, 2.6146290117458038, 2.605830432217782, 2.5742341630382173, 2.3905680059174137, 2.3623305536826615, 2.2100953846523073, 2.047190334149854, 1.9096787835648599, 1.9096787835648599, 1.8886917322827, 1.712505040040543, 1.4175588968227548, 1.3809136818357022, 1.1253412868673076, 1.0904891394022966, 0.8836339964142832, 0.8493613585822458, 0.8363195411534453, 0.6009512783548976, 0.30692226461244376 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/ListFiles", "builtin/EditFile", "apple/calendar.list", "apple/photos.albums.list", "builtin/Memory", "apple/contacts.list", "apple/shortcuts.run", "apple/alarms.list", "apple/maps.show", "apple/health.sleep.query", "apple/calendar.events.list", "apple/reminders.list", "apple/device.get", "apple/clipboard.write", "apple/clipboard.read", "apple/health.workouts.list", "apple/reminders.lists", "apple/links.open", "builtin/ScheduledTasks", "apple/contacts.create", "apple/photos.search", "apple/contacts.update", "apple/photos.get", "builtin/AskUserQuestions", "apple/music.playback", "apple/alarms.create", "apple/contacts.get", "apple/battery.get", "apple/music.search", "apple/music.status", "apple/alarms.cancel", "apple/health.workouts.get", "apple/reminders.create", "apple/calendar.events.get", "apple/calendar.events.update", "apple/weather.current", "apple/location.get", "apple/maps.eta", "apple/health.statistics", "apple/calendar.events.delete", "apple/weather.daily", "apple/maps.search", "apple/reminders.get", "apple/location.geocode", "apple/contacts.delete", "apple/weather.hourly", "apple/calendar.events.create" ], "scores": [ 19.84148964769966, 17.794590904994607, 10.843591632532075, 10.24338125913383, 9.496322200285285, 9.342528076014883, 8.080213339185143, 7.604088786080613, 7.527884141076022, 7.229899799619796, 5.816783255556098, 5.435060647018335, 5.167800539892758, 5.0258644793281775, 4.919680510022155, 4.748517860972697, 4.67189124737578, 4.2686312250335465, 4.1228942449914925, 4.060637122403543, 4.059964335202037, 4.00922799620642, 3.8362554029013376, 3.8299383300103536, 3.7633256449131913, 3.400782226857424, 3.2056354579302893, 2.797246570370531, 2.743489440674309, 2.628985343147514, 2.6146290117458038, 2.605830432217782, 2.5742341630382173, 2.4109428811501274, 2.3905680059174137, 2.1020207244898015, 2.047190334149854, 2.0023739163303613, 1.8260407962047396, 1.5965567909467069, 1.5069513200189237, 1.4175588968227548, 1.1544910774629917, 1.0904891394022966, 0.9510032826687982, 0.8836339964142832, 0.6009512783548976, 0.5918458513942811, 0.30692226461244376 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } } } }, { "query_id": "N066", "query": "Read file contents, not lasting user memories", "category": "interference", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Ordinary file text, not Memory namespace.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Ordinary file text, not Memory namespace.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "builtin/ReadFile", "builtin/EditFile", "apple/clipboard.read", "builtin/ScheduledTasks", "builtin/WriteFile", "builtin/AskUserQuestions", "builtin/ListFiles", "apple/clipboard.write", "apple/health.sleep.query", "apple/photos.search", "apple/alarms.list", "apple/battery.get", "apple/device.get", "apple/health.statistics", "apple/maps.eta", "apple/maps.show", "apple/photos.get", "apple/weather.current", "apple/weather.daily", "apple/calendar.events.list", "apple/contacts.list", "apple/health.workouts.list", "apple/links.open", "apple/maps.search", "apple/photos.albums.list", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.delete", "apple/contacts.get", "apple/health.workouts.get", "apple/location.geocode", "apple/location.get", "apple/music.playback", "apple/music.search", "apple/music.status", "apple/reminders.create", "apple/weather.hourly", "apple/reminders.list", "apple/shortcuts.run" ], "scores": [ 9.5, 8, 5.5, 5, 5, 5, 4, 3, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/ReadFile", "builtin/EditFile", "apple/clipboard.write", "builtin/ScheduledTasks", "builtin/WriteFile", "builtin/AskUserQuestions", "apple/clipboard.read", "apple/device.get", "builtin/ListFiles", "apple/maps.show", "apple/health.sleep.query", "apple/weather.current", "apple/alarms.list", "apple/photos.search", "apple/music.playback", "apple/photos.get", "apple/battery.get", "apple/maps.eta", "apple/calendar.events.delete", "apple/health.statistics", "apple/alarms.cancel", "apple/contacts.get", "apple/weather.daily", "apple/photos.albums.list", "apple/health.workouts.get", "apple/music.search", "apple/health.workouts.list", "apple/reminders.create", "apple/calendar.events.list", "apple/location.geocode", "apple/location.get", "apple/music.status", "apple/contacts.list", "apple/links.open", "apple/maps.search", "apple/weather.hourly", "apple/alarms.create" ], "scores": [ 15.440590151552316, 5.20810680629957, 4.7373151115820535, 4.3968000490354155, 4.059026330354907, 3.714454817908332, 3.282365175825118, 3.2754104614901403, 3.247895248237003, 3.053494701017844, 2.820626131006682, 2.6453548442552615, 2.6261405201969312, 2.2258277918052776, 2.2055776060257926, 2.167107836671426, 2.1137609941615927, 2.0882313374885, 2.0882313374885, 2.041771298060864, 1.9605841929851233, 1.8905693842378057, 1.596129569605437, 1.5363986946963615, 1.3627028549488005, 1.2665935090803384, 1.2162317258779303, 1.2149135706002092, 1.1119626517018855, 1.0956951902365162, 1.0700151212109088, 1.031117386694429, 0.9834495738143223, 0.9297239873406339, 0.8598345546612385, 0.8549912766967913, 0.8464971647744733, 0.8299733053491117 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "builtin/Memory", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/AskUserQuestions", "apple/clipboard.read", "builtin/ListFiles", "apple/clipboard.write", "builtin/ScheduledTasks", "apple/device.get", "apple/health.sleep.query", "apple/maps.show", "apple/weather.current", "apple/photos.search", "apple/calendar.events.delete", "apple/alarms.list", "apple/battery.get", "apple/maps.eta", "apple/photos.get", "apple/alarms.cancel", "apple/music.playback", "apple/health.statistics", "apple/photos.albums.list", "apple/contacts.get", "apple/health.workouts.list", "apple/calendar.events.list", "apple/weather.daily", "apple/health.workouts.get", "apple/contacts.list", "apple/maps.search", "apple/music.search", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/links.open", "apple/weather.hourly", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "apple/reminders.list" ], "scores": [ 28.188991397619855, 13.10696022217413, 11.99099915861498, 10.649898971965822, 8.409346679470302, 5.857157575876017, 5.804260981819989, 5.066242127384126, 4.272530642246411, 3.255663577104694, 2.7236461814639616, 2.644000399352235, 2.3623305536826615, 2.3584158561718, 2.0639215021837605, 1.9901723522571306, 1.9096787835648599, 1.9096787835648599, 1.8825881110606586, 1.8791219646015662, 1.720954340406063, 1.712505040040543, 1.6193758131939588, 1.60914410629741, 1.5424466287322323, 1.4237153194905796, 1.3809136818357022, 1.3042089742613945, 1.2500038811300134, 1.0904891394022966, 0.9930830738422539, 0.9207888081104977, 0.8836339964142832, 0.8493613585822458, 0.8407929251848839, 0.8363195411534453, 0.8075967500940832, 0.7660997030790287, 0.6748604050062972, 0.3147136525322586 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/Memory", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "builtin/AskUserQuestions", "builtin/ListFiles", "apple/clipboard.write", "apple/clipboard.read", "apple/device.get", "builtin/ScheduledTasks", "apple/alarms.list", "apple/maps.show", "apple/battery.get", "apple/photos.get", "apple/photos.albums.list", "apple/health.sleep.query", "apple/contacts.list", "apple/photos.search", "apple/contacts.get", "apple/calendar.events.delete", "apple/health.workouts.list", "apple/weather.current", "apple/calendar.events.list", "apple/alarms.cancel", "apple/health.workouts.get", "apple/location.get", "apple/maps.eta", "apple/music.playback", "apple/health.statistics", "apple/reminders.list", "apple/weather.daily", "apple/maps.search", "apple/music.search", "apple/calendar.events.get", "apple/reminders.get", "apple/reminders.create", "apple/calendar.list", "apple/location.geocode", "apple/links.open", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "apple/weather.hourly" ], "scores": [ 38.14829751460212, 19.84148964769966, 17.794590904994607, 10.24338125913383, 8.409346679470302, 5.804260981819989, 4.748517860972697, 4.67189124737578, 3.970921885679278, 3.9366365069266833, 2.6891718960326196, 2.644000399352235, 2.628985343147514, 2.6263477150046297, 2.5460183721553955, 2.4023216387639934, 2.1766464400914503, 2.147561968171432, 2.1425381623194113, 2.0639215021837605, 2.018701912449015, 2.0023739163303613, 2.001624911028109, 1.8791219646015662, 1.8264601231288218, 1.8260407962047396, 1.5965567909467069, 1.5839818954898712, 1.5069513200189237, 1.196196413651231, 1.1544910774629917, 1.0904891394022966, 0.9930830738422539, 0.9766794376224938, 0.9510032826687982, 0.9207888081104977, 0.9034987037373633, 0.8836339964142832, 0.8407929251848839, 0.8075967500940832, 0.7660997030790287, 0.6748604050062972, 0.5918458513942811 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N067", "query": "Replace the entire file with this content, allowing overwrite", "category": "interference", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "Whole replacement uses WriteFile, not one-segment EditFile.", "acceptable": [ "builtin/WriteFile" ], "rationale": "Whole replacement uses WriteFile, not one-segment EditFile.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.write", "builtin/ListFiles", "builtin/Memory", "apple/contacts.update", "apple/music.playback", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 5.5, 5.5, 4, 3, 3, 3, 2, 2, 1, 1, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/EditFile", "builtin/Memory", "apple/contacts.update", "apple/clipboard.write", "apple/music.playback", "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "apple/clipboard.read", "apple/photos.search" ], "scores": [ 6.522108665824732, 5.91486491675378, 5.58626988607487, 5.3910694014157645, 3.7846374736434, 3.714454817908332, 3.472413797389085, 3.053494701017844, 1.931435403678754, 1.4705301552404113 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "bm25": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "builtin/ReadFile", "apple/clipboard.write", "builtin/Memory", "builtin/ListFiles", "apple/contacts.update", "apple/music.playback", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 13.463057050047663, 12.47058213032427, 8.549670258073231, 8.133936601874604, 7.950705198312901, 5.804260981819989, 5.552444956865532, 3.3049362358959344, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/contacts.update", "builtin/Memory", "apple/clipboard.write", "builtin/ListFiles", "apple/music.playback", "apple/calendar.events.update", "apple/reminders.update", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 19.26664879642729, 15.639711397805081, 14.802427847186195, 7.8167195842920805, 7.5332941309301304, 7.509132808105824, 5.804260981819989, 2.956260339537846, 2.4919310360729865, 2.432653518515385, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N068", "query": "Edit exactly one matched passage, preserving the rest of the file", "category": "interference", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "Unique segment edit, not full write.", "acceptable": [ "builtin/EditFile" ], "rationale": "Unique segment edit, not full write.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "builtin/ReadFile", "builtin/WriteFile", "builtin/ListFiles", "builtin/Memory", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.update", "apple/health.workouts.get", "apple/location.get", "apple/maps.route", "apple/music.playback", "apple/photos.get", "apple/photos.search", "apple/reminders.delete", "apple/shortcuts.run" ], "scores": [ 9.5, 4.5, 4.5, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/EditFile", "builtin/Memory", "builtin/ListFiles", "builtin/WriteFile", "builtin/ReadFile", "apple/clipboard.read", "apple/contacts.create", "apple/photos.search", "apple/contacts.update", "apple/reminders.delete", "apple/health.workouts.get", "apple/calendar.events.update", "apple/contacts.delete", "apple/location.get", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/calendar.events.create", "apple/alarms.create", "apple/music.playback" ], "scores": [ 14.004787736959418, 7.908786667152112, 4.735702577463764, 3.714454817908332, 3.472413797389085, 1.931435403678754, 1.6696723659319328, 1.4705301552404113, 1.4593071471968346, 1.4160558529896354, 1.3959771463946806, 1.340122705366102, 1.3390180568934824, 1.1929880582775596, 1.1248369805109821, 1.0997080487563646, 1.0526744179854908, 1.0415379820610877, 0.9892126698597037, 0.9697256428343111, 0.9602672350864689, 0.7720168266239731 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/EditFile", "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.delete", "apple/photos.search", "apple/contacts.delete", "apple/calendar.events.update", "apple/location.get", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/calendar.events.create", "apple/alarms.create", "apple/shortcuts.run", "apple/music.playback" ], "scores": [ 25.300360281348723, 10.490060070078, 10.321912047288809, 9.06818633791728, 7.455678977574497, 1.8024824369174723, 1.5973405741141093, 1.3840390692635096, 1.3691132429326038, 1.3691132429326038, 1.3532076178115147, 1.2889866107203032, 1.2739620523469748, 1.1390026644524067, 1.0829958778625002, 1.0444767918636064, 0.9971873002352808, 0.9860265225742033, 0.9337714661875144, 0.9239781133099412, 0.904995019691811, 0.7660997030790287, 0.720071910007749 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "builtin/Memory", "builtin/ListFiles", "apple/contacts.update", "apple/calendar.events.update", "apple/reminders.update", "apple/clipboard.read", "apple/contacts.create", "apple/health.workouts.get", "apple/reminders.delete", "apple/photos.search", "apple/contacts.delete", "apple/location.get", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/calendar.events.create", "apple/alarms.create", "apple/shortcuts.run", "apple/music.playback" ], "scores": [ 31.03129559027186, 12.982913464988664, 11.657343329863814, 9.20963166765084, 7.455678977574497, 4.332244226261794, 4.150327022640813, 2.807942621296027, 1.8024824369174723, 1.5973405741141093, 1.3691132429326038, 1.3691132429326038, 1.3532076178115147, 1.2889866107203032, 1.1390026644524067, 1.0829958778625002, 1.0444767918636064, 0.9971873002352808, 0.9860265225742033, 0.9337714661875144, 0.9239781133099412, 0.904995019691811, 0.7660997030790287, 0.720071910007749 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N069", "query": "Delete a lasting memory file, not a normal workspace file", "category": "interference", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Memory explicitly supports delete within its namespace.", "acceptable": [ "builtin/Memory" ], "rationale": "Memory explicitly supports delete within its namespace.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/Memory", "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "apple/calendar.events.delete", "apple/contacts.delete", "apple/reminders.delete", "builtin/ListFiles", "builtin/ScheduledTasks", "apple/contacts.list", "apple/health.sleep.query", "apple/links.open", "apple/maps.search", "apple/photos.albums.list", "apple/photos.search", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/clipboard.read", "apple/clipboard.write", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.get", "apple/reminders.create", "apple/weather.current", "apple/weather.daily", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.list", "apple/shortcuts.run" ], "scores": [ 8.5, 5, 5, 4.5, 4, 4, 4, 3.5, 3.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory", "builtin/ScheduledTasks", "apple/reminders.delete", "builtin/WriteFile", "apple/contacts.delete", "builtin/ReadFile", "builtin/EditFile", "apple/calendar.events.delete", "builtin/ListFiles", "apple/clipboard.read", "apple/health.sleep.query", "apple/photos.search", "apple/weather.current", "apple/photos.albums.list", "apple/clipboard.write", "apple/alarms.list", "apple/health.statistics", "apple/music.search", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/contacts.list", "apple/maps.show", "apple/links.open", "apple/maps.search", "apple/alarms.create", "apple/weather.daily" ], "scores": [ 14.145907217334798, 3.909073407360251, 3.752882783565409, 3.714454817908332, 3.6211209622787783, 3.472413797389085, 3.374072336740228, 3.224841952905041, 3.053494701017844, 1.931435403678754, 1.5629367438392368, 1.4705301552404113, 1.3848806590980578, 1.3627028549488005, 1.3098389254907072, 1.270718613665206, 1.2432451371448472, 1.2162317258779303, 1.2162317258779303, 1.1119626517018855, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9834495738143223, 0.9297239873406339, 0.9098422129056576, 0.8598345546612385, 0.8549912766967913, 0.8299733053491117, 0.7495532564546209 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory", "builtin/WriteFile", "builtin/EditFile", "apple/reminders.delete", "apple/contacts.delete", "builtin/ReadFile", "apple/calendar.events.delete", "builtin/ListFiles", "builtin/ScheduledTasks", "apple/clipboard.read", "apple/photos.search", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/health.statistics", "apple/music.search", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/links.open", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/shortcuts.run", "apple/maps.show", "apple/alarms.create", "apple/weather.daily", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list" ], "scores": [ 26.406389147851378, 11.515668544828896, 10.390707067622014, 10.156196128710354, 9.97993715650646, 9.788911069686547, 8.698195352050329, 7.173973165253237, 4.032095731203915, 1.8024824369174723, 1.637106225525635, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 1.0093267431030897, 0.9930830738422539, 0.9930830738422539, 0.9207888081104977, 0.8836339964142832, 0.8493613585822458, 0.8407929251848839, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7660997030790287, 0.7436087609996902, 0.6748604050062972, 0.6063450614565641, 0.3147136525322586, 0.30990819698276545, 0.29633377370106745 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "builtin/ListFiles", "builtin/ScheduledTasks", "apple/clipboard.read", "apple/photos.search", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/health.statistics", "apple/music.search", "apple/photos.get", "apple/reminders.create", "apple/location.geocode", "apple/location.get", "apple/links.open", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/shortcuts.run", "apple/maps.show", "apple/alarms.create", "apple/weather.daily", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list" ], "scores": [ 36.93170948650841, 14.176669962528752, 12.37806806163308, 10.390707067622014, 10.156196128710354, 9.97993715650646, 8.698195352050329, 7.173973165253237, 4.032095731203915, 1.8024824369174723, 1.637106225525635, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 1.0093267431030897, 0.9930830738422539, 0.9930830738422539, 0.9207888081104977, 0.8836339964142832, 0.8493613585822458, 0.8407929251848839, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7660997030790287, 0.7436087609996902, 0.6748604050062972, 0.6063450614565641, 0.3147136525322586, 0.30990819698276545, 0.29633377370106745 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N070", "query": "Current date and time zone, not a weather forecast", "category": "interference", "domain": "clock", "intent": "capability:builtin/GetCurrentTime", "target_capability": "Current clock metadata.", "acceptable": [ "builtin/GetCurrentTime" ], "rationale": "Current clock metadata.", "basis": [ "builtin/GetCurrentTime" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/weather.current", "builtin/GetCurrentTime", "apple/weather.daily", "apple/weather.hourly", "apple/reminders.list", "apple/reminders.create", "apple/alarms.create", "apple/health.statistics", "apple/maps.eta", "apple/music.status", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/health.sleep.query", "apple/maps.search", "apple/calendar.events.update", "apple/location.geocode", "apple/location.get", "apple/maps.route", "apple/maps.show", "apple/photos.search", "apple/contacts.list", "apple/health.workouts.list", "apple/links.open", "apple/photos.albums.list", "apple/reminders.update", "builtin/EditFile", "apple/alarms.cancel", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.create", "apple/clipboard.write", "apple/location.reverseGeocode", "apple/music.playback", "apple/music.search", "apple/photos.get" ], "scores": [ 10, 10, 8, 7, 4.5, 3.5, 3, 3, 3, 3, 3, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "apple/weather.daily", "apple/weather.current", "apple/weather.hourly", "apple/reminders.list", "apple/maps.eta", "apple/music.status", "apple/reminders.create", "apple/location.get", "apple/photos.search", "apple/maps.route", "apple/location.geocode", "apple/alarms.create", "apple/health.statistics", "apple/maps.show", "apple/maps.search", "builtin/EditFile", "apple/calendar.events.list", "apple/calendar.events.update", "apple/location.reverseGeocode", "apple/alarms.cancel", "apple/health.sleep.query", "apple/photos.albums.list", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/battery.get", "apple/music.playback", "apple/contacts.list", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 12.020970580786653, 11.201601815634532, 9.82094798026129, 8.26142826982396, 6.736780131899655, 4.858079459396229, 4.091268786859684, 3.613445719512777, 3.4519871673732645, 3.154464769961352, 2.875211078836437, 2.5803144687932615, 2.2813026270664234, 2.2077638557947386, 2.1940615415065716, 2.061789892703679, 1.8329870145700418, 1.8251756430963588, 1.7970069904926336, 1.6335467459150792, 1.5741603564255942, 1.5629367438392368, 1.3627028549488005, 1.3098389254907072, 1.270718613665206, 1.2162317258779303, 1.2162317258779303, 1.018772408912019, 0.9418286545359099, 0.9297239873406339, 0.8598345546612385, 0.47664530096715424 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/weather.current", "apple/weather.daily", "builtin/GetCurrentTime", "apple/weather.hourly", "apple/reminders.list", "apple/reminders.create", "apple/photos.search", "apple/maps.eta", "apple/health.statistics", "apple/alarms.create", "apple/music.status", "apple/health.sleep.query", "apple/calendar.events.list", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/reminders.update", "apple/location.get", "builtin/EditFile", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/health.workouts.list", "apple/photos.albums.list", "apple/location.reverseGeocode", "apple/calendar.events.create", "apple/contacts.list", "apple/clipboard.write", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/photos.get", "apple/music.playback", "apple/links.open", "apple/battery.get" ], "scores": [ 19.91764793385925, 17.0957105510283, 14.720804988878164, 14.158691615035215, 6.112162322851736, 5.7469796184511726, 4.043930391678637, 3.79822663709421, 3.451677092263209, 3.3224098087577434, 3.2633129681212356, 3.2588202614648134, 3.0983833477237352, 3.001057004332835, 2.640834568801287, 2.619493380632581, 2.4536917077144595, 2.3540016180843235, 2.32646708381686, 2.2611434407596116, 2.2274670693108582, 1.9578030188106674, 1.8062965920874077, 1.6193758131939588, 1.592799090577726, 1.4937171307863306, 1.2500038811300134, 1.0742789579478196, 1.0402577760160727, 1.0352314574515493, 0.9930830738422539, 0.9930830738422539, 0.8767732783360794, 0.8407929251848839, 0.8385204047248325 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "builtin/GetCurrentTime", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/reminders.list", "apple/reminders.create", "apple/photos.search", "apple/maps.eta", "apple/health.statistics", "apple/alarms.create", "apple/music.status", "apple/health.sleep.query", "apple/calendar.events.list", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/reminders.update", "apple/location.get", "builtin/EditFile", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/health.workouts.list", "apple/photos.albums.list", "apple/location.reverseGeocode", "apple/calendar.events.create", "apple/contacts.list", "apple/clipboard.write", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/photos.get", "apple/music.playback", "apple/links.open", "apple/battery.get" ], "scores": [ 20.930451235812303, 19.91764793385925, 17.0957105510283, 14.158691615035215, 6.112162322851736, 5.7469796184511726, 4.043930391678637, 3.79822663709421, 3.451677092263209, 3.3224098087577434, 3.2633129681212356, 3.2588202614648134, 3.0983833477237352, 3.001057004332835, 2.640834568801287, 2.619493380632581, 2.4536917077144595, 2.3540016180843235, 2.32646708381686, 2.2611434407596116, 2.2274670693108582, 1.9578030188106674, 1.8062965920874077, 1.6193758131939588, 1.592799090577726, 1.4937171307863306, 1.2500038811300134, 1.0742789579478196, 1.0402577760160727, 1.0352314574515493, 0.9930830738422539, 0.9930830738422539, 0.8767732783360794, 0.8407929251848839, 0.8385204047248325 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N071", "query": "Delete this non-recurring calendar event, not a reminder", "category": "interference", "domain": "calendar", "intent": "capability:apple/calendar.events.delete", "target_capability": "Calendar-specific delete.", "acceptable": [ "apple/calendar.events.delete" ], "rationale": "Calendar-specific delete.", "basis": [ "apple/calendar.events.delete" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.create", "apple/calendar.events.update", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/links.open", "apple/reminders.complete", "apple/reminders.update", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/calendar.list", "apple/contacts.delete", "apple/reminders.lists", "builtin/ScheduledTasks", "apple/health.statistics", "apple/weather.daily", "apple/contacts.list", "apple/health.sleep.query", "apple/maps.search", "apple/photos.albums.list", "builtin/Memory", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/clipboard.write", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.get", "apple/weather.current", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 13.5, 11, 10.5, 10.5, 9.5, 9, 5.5, 5.5, 5.5, 5, 4.5, 4.5, 4, 4, 4, 4, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/reminders.create", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.list", "apple/reminders.list", "apple/health.statistics", "builtin/Memory", "builtin/ScheduledTasks", "apple/weather.daily", "apple/health.sleep.query", "apple/weather.current", "apple/photos.albums.list", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/contacts.list", "apple/maps.show", "apple/links.open", "apple/maps.search", "apple/alarms.create" ], "scores": [ 14.170092470836472, 12.798046107614855, 10.123599793430516, 10.120863042903943, 9.88390275065293, 6.689897641695674, 6.1757878588756, 5.567694926079179, 4.168622689313736, 3.6211209622787783, 3.612813620113822, 3.5412226376495974, 3.353990223330057, 3.110945717900015, 2.5890037873478104, 2.4404053139066892, 2.3610516557490286, 2.225708843224096, 1.5629367438392368, 1.3848806590980578, 1.3627028549488005, 1.3098389254907072, 1.270718613665206, 1.2162317258779303, 1.2162317258779303, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9834495738143223, 0.9297239873406339, 0.9098422129056576, 0.8598345546612385, 0.8549912766967913, 0.8299733053491117 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.list", "builtin/ScheduledTasks", "builtin/Memory", "apple/links.open", "apple/health.statistics", "apple/weather.daily", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/alarms.create", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 29.35636414758753, 25.168425516564266, 21.128675332589378, 20.63394785192433, 20.083319583634925, 16.85759706301727, 12.114257774150188, 10.830321760182576, 9.97993715650646, 9.605636229394815, 8.459127065136697, 8.160215498183916, 8.138621340785264, 7.725956296832728, 5.1566922418122, 3.1123028295377932, 2.2123492493637396, 2.1806597121263263, 1.896597854769992, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 0.9930830738422539, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6748604050062972, 0.29633377370106745, 0.28389860771412034 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "builtin/ScheduledTasks", "apple/calendar.list", "apple/links.open", "builtin/Memory", "apple/health.statistics", "apple/weather.daily", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/alarms.create", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 29.18114380102632, 25.0059869254732, 20.958293207142916, 20.524322367180964, 19.93217021600229, 16.85759706301727, 11.938133799495434, 10.677389802171442, 9.97993715650646, 9.605636229394815, 8.459127065136697, 8.160215498183916, 8.138621340785264, 7.745849233758733, 7.725956296832728, 3.1938965655894647, 3.1123028295377932, 2.1806597121263263, 1.896597854769992, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 0.9930830738422539, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6748604050062972, 0.29633377370106745, 0.28389860771412034 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N072", "query": "Delete this non-recurring reminder, not a calendar event", "category": "interference", "domain": "reminders", "intent": "capability:apple/reminders.delete", "target_capability": "Reminder-specific delete.", "acceptable": [ "apple/reminders.delete" ], "rationale": "Reminder-specific delete.", "basis": [ "apple/reminders.delete" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.create", "apple/calendar.events.update", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/links.open", "apple/reminders.complete", "apple/reminders.update", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/calendar.list", "apple/contacts.delete", "apple/reminders.lists", "builtin/ScheduledTasks", "apple/health.statistics", "apple/weather.daily", "apple/contacts.list", "apple/health.sleep.query", "apple/maps.search", "apple/photos.albums.list", "builtin/Memory", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/clipboard.write", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.get", "apple/weather.current", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 13.5, 11, 10.5, 10.5, 9.5, 9, 5.5, 5.5, 5.5, 5, 4.5, 4.5, 4, 4, 4, 4, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/reminders.create", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.list", "apple/reminders.list", "apple/health.statistics", "builtin/Memory", "builtin/ScheduledTasks", "apple/weather.daily", "apple/health.sleep.query", "apple/weather.current", "apple/photos.albums.list", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/contacts.list", "apple/maps.show", "apple/links.open", "apple/maps.search", "apple/alarms.create" ], "scores": [ 14.170092470836472, 12.798046107614855, 10.123599793430516, 10.120863042903943, 9.88390275065293, 6.689897641695674, 6.1757878588756, 5.567694926079179, 4.168622689313736, 3.6211209622787783, 3.612813620113822, 3.5412226376495974, 3.353990223330057, 3.110945717900015, 2.5890037873478104, 2.4404053139066892, 2.3610516557490286, 2.225708843224096, 1.5629367438392368, 1.3848806590980578, 1.3627028549488005, 1.3098389254907072, 1.270718613665206, 1.2162317258779303, 1.2162317258779303, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9834495738143223, 0.9297239873406339, 0.9098422129056576, 0.8598345546612385, 0.8549912766967913, 0.8299733053491117 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.list", "builtin/ScheduledTasks", "builtin/Memory", "apple/links.open", "apple/health.statistics", "apple/weather.daily", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/alarms.create", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 29.35636414758753, 25.168425516564266, 21.128675332589378, 20.63394785192433, 20.083319583634925, 16.85759706301727, 12.114257774150188, 10.830321760182576, 9.97993715650646, 9.605636229394815, 8.459127065136697, 8.160215498183916, 8.138621340785264, 7.725956296832728, 5.1566922418122, 3.1123028295377932, 2.2123492493637396, 2.1806597121263263, 1.896597854769992, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 0.9930830738422539, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6748604050062972, 0.29633377370106745, 0.28389860771412034 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/calendar.events.delete", "apple/reminders.delete", "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.delete", "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "builtin/ScheduledTasks", "apple/calendar.list", "apple/links.open", "builtin/Memory", "apple/health.statistics", "apple/weather.daily", "apple/health.sleep.query", "apple/photos.albums.list", "apple/contacts.list", "apple/weather.current", "apple/maps.search", "apple/clipboard.write", "apple/alarms.list", "apple/music.search", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/music.status", "apple/maps.show", "apple/alarms.create", "apple/health.workouts.list", "apple/photos.search" ], "scores": [ 29.18114380102632, 25.0059869254732, 20.958293207142916, 20.524322367180964, 19.93217021600229, 16.85759706301727, 11.938133799495434, 10.677389802171442, 9.97993715650646, 9.605636229394815, 8.459127065136697, 8.160215498183916, 8.138621340785264, 7.745849233758733, 7.725956296832728, 3.1938965655894647, 3.1123028295377932, 2.1806597121263263, 1.896597854769992, 1.6244277159640166, 1.6193758131939588, 1.2500038811300134, 1.1309556352520849, 1.0904891394022966, 1.0742789579478196, 1.0402577760160727, 0.9930830738422539, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.6748604050062972, 0.29633377370106745, 0.28389860771412034 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "N073", "query": "Read the one contact with this identifier, do not search all contacts", "category": "interference", "domain": "contacts", "intent": "capability:apple/contacts.get", "target_capability": "Read by ID, not contact-list query.", "acceptable": [ "apple/contacts.get" ], "rationale": "Read by ID, not contact-list query.", "basis": [ "apple/contacts.get" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.get", "apple/contacts.list", "apple/contacts.delete", "apple/contacts.update", "apple/maps.search", "apple/music.search", "apple/contacts.create", "apple/links.open", "apple/photos.search", "builtin/ReadFile", "apple/clipboard.read", "apple/health.sleep.query", "apple/photos.get", "builtin/ScheduledTasks", "apple/photos.albums.list", "builtin/Memory", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/clipboard.write", "apple/device.get", "apple/health.statistics", "apple/health.workouts.get", "apple/location.get", "apple/maps.eta", "apple/music.playback", "apple/weather.current", "apple/weather.daily", "builtin/EditFile", "apple/calendar.events.list", "apple/health.workouts.list", "apple/location.geocode", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/calendar.list", "apple/maps.route", "apple/maps.show", "apple/music.status", "apple/reminders.create", "apple/reminders.delete", "apple/weather.hourly", "builtin/ListFiles", "apple/location.reverseGeocode", "apple/reminders.list" ], "scores": [ 7, 6.5, 6, 6, 6, 5.5, 5, 4.5, 4.5, 4.5, 4, 3.5, 3.5, 3.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/contacts.get", "apple/contacts.list", "apple/health.sleep.query", "apple/contacts.create", "apple/contacts.delete", "apple/music.search", "apple/calendar.events.update", "apple/maps.search", "apple/contacts.update", "apple/links.open", "apple/device.get", "apple/calendar.list", "apple/music.playback", "apple/photos.albums.list", "apple/calendar.events.create", "apple/photos.get", "builtin/EditFile", "builtin/Memory", "apple/health.workouts.get", "apple/weather.current", "apple/photos.search", "apple/clipboard.write", "apple/alarms.list", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/health.statistics", "apple/alarms.create", "builtin/ReadFile", "builtin/ListFiles", "apple/weather.daily", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/clipboard.read", "apple/health.workouts.list", "apple/calendar.events.delete", "apple/reminders.create", "apple/calendar.events.list", "apple/location.geocode", "apple/maps.route", "apple/alarms.cancel", "apple/music.status", "apple/maps.show", "apple/weather.hourly" ], "scores": [ 8.791800038997057, 6.969249535453148, 5.631741717659189, 5.5780573739264145, 4.960139019172261, 4.944548611584215, 4.694829257780522, 4.5833081624030765, 4.51684744765666, 3.8470737045521126, 3.7981113423790736, 3.7650116726350085, 3.707208496483832, 3.676260673620342, 3.3972234794386367, 3.102973664021296, 2.8657419942233053, 2.7856097751832736, 2.662570655475019, 2.6261405201969312, 2.5792178262186876, 2.409546974247072, 2.2258277918052776, 2.224105444971989, 2.0882313374885, 2.0882313374885, 1.9605841929851233, 1.7902405404355806, 1.7356930089104854, 1.6822078764459198, 1.5363986946963615, 1.5099890589011253, 1.4160558529896354, 1.3439750578113865, 1.2149135706002092, 1.1248369805109821, 1.1119626517018855, 1.0956951902365162, 1.0700151212109088, 1.0526744179854908, 1.0415379820610877, 0.9834495738143223, 0.9098422129056576, 0.8464971647744733 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/contacts.get", "apple/contacts.delete", "apple/contacts.list", "apple/contacts.update", "apple/music.search", "apple/maps.search", "apple/contacts.create", "apple/photos.search", "apple/health.sleep.query", "builtin/ReadFile", "apple/links.open", "apple/calendar.events.update", "apple/photos.get", "apple/clipboard.read", "apple/photos.albums.list", "builtin/Memory", "apple/music.playback", "apple/calendar.events.create", "apple/device.get", "builtin/ScheduledTasks", "apple/calendar.list", "builtin/EditFile", "apple/health.workouts.get", "apple/weather.current", "apple/clipboard.write", "apple/location.geocode", "apple/alarms.list", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/health.statistics", "builtin/ListFiles", "apple/alarms.create", "apple/health.workouts.list", "apple/calendar.events.list", "apple/weather.daily", "apple/reminders.delete", "apple/location.reverseGeocode", "apple/calendar.events.delete", "apple/maps.route", "apple/alarms.cancel", "apple/reminders.create", "apple/weather.hourly", "apple/music.status", "apple/maps.show", "apple/reminders.list" ], "scores": [ 17.316380285268473, 13.806182908520192, 12.59709629845496, 12.345369642655621, 11.86447194496287, 11.518319839631785, 11.488322556861633, 7.635378170162728, 5.756385189718303, 5.075806043944951, 4.399434048907869, 4.318700352825255, 4.298360915817436, 4.054675138958545, 3.9267778223731358, 3.468502442807435, 3.346960627712961, 3.1322633171083183, 3.120526480872743, 2.9226004737628144, 2.896956848003894, 2.8505175168071517, 2.6733222171939985, 2.3623305536826615, 2.118755749811426, 2.002037074655073, 1.9901723522571306, 1.9883640230346527, 1.9096787835648599, 1.9096787835648599, 1.712505040040543, 1.651417995754508, 1.579855424698108, 1.5424466287322323, 1.4237153194905796, 1.3809136818357022, 1.3691132429326038, 1.144724074947355, 1.0829958778625002, 0.9971873002352808, 0.9860265225742033, 0.9207888081104977, 0.8363195411534453, 0.8075967500940832, 0.7436087609996902, 0.3147136525322586 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/contacts.get", "apple/contacts.list", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.create", "apple/music.search", "apple/maps.search", "apple/photos.albums.list", "apple/calendar.list", "apple/health.sleep.query", "apple/photos.search", "builtin/ReadFile", "apple/alarms.list", "apple/photos.get", "apple/calendar.events.update", "apple/device.get", "apple/calendar.events.list", "apple/links.open", "apple/health.workouts.list", "apple/music.playback", "apple/health.workouts.get", "apple/calendar.events.create", "builtin/Memory", "apple/location.get", "apple/reminders.list", "apple/clipboard.read", "apple/battery.get", "builtin/EditFile", "builtin/ScheduledTasks", "apple/clipboard.write", "apple/weather.current", "apple/location.geocode", "builtin/ListFiles", "apple/maps.eta", "apple/alarms.create", "apple/health.statistics", "apple/reminders.delete", "apple/weather.daily", "apple/location.reverseGeocode", "apple/calendar.events.delete", "apple/maps.route", "apple/alarms.cancel", "apple/calendar.events.get", "apple/reminders.get", "apple/reminders.create", "apple/music.status", "apple/maps.show", "apple/weather.hourly" ], "scores": [ 17.849774341290473, 15.377454787411025, 13.806182908520192, 12.345369642655621, 11.488322556861633, 8.174859853217255, 7.979245789539698, 6.707136311329201, 5.607873011311175, 5.435060647018335, 5.17432833749417, 4.7202943297386275, 4.64298484635531, 4.539147981147332, 4.318700352825255, 3.835784789447327, 3.8090423705980267, 3.805978404988366, 3.700132940951465, 3.346960627712961, 3.1955733660614256, 3.1322633171083183, 2.9975321853719246, 2.9650434606571463, 2.9595717523937832, 2.8694088104583075, 2.628985343147514, 2.443999803975162, 2.407070177600925, 2.118755749811426, 2.0023739163303613, 2.002037074655073, 1.651417995754508, 1.5965567909467069, 1.579855424698108, 1.5069513200189237, 1.3691132429326038, 1.1544910774629917, 1.144724074947355, 1.0829958778625002, 0.9971873002352808, 0.9860265225742033, 0.9766794376224938, 0.9510032826687982, 0.9207888081104977, 0.8075967500940832, 0.7436087609996902, 0.5918458513942811 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N074", "query": "Need ordered route steps rather than just ETA", "category": "interference", "domain": "maps", "intent": "capability:apple/maps.route", "target_capability": "Only route provides ordered steps.", "acceptable": [ "apple/maps.route" ], "rationale": "Only route provides ordered steps.", "basis": [ "apple/maps.route" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/ScheduledTasks", "apple/health.statistics", "apple/calendar.events.list", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 6, 4, 2, 1.5, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/weather.hourly", "apple/health.statistics", "apple/weather.daily" ], "scores": [ 11.517844199927476, 7.161051924669758, 4.729910602707946, 3.946846973995474, 2.3354820288800684, 2.219316713009146, 2.1709031724984498 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/health.statistics", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 18.934429400709874, 12.5272408036291, 6.093703470537891, 4.012088681465918, 3.381892963754189, 2.3074020091791767, 2.1370314849333996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.route", "apple/maps.eta", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/health.statistics", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 18.934429400709874, 12.5272408036291, 6.093703470537891, 4.012088681465918, 3.381892963754189, 2.3074020091791767, 2.1370314849333996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N075", "query": "Current conditions only, not hourly or daily forecasts", "category": "interference", "domain": "weather", "intent": "capability:apple/weather.current", "target_capability": "Current conditions endpoint.", "acceptable": [ "apple/weather.current" ], "rationale": "Current conditions endpoint.", "basis": [ "apple/weather.current" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/GetCurrentTime", "builtin/ScheduledTasks", "apple/health.statistics", "apple/maps.show", "apple/music.status", "apple/links.open", "apple/maps.search", "apple/music.playback", "apple/photos.albums.list", "apple/alarms.create", "apple/alarms.list", "apple/location.geocode", "apple/maps.eta", "apple/music.search", "apple/photos.get", "apple/contacts.list", "apple/health.sleep.query", "apple/photos.search", "builtin/EditFile", "apple/alarms.cancel", "apple/battery.get", "apple/calendar.events.update", "apple/clipboard.write", "apple/location.get", "apple/location.reverseGeocode", "apple/maps.route", "apple/reminders.create", "apple/shortcuts.run", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.list", "builtin/Memory" ], "scores": [ 7, 7, 6, 4, 4, 3, 3, 3, 2.5, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/weather.daily", "apple/weather.current", "apple/weather.hourly", "apple/health.statistics", "apple/music.status", "apple/maps.show", "apple/photos.albums.list", "apple/music.search", "apple/location.geocode", "apple/alarms.list", "apple/maps.eta", "apple/photos.get", "apple/links.open", "builtin/GetCurrentTime", "apple/maps.search", "apple/alarms.create", "apple/music.playback", "builtin/ScheduledTasks", "builtin/EditFile", "apple/shortcuts.run", "apple/location.reverseGeocode", "apple/health.sleep.query", "apple/photos.search", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/calendar.events.update", "apple/reminders.create", "apple/location.get", "apple/battery.get", "apple/contacts.list" ], "scores": [ 9.10300216480854, 8.542561466617176, 7.8835763316185785, 4.427080568803885, 4.264065309634576, 3.4782808701074854, 2.997597226393302, 2.9329121219677106, 2.5803144687932615, 2.55493794226612, 2.4567439609153885, 2.423030341884818, 2.3716701253561983, 2.205227515488847, 2.061789892703679, 2.0014596859910685, 1.8836573090718198, 1.866057168636687, 1.8329870145700418, 1.704499249357205, 1.6335467459150792, 1.5629367438392368, 1.4834491626867772, 1.3098389254907072, 1.2842193286009138, 1.2706333365586515, 1.1274973225408556, 1.1119626517018855, 1.031117386694429, 1.018772408912019, 0.9297239873406339 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "apple/weather.daily", "apple/weather.hourly", "apple/weather.current", "builtin/GetCurrentTime", "builtin/ScheduledTasks", "apple/music.status", "apple/health.statistics", "apple/maps.show", "apple/photos.albums.list", "apple/music.search", "apple/music.playback", "builtin/EditFile", "apple/location.geocode", "apple/alarms.list", "apple/links.open", "apple/maps.search", "apple/maps.eta", "apple/photos.get", "apple/alarms.create", "apple/photos.search", "apple/health.sleep.query", "apple/shortcuts.run", "apple/location.reverseGeocode", "apple/contacts.list", "apple/maps.route", "apple/alarms.cancel", "apple/clipboard.write", "apple/calendar.events.update", "apple/reminders.create", "apple/location.get", "apple/battery.get", "builtin/Memory", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list" ], "scores": [ 15.910925430880996, 15.535737656932621, 11.604466775729644, 5.216525260283588, 3.9850167142919273, 3.924506849199173, 3.8482050863252866, 3.171997276621645, 3.170576283019253, 2.6146290117458038, 2.4984268407764976, 2.3540016180843235, 2.32646708381686, 2.25445203382705, 2.250604503418466, 2.2274670693108582, 2.207690207277296, 2.1300610037508156, 1.7767995853368768, 1.6742561077164677, 1.6244277159640166, 1.609381947431756, 1.592799090577726, 1.2500038811300134, 1.2141942578109772, 1.2006046822662546, 1.0742789579478196, 1.0689954759741835, 0.9207888081104977, 0.8493613585822458, 0.8385204047248325, 0.38764824653409546, 0.3147136525322586, 0.30990819698276545, 0.29633377370106745 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/weather.daily", "apple/weather.hourly", "apple/weather.current", "builtin/ScheduledTasks", "builtin/GetCurrentTime", "apple/music.status", "apple/health.statistics", "apple/maps.show", "apple/photos.albums.list", "apple/music.search", "apple/music.playback", "builtin/EditFile", "apple/location.geocode", "apple/alarms.list", "apple/links.open", "apple/maps.search", "apple/maps.eta", "apple/photos.get", "apple/alarms.create", "apple/photos.search", "apple/health.sleep.query", "apple/shortcuts.run", "apple/location.reverseGeocode", "apple/contacts.list", "apple/maps.route", "apple/alarms.cancel", "apple/clipboard.write", "apple/calendar.events.update", "apple/reminders.create", "apple/location.get", "apple/battery.get", "builtin/Memory", "apple/reminders.list", "apple/calendar.events.list", "apple/health.workouts.list" ], "scores": [ 15.910925430880996, 15.535737656932621, 11.604466775729644, 7.792751602329204, 7.081303815323238, 3.924506849199173, 3.8482050863252866, 3.171997276621645, 3.170576283019253, 2.6146290117458038, 2.4984268407764976, 2.3540016180843235, 2.32646708381686, 2.25445203382705, 2.250604503418466, 2.2274670693108582, 2.207690207277296, 2.1300610037508156, 1.7767995853368768, 1.6742561077164677, 1.6244277159640166, 1.609381947431756, 1.592799090577726, 1.2500038811300134, 1.2141942578109772, 1.2006046822662546, 1.0742789579478196, 1.0689954759741835, 0.9207888081104977, 0.8493613585822458, 0.8385204047248325, 0.38764824653409546, 0.3147136525322586, 0.30990819698276545, 0.29633377370106745 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } } } }, { "query_id": "N076", "query": "Show Anan player status without changing playback", "category": "interference", "domain": "music", "intent": "capability:apple/music.status", "target_capability": "Read-only ApplicationMusicPlayer snapshot.", "acceptable": [ "apple/music.status" ], "rationale": "Read-only ApplicationMusicPlayer snapshot.", "basis": [ "apple/music.status" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/music.status", "apple/music.playback", "apple/maps.show", "apple/clipboard.read", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.update", "apple/contacts.delete", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/music.search", "apple/photos.search" ], "scores": [ 7, 6, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/music.status", "apple/music.playback", "apple/clipboard.read", "apple/maps.show", "apple/location.reverseGeocode", "apple/calendar.events.update", "apple/location.geocode", "apple/music.search", "apple/battery.get", "apple/contacts.delete", "apple/maps.search", "apple/alarms.list", "apple/alarms.cancel", "apple/alarms.create", "apple/photos.search" ], "scores": [ 14.657801238845083, 7.627930974000441, 4.9183061938508885, 4.126968314047327, 2.8055106012874256, 2.5943235765196997, 2.5938411871930867, 2.4762793824936646, 2.4696228887134084, 2.430546544538726, 2.072598362607447, 1.9107839181010244, 1.8905693842378057, 1.7430490933692009, 1.6973910453637415 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/music.status", "apple/music.playback", "apple/maps.show", "apple/clipboard.read", "apple/location.reverseGeocode", "apple/location.geocode", "apple/calendar.events.update", "apple/battery.get", "apple/contacts.delete", "apple/music.search", "apple/maps.search", "apple/alarms.list", "apple/alarms.cancel", "apple/alarms.create", "apple/photos.search" ], "scores": [ 23.06152614614118, 15.040087233138701, 12.510782572457543, 4.894657100303657, 2.877556815390035, 2.606627671177044, 2.587420718259699, 2.4735473042818215, 2.456488742268665, 2.454140563572183, 2.054068595663089, 1.9003916383525448, 1.8791219646015662, 1.724696020263421, 1.6656673071721346 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/music.status", "apple/music.playback", "apple/maps.show", "apple/alarms.list", "apple/clipboard.read", "apple/location.reverseGeocode", "apple/location.geocode", "apple/calendar.events.update", "apple/battery.get", "apple/contacts.delete", "apple/music.search", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/health.workouts.list", "apple/maps.search", "apple/alarms.cancel", "apple/alarms.create", "apple/photos.search" ], "scores": [ 23.06152614614118, 15.040087233138701, 7.1387720841183455, 4.336796897647155, 3.6182902363542286, 2.877556815390035, 2.606627671177044, 2.587420718259699, 2.4735473042818215, 2.456488742268665, 2.454140563572183, 2.311584248804978, 2.311584248804978, 2.253850044094424, 2.253850044094424, 2.19892950792101, 2.09674493165168, 2.054068595663089, 1.8791219646015662, 1.724696020263421, 1.6656673071721346 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N077", "query": "Show what is inside this directory", "category": "paraphrase", "domain": "files", "intent": "capability:builtin/ListFiles", "target_capability": "Directory names, not recursive content.", "acceptable": [ "builtin/ListFiles" ], "rationale": "Directory names, not recursive content.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/maps.show", "apple/clipboard.read", "builtin/ListFiles", "builtin/Memory" ], "scores": [ 4, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "name_description": { "ranking": [ "builtin/ListFiles", "apple/maps.show", "builtin/Memory", "apple/clipboard.read" ], "scores": [ 4.7220808488051, 4.126968314047327, 3.269566973945597, 2.986870790172134 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.show", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read" ], "scores": [ 12.510782572457543, 4.866980797417192, 3.2314784752437844, 2.9725087752942105 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/ListFiles", "apple/maps.show", "builtin/Memory", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/health.workouts.list", "apple/clipboard.read" ], "scores": [ 9.380532797138189, 7.1387720841183455, 3.2314784752437844, 2.4364052592946104, 2.311584248804978, 2.311584248804978, 2.253850044094424, 2.253850044094424, 2.19892950792101, 2.09674493165168, 1.6961419113447813 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N078", "query": "能把这个文件打开给我看文字吗", "category": "paraphrase", "domain": "files", "intent": "capability:builtin/ReadFile", "target_capability": "Ordinary UTF-8 text reading.", "acceptable": [ "builtin/ReadFile" ], "rationale": "Ordinary UTF-8 text reading.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open" ], "scores": [ 2.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/links.open" ], "scores": [ 2.0426419959677764 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/links.open" ], "scores": [ 3.0453088181459442 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/ListFiles", "apple/maps.show", "builtin/WriteFile", "apple/links.open" ], "scores": [ 7.615469776074555, 5.5906952380295865, 4.543804441374347, 3.9133926474172007, 3.1217696103861448 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N079", "query": "Write this paragraph into a brand-new file", "category": "paraphrase", "domain": "files", "intent": "capability:builtin/WriteFile", "target_capability": "New complete file creation.", "acceptable": [ "builtin/WriteFile" ], "rationale": "New complete file creation.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "apple/clipboard.write", "builtin/ReadFile", "builtin/ListFiles", "builtin/Memory", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks", "apple/calendar.events.update", "apple/shortcuts.run" ], "scores": [ 7.5, 5, 4.5, 4, 3, 2.5, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/Memory", "builtin/ReadFile", "builtin/EditFile", "builtin/ListFiles", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 7.594051058282012, 3.7936376826106004, 3.4744596028470913, 3.472413797389085, 3.374072336740228, 3.053494701017844, 1.931435403678754, 1.4705301552404113, 1.3804900280474186 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/EditFile", "builtin/ReadFile", "builtin/Memory", "builtin/ListFiles", "apple/clipboard.read", "apple/calendar.events.update", "apple/photos.search", "builtin/ScheduledTasks", "apple/shortcuts.run" ], "scores": [ 18.084202584958113, 11.754253707105612, 10.50348858999096, 8.549670258073231, 6.418646056526877, 5.804260981819989, 1.8024824369174723, 1.7823784642970022, 1.3532076178115147, 1.214637928883098, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/WriteFile", "apple/clipboard.write", "builtin/ReadFile", "builtin/EditFile", "builtin/Memory", "builtin/ListFiles", "apple/alarms.create", "apple/contacts.create", "apple/calendar.events.create", "apple/reminders.create", "apple/clipboard.read", "apple/photos.search", "builtin/ScheduledTasks", "apple/calendar.events.update", "apple/shortcuts.run" ], "scores": [ 26.96487829706563, 11.754253707105612, 11.138827250019764, 10.10529654897063, 5.998391305623579, 5.804260981819989, 2.9026295885011453, 2.7369467418327798, 2.6610014176998558, 2.5891569919465325, 1.8024824369174723, 1.3532076178115147, 1.214637928883098, 1.2119692675107079, 0.7660997030790287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N080", "query": "只改这一处错别字，其余内容别动", "category": "paraphrase", "domain": "files", "intent": "capability:builtin/EditFile", "target_capability": "One matched edit.", "acceptable": [ "builtin/EditFile" ], "rationale": "One matched edit.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/ReadFile" ], "scores": [ 4.971371455436067, 4.971371455436067 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N081", "query": "下次聊天也要记得我喜欢早上运动", "category": "paraphrase", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Cross-conversation lasting preference.", "acceptable": [ "builtin/Memory" ], "rationale": "Cross-conversation lasting preference.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list" ], "scores": [ 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list" ], "scores": [ 3.578355194658009, 3.17801937787516, 3.009663257150823, 2.858247126213771 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list" ], "scores": [ 4.743213198415311, 4.4165274417413745, 3.203146750471291, 2.979870702571842 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "N082", "query": "What do you remember about my preferences?", "category": "paraphrase", "domain": "memory", "intent": "capability:builtin/Memory", "target_capability": "Inspect stored lasting memory.", "acceptable": [ "builtin/Memory" ], "rationale": "Inspect stored lasting memory.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/health.sleep.query", "apple/photos.albums.list" ], "scores": [ 1.5, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/health.sleep.query", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 2.9863868734039265, 2.3135578186715415, 1.5645258580128296 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/health.sleep.query", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 3.032739008254341, 2.3074020091791767, 1.8100326189850766 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/Memory", "apple/health.sleep.query", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 9.433359994200462, 3.032739008254341, 2.3074020091791767, 1.8100326189850766 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N083", "query": "What weekday is it on this device right now?", "category": "paraphrase", "domain": "clock", "intent": "capability:builtin/GetCurrentTime", "target_capability": "Current device weekday.", "acceptable": [ "builtin/GetCurrentTime" ], "rationale": "Current device weekday.", "basis": [ "builtin/GetCurrentTime" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/device.get", "builtin/GetCurrentTime", "apple/battery.get", "apple/calendar.events.create", "apple/clipboard.write", "apple/weather.current", "apple/weather.daily", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 4, 2, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "apple/device.get", "apple/battery.get", "apple/clipboard.write", "apple/calendar.events.create", "apple/weather.current", "apple/weather.daily" ], "scores": [ 7.303500917572945, 3.282365175825118, 2.1395511304507666, 1.9961579936474543, 1.7602177194007949, 1.726212147301921, 1.5741568020017251 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/device.get", "builtin/GetCurrentTime", "apple/battery.get", "apple/clipboard.write", "apple/calendar.events.create", "apple/weather.current", "apple/weather.daily", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 8.03155338721935, 6.540679467896122, 1.8909826161831347, 1.756468859475338, 1.5538294345152088, 1.5064312850912929, 1.367394238902415, 0.8661467368021426, 0.5778139638332612 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/GetCurrentTime", "apple/device.get", "apple/battery.get", "apple/clipboard.write", "apple/calendar.events.create", "apple/weather.current", "apple/weather.daily", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 16.309370089847565, 8.03155338721935, 1.8909826161831347, 1.756468859475338, 1.5538294345152088, 1.5064312850912929, 1.367394238902415, 0.8661467368021426, 0.5778139638332612 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N084", "query": "给我两个选项让我点一下再继续", "category": "paraphrase", "domain": "questions", "intent": "capability:builtin/AskUserQuestions", "target_capability": "Interactive question card (no model run in experiment).", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Interactive question card (no model run in experiment).", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/AskUserQuestions" ], "scores": [ 5.409527227963967 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N085", "query": "Set up a weekly read-only summary check", "category": "paraphrase", "domain": "scheduling", "intent": "capability:builtin/ScheduledTasks", "target_capability": "Weekly scheduled task.", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Weekly scheduled task.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/ScheduledTasks", "apple/clipboard.read", "builtin/ReadFile", "apple/alarms.list", "apple/health.statistics", "apple/photos.get", "apple/photos.search", "apple/links.open", "apple/music.playback", "builtin/Memory", "apple/alarms.cancel", "apple/alarms.create", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.list", "apple/calendar.events.update", "apple/contacts.get", "apple/device.get", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.albums.list", "apple/reminders.complete", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/EditFile" ], "scores": [ 5, 4, 4, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ScheduledTasks", "apple/reminders.complete", "apple/location.reverseGeocode", "apple/calendar.events.create", "apple/location.geocode", "apple/health.statistics", "apple/alarms.list", "apple/photos.search", "apple/maps.route", "apple/photos.get", "apple/maps.search", "apple/music.status", "builtin/ReadFile", "apple/music.search", "apple/shortcuts.run", "apple/photos.albums.list", "builtin/Memory", "apple/contacts.get", "apple/links.open", "builtin/EditFile", "apple/clipboard.read", "apple/maps.show", "apple/alarms.cancel", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/alarms.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "apple/music.playback", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 6.110797202469009, 4.986694102554394, 2.8055106012874256, 2.722090979800692, 2.5938411871930867, 2.373837406486867, 2.2393285067409856, 2.2184966134721584, 2.2055634157869286, 2.1043278842905497, 2.072598362607447, 1.892501474377823, 1.7356930089104854, 1.7166803960897803, 1.704499249357205, 1.634894371444502, 1.6208496848238496, 1.596129569605437, 1.51183557069496, 1.3632427748418257, 1.3439750578113865, 1.2842193286009138, 1.2706333365586515, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.1714863806419569, 1.1274973225408556, 1.0956951902365162, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 0.9418286545359099, 0.8464971647744733, 0.7868454382417407 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ScheduledTasks", "apple/reminders.complete", "builtin/ReadFile", "apple/clipboard.read", "apple/location.reverseGeocode", "apple/location.geocode", "apple/calendar.events.create", "apple/health.statistics", "apple/links.open", "apple/maps.route", "apple/alarms.list", "apple/photos.search", "apple/maps.search", "apple/photos.get", "builtin/Memory", "apple/music.status", "apple/music.playback", "apple/music.search", "apple/shortcuts.run", "apple/contacts.get", "apple/photos.albums.list", "builtin/EditFile", "apple/health.workouts.get", "apple/health.workouts.list", "apple/weather.current", "apple/maps.show", "apple/alarms.cancel", "apple/calendar.events.list", "apple/alarms.create", "apple/health.sleep.query", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/calendar.events.update", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 7.253588478365282, 5.0937424363493315, 4.557289964100901, 4.054675138958545, 2.877556815390035, 2.606627671177044, 2.4284016484087565, 2.326976247995523, 2.3128478775067216, 2.1935679034724593, 2.164108834052035, 2.111667130648512, 2.054068595663089, 2.0264829671269666, 1.9987894289795634, 1.798233682123699, 1.621653562440418, 1.62154593790355, 1.609381947431756, 1.60914410629741, 1.5512004698252941, 1.3906556055226948, 1.3042089742613945, 1.2461128550311649, 1.231374918430577, 1.2141942578109772, 1.2006046822662546, 1.113807122507814, 1.1019391803305796, 1.0992184654999453, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 1.0689954759741835, 0.8363195411534453, 0.7745686203791382 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ScheduledTasks", "apple/reminders.complete", "builtin/ReadFile", "apple/location.reverseGeocode", "apple/clipboard.read", "apple/alarms.list", "apple/photos.get", "apple/location.geocode", "apple/photos.albums.list", "apple/calendar.events.create", "apple/links.open", "apple/maps.route", "apple/contacts.get", "apple/health.statistics", "apple/maps.search", "apple/photos.search", "apple/health.workouts.get", "apple/device.get", "apple/music.status", "apple/battery.get", "apple/health.workouts.list", "apple/calendar.events.list", "apple/music.playback", "apple/music.search", "apple/shortcuts.run", "builtin/Memory", "apple/maps.show", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.update", "builtin/EditFile", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/calendar.list", "apple/reminders.list", "apple/weather.current", "apple/health.sleep.query", "apple/maps.eta", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 11.88906579848162, 5.0937424363493315, 4.201778249894578, 2.877556815390035, 2.8694088104583075, 2.8631083778275244, 2.7702425710709373, 2.606627671177044, 2.477843028786731, 2.4284016484087565, 2.3128478775067216, 2.1935679034724593, 2.1425381623194113, 2.1214225279739036, 2.054068595663089, 1.900813242648144, 1.8264601231288218, 1.8002653409399807, 1.798233682123699, 1.7904649384226818, 1.7223681387479477, 1.6917167140453433, 1.621653562440418, 1.62154593790355, 1.609381947431756, 1.5278191715440528, 1.2141942578109772, 1.2006046822662546, 1.1019391803305796, 1.0689954759741835, 0.9841378926907047, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.8714182810782765, 0.7778939227999769, 0.7580363862218745, 0.5918458513942811, 0.5481460160064278 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N086", "query": "Move this existing non-recurring meeting to a different start and end", "category": "paraphrase", "domain": "calendar", "intent": "capability:apple/calendar.events.update", "target_capability": "Update event time fields.", "acceptable": [ "apple/calendar.events.update" ], "rationale": "Update event time fields.", "basis": [ "apple/calendar.events.update" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.update", "apple/calendar.events.create", "apple/photos.search", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/health.statistics", "apple/reminders.delete", "builtin/WriteFile", "apple/calendar.events.list", "apple/health.sleep.query", "apple/health.workouts.list", "apple/maps.show", "apple/reminders.complete", "apple/reminders.update", "apple/clipboard.write", "builtin/Memory" ], "scores": [ 4.5, 3, 3, 2.5, 2.5, 2, 2, 1.5, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.events.update", "apple/calendar.events.get", "apple/photos.search", "apple/reminders.delete", "apple/calendar.events.delete", "builtin/WriteFile", "apple/calendar.events.create", "apple/reminders.complete", "apple/maps.show", "apple/reminders.update", "apple/health.statistics" ], "scores": [ 6.849960597094626, 6.296294548004108, 6.0878998829043915, 5.793863443924959, 5.354388674089671, 5.065193500893222, 3.9676810351741616, 2.769559846226892, 2.6351421055956448, 2.430546544538726, 1.9791353630524071 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.events.update", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/reminders.delete", "apple/calendar.events.create", "apple/photos.search", "builtin/WriteFile", "apple/reminders.complete", "apple/health.statistics", "apple/reminders.update", "apple/maps.show", "apple/health.sleep.query", "apple/clipboard.write", "apple/calendar.events.list", "apple/health.workouts.list", "builtin/Memory" ], "scores": [ 8.264872077694031, 7.1378899019620725, 6.606075755622257, 5.881337557833275, 5.327769267623741, 5.2814844234348, 4.802385981774975, 2.829013416224716, 2.7391025393908537, 2.456488742268665, 1.6769433783228374, 1.5353669870143491, 1.4956890564914902, 1.4415911021883758, 1.378447345389575, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.update", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/reminders.delete", "apple/photos.search", "apple/calendar.events.create", "builtin/WriteFile", "apple/health.statistics", "apple/reminders.complete", "builtin/ScheduledTasks", "apple/reminders.update", "apple/maps.show", "apple/health.sleep.query", "apple/clipboard.write", "apple/calendar.events.list", "apple/health.workouts.list", "builtin/Memory" ], "scores": [ 8.113722710061396, 6.962669555400861, 6.435693630175795, 5.718898966742206, 5.2814844234348, 5.218143782880374, 4.802385981774975, 2.7391025393908537, 2.6528894415699615, 2.5891569919465325, 2.30355678425753, 1.6769433783228374, 1.5353669870143491, 1.4956890564914902, 1.4415911021883758, 1.378447345389575, 0.5689536714487385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N087", "query": "Rename an existing non-recurring to-do item", "category": "paraphrase", "domain": "reminders", "intent": "capability:apple/reminders.update", "target_capability": "Update reminder title.", "acceptable": [ "apple/reminders.update" ], "rationale": "Update reminder title.", "basis": [ "apple/reminders.update" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "apple/clipboard.write", "apple/reminders.delete", "apple/links.open", "builtin/WriteFile", "apple/calendar.events.get", "apple/contacts.list", "apple/health.sleep.query", "apple/location.geocode", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.list", "apple/reminders.update" ], "scores": [ 2.5, 2.5, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.06666666666666667 } }, "name_description": { "ranking": [ "apple/reminders.delete", "apple/calendar.events.delete", "builtin/WriteFile", "apple/calendar.events.update", "apple/calendar.events.create", "apple/clipboard.write", "apple/health.sleep.query", "apple/location.geocode", "apple/reminders.complete", "apple/calendar.events.get", "apple/contacts.list", "apple/reminders.list", "apple/reminders.update", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 5.793863443924959, 5.354388674089671, 5.065193500893222, 4.536402778423085, 3.9676810351741616, 3.947200460209589, 2.9863868734039265, 2.8181396909485716, 2.769559846226892, 2.6465143302464305, 2.448649573649498, 2.448649573649498, 2.430546544538726, 2.3135578186715415, 1.5645258580128296 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.07692307692307693 } }, "bm25": { "ranking": [ "apple/calendar.events.delete", "apple/clipboard.write", "apple/reminders.delete", "apple/calendar.events.update", "builtin/WriteFile", "apple/calendar.events.create", "apple/health.sleep.query", "apple/location.geocode", "apple/reminders.complete", "apple/calendar.events.get", "apple/reminders.update", "apple/contacts.list", "apple/reminders.list", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 7.1378899019620725, 6.932704567516031, 5.881337557833275, 5.411237147255189, 4.802385981774975, 3.9691582916733155, 3.032739008254341, 2.8320318668461497, 2.829013416224716, 2.736784243699854, 2.456488742268665, 2.438454839822002, 2.438454839822002, 2.3074020091791767, 1.8100326189850766 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.09090909090909091 } }, "combined": { "ranking": [ "apple/calendar.events.update", "apple/calendar.events.delete", "apple/clipboard.write", "apple/reminders.delete", "apple/reminders.update", "builtin/WriteFile", "apple/calendar.events.create", "apple/contacts.update", "apple/health.sleep.query", "apple/location.geocode", "apple/reminders.complete", "builtin/ScheduledTasks", "apple/calendar.events.get", "apple/contacts.list", "apple/reminders.list", "apple/photos.albums.list", "apple/links.open" ], "scores": [ 8.696683794059664, 6.962669555400861, 6.932704567516031, 5.718898966742206, 5.658403794554779, 4.802385981774975, 3.859532806929948, 3.522428550243476, 3.032739008254341, 2.8320318668461497, 2.6528894415699615, 2.5891569919465325, 2.5664021182533916, 2.438454839822002, 2.438454839822002, 2.3074020091791767, 1.8100326189850766 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } } } }, { "query_id": "N088", "query": "Change the email addresses on an existing contact", "category": "paraphrase", "domain": "contacts", "intent": "capability:apple/contacts.update", "target_capability": "Supplied email array replaces contact emails.", "acceptable": [ "apple/contacts.update" ], "rationale": "Supplied email array replaces contact emails.", "basis": [ "apple/contacts.update" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.list", "apple/contacts.update", "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete", "apple/calendar.events.update", "apple/location.geocode", "builtin/ScheduledTasks", "builtin/WriteFile", "apple/calendar.events.list", "apple/music.playback", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.list", "apple/reminders.update", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.search" ], "scores": [ 7, 6.5, 5.5, 5, 4.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.get", "apple/contacts.update", "builtin/WriteFile", "apple/contacts.create", "apple/contacts.delete", "apple/location.geocode", "apple/reminders.complete", "apple/calendar.events.update", "apple/reminders.update", "apple/calendar.events.list", "apple/reminders.list", "apple/photos.albums.list", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 11.02606461196649, 7.5079945717602055, 5.981213781756763, 5.065193500893222, 3.908385007994482, 3.6211209622787783, 3.0990449327715073, 2.443914691723882, 2.146531974526627, 2.1447626117230607, 1.934298977912546, 1.7229585090753954, 1.4803458231303015, 1.2365724396619533, 0.8833160037757247 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "bm25": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete", "builtin/WriteFile", "apple/location.geocode", "apple/calendar.events.update", "apple/reminders.complete", "apple/reminders.update", "apple/calendar.events.list", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/reminders.list", "apple/clipboard.write", "apple/photos.albums.list", "apple/music.playback", "apple/maps.search" ], "scores": [ 16.202182820544238, 16.003843554347387, 14.674396084996033, 13.640479383087172, 11.146694165037609, 4.802385981774975, 4.078889372840208, 3.027516332067192, 2.4963777043542335, 2.167654522960934, 1.9662731002711833, 1.7350573342035134, 1.7199245769099551, 1.7157851251886742, 1.4956890564914902, 1.4764069862460605, 1.2109257920942886, 1.0790560593678742 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.get", "apple/contacts.create", "apple/contacts.delete", "builtin/WriteFile", "apple/calendar.events.update", "apple/location.geocode", "apple/reminders.update", "apple/reminders.complete", "builtin/EditFile", "apple/calendar.events.list", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/reminders.list", "apple/clipboard.write", "apple/photos.albums.list", "apple/music.playback", "apple/maps.search" ], "scores": [ 18.213307207115665, 15.81906414820596, 14.674396084996033, 13.640479383087172, 11.146694165037609, 4.802385981774975, 4.758912882719043, 4.078889372840208, 3.8496555889810375, 2.227533323467932, 2.174019538423646, 1.754517694278828, 1.7350573342035134, 1.62788265285601, 1.5310057190472472, 1.4956890564914902, 1.3174071195747592, 1.08051660188761, 1.0790560593678742 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N089", "query": "What percentage of battery is left?", "category": "paraphrase", "domain": "device", "intent": "capability:apple/battery.get", "target_capability": "Snapshot levelPercent, null when unavailable.", "acceptable": [ "apple/battery.get" ], "rationale": "Snapshot levelPercent, null when unavailable.", "basis": [ "apple/battery.get" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/battery.get", "builtin/ScheduledTasks" ], "scores": [ 4, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/battery.get" ], "scores": [ 5.874055306494094 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/battery.get", "builtin/ScheduledTasks" ], "scores": [ 14.036671830278951, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/battery.get", "builtin/ScheduledTasks" ], "scores": [ 14.036671830278951, 0.7641391704553132 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N090", "query": "Find images created between two explicit dates", "category": "paraphrase", "domain": "photos", "intent": "capability:apple/photos.search", "target_capability": "Photo metadata search by creation time range.", "acceptable": [ "apple/photos.search" ], "rationale": "Photo metadata search by creation time range.", "basis": [ "apple/photos.search" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/photos.search", "apple/calendar.events.update", "apple/maps.eta", "apple/maps.route", "apple/weather.daily", "apple/maps.search", "apple/reminders.create", "apple/alarms.create", "apple/calendar.events.create", "apple/clipboard.read", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.show", "apple/photos.albums.list", "apple/photos.get", "apple/reminders.list", "apple/weather.current", "apple/weather.hourly", "builtin/GetCurrentTime", "apple/reminders.update" ], "scores": [ 3, 2, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/photos.search", "apple/maps.eta", "apple/maps.route", "apple/photos.get", "apple/weather.daily", "apple/calendar.events.update", "builtin/GetCurrentTime", "apple/reminders.create", "apple/clipboard.read", "apple/photos.albums.list", "apple/reminders.list", "apple/location.reverseGeocode", "apple/location.geocode", "apple/maps.show", "apple/maps.search", "apple/calendar.events.create", "apple/weather.current", "apple/weather.hourly" ], "scores": [ 5.922503418434697, 4.993370105840739, 4.459464024753976, 3.728316885706285, 3.7266753596634477, 3.1179287586720683, 2.7935194165293797, 2.5014830678108915, 2.4221849112268674, 2.1038511081773286, 2.0915170203879607, 1.9137849934359552, 1.769394254695836, 1.5045297512928304, 1.41382735889783, 1.3859756210546532, 1.3592000163157691, 1.3334393574442729 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/photos.search", "apple/maps.eta", "apple/maps.route", "apple/photos.get", "apple/weather.daily", "apple/reminders.create", "apple/calendar.events.update", "builtin/GetCurrentTime", "apple/clipboard.read", "apple/photos.albums.list", "apple/maps.search", "apple/alarms.create", "apple/location.reverseGeocode", "apple/reminders.list", "apple/location.geocode", "apple/maps.show", "apple/calendar.events.create", "apple/weather.current", "apple/weather.hourly", "apple/reminders.update" ], "scores": [ 7.398953335418348, 4.912427915285203, 4.356393016797557, 3.7098468272112193, 3.308083319307457, 2.9977146275649744, 2.816986443432833, 2.525239230283221, 2.410538121601221, 2.0982532767690447, 2.078782606482171, 1.9197383157600845, 1.8595379756185302, 1.8233571925740666, 1.6844578417802005, 1.4175298978595754, 1.3134609719514998, 1.2733950432027226, 1.2480153304315296, 0.7910171978568801 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/photos.search", "apple/photos.get", "builtin/GetCurrentTime", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/photos.albums.list", "apple/weather.daily", "apple/reminders.list", "apple/reminders.create", "apple/calendar.events.update", "apple/clipboard.read", "apple/alarms.create", "apple/location.reverseGeocode", "apple/music.search", "apple/location.geocode", "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/alarms.list", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/maps.show", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.get", "apple/calendar.events.create", "apple/health.workouts.list", "apple/weather.current", "apple/weather.hourly", "apple/reminders.update" ], "scores": [ 9.104210823364433, 5.2731508777673195, 5.262185972116001, 4.912427915285203, 4.356393016797557, 3.686703556543879, 3.541421407480609, 3.308083319307457, 3.196192722645459, 2.9977146275649744, 2.816986443432833, 2.410538121601221, 1.9197383157600845, 1.8595379756185302, 1.7052574879460844, 1.6844578417802005, 1.607920950061708, 1.607920950061708, 1.5633040505560998, 1.5210963760156138, 1.5210963760156138, 1.5210963760156138, 1.4811079163986542, 1.4431681307115642, 1.4175298978595754, 1.407123515710695, 1.407123515710695, 1.407123515710695, 1.3134609719514998, 1.30903966193533, 1.2733950432027226, 1.2480153304315296, 0.7910171978568801 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "N091", "query": "Delete an ordinary workspace file permanently", "category": "unsupported", "domain": "files", "intent": "capability:files-unsupported", "target_capability": "No ordinary DeleteFile tool; Memory delete is limited to lasting memory.", "acceptable": [], "rationale": "No ordinary DeleteFile tool; Memory delete is limited to lasting memory.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "apple/calendar.events.delete", "apple/contacts.delete", "apple/reminders.delete", "builtin/ListFiles", "builtin/Memory", "builtin/ScheduledTasks", "apple/clipboard.read", "apple/links.open", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 5, 5, 4.5, 4, 4, 4, 3.5, 3, 1.5, 1, 1, 1, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "builtin/Memory", "apple/reminders.delete", "builtin/WriteFile", "apple/contacts.delete", "builtin/ReadFile", "builtin/EditFile", "apple/calendar.events.delete", "builtin/ListFiles", "apple/links.open", "apple/clipboard.read", "builtin/ScheduledTasks", "apple/photos.search" ], "scores": [ 5.91486491675378, 3.752882783565409, 3.714454817908332, 3.6211209622787783, 3.472413797389085, 3.374072336740228, 3.224841952905041, 3.053494701017844, 2.0426419959677764, 1.931435403678754, 1.8844063547818743, 1.4705301552404113 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "apple/reminders.delete", "apple/contacts.delete", "builtin/ReadFile", "apple/calendar.events.delete", "builtin/Memory", "builtin/ListFiles", "builtin/ScheduledTasks", "apple/links.open", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 11.515668544828896, 10.390707067622014, 10.156196128710354, 9.97993715650646, 9.788911069686547, 8.698195352050329, 7.456877027846374, 7.173973165253237, 2.1553040912044485, 1.9842846172046136, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "builtin/Memory", "builtin/ListFiles", "builtin/ScheduledTasks", "apple/links.open", "apple/clipboard.read", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 14.176669962528752, 12.37806806163308, 10.390707067622014, 10.156196128710354, 9.97993715650646, 8.698195352050329, 7.456877027846374, 7.173973165253237, 2.1553040912044485, 1.9842846172046136, 1.8024824369174723, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "false_recall": true } } } }, { "query_id": "N092", "query": "Run a Python program with a local shell", "category": "unsupported", "domain": "files", "intent": "capability:files-unsupported", "target_capability": "Shell is excluded when Sandbox is off.", "acceptable": [], "rationale": "Shell is excluded when Sandbox is off.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/shortcuts.run", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.write", "apple/device.get", "apple/health.statistics", "apple/maps.eta", "apple/maps.route", "apple/weather.daily", "builtin/ScheduledTasks", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/device.get", "apple/maps.eta", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/calendar.events.update", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 2.7491696581568283, 2.5246596325529898, 2.030662501799527, 2.0063505429558752, 1.9146581234509097, 1.871884535676648, 1.7918255362028437, 1.5731568973736139, 1.4761555867694751, 1.3457586502029633 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/device.get", "apple/maps.eta", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/calendar.events.update", "apple/weather.daily", "apple/health.statistics", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 9.571919453519563, 2.4574770197500073, 1.8073716355678369, 1.784302970733479, 1.7184999580546296, 1.6573777977338993, 1.582338740761588, 1.393115594519765, 1.290252793313428, 1.1713329690232368, 0.7023573504548203, 0.7023573504548203 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/device.get", "apple/maps.eta", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/calendar.events.update", "apple/weather.daily", "apple/health.statistics", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 15.645551689004161, 2.4574770197500073, 1.8073716355678369, 1.784302970733479, 1.7184999580546296, 1.6573777977338993, 1.582338740761588, 1.393115594519765, 1.290252793313428, 1.1713329690232368, 0.7023573504548203, 0.7023573504548203 ], "metrics": { "false_recall": true } } } }, { "query_id": "N093", "query": "Copy a binary file between phone and remote sandbox", "category": "unsupported", "domain": "files", "intent": "capability:files-unsupported", "target_capability": "CopyFile and remote environment are disabled in this profile.", "acceptable": [], "rationale": "CopyFile and remote environment are disabled in this profile.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "builtin/EditFile", "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "apple/contacts.update", "builtin/Memory", "apple/clipboard.read", "apple/contacts.list", "apple/maps.eta", "apple/maps.route", "apple/photos.search", "apple/contacts.create", "apple/shortcuts.run" ], "scores": [ 5, 5, 4.5, 3.5, 1.5, 1.5, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "builtin/WriteFile", "builtin/Memory", "builtin/ReadFile", "builtin/EditFile", "apple/maps.eta", "builtin/ListFiles", "apple/contacts.list", "apple/maps.route", "apple/contacts.update", "apple/clipboard.read", "apple/photos.search" ], "scores": [ 3.714454817908332, 3.4744596028470913, 3.472413797389085, 3.374072336740228, 3.3087116263123417, 3.053494701017844, 3.0195051802203676, 2.954934273461146, 2.9236734812969383, 1.931435403678754, 1.4705301552404113 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "builtin/WriteFile", "builtin/EditFile", "builtin/ReadFile", "builtin/ListFiles", "builtin/Memory", "apple/contacts.update", "apple/maps.eta", "apple/maps.route", "apple/contacts.list", "apple/clipboard.read", "apple/contacts.create", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 11.27380616240516, 10.161465587506916, 9.537833286477701, 6.896461082510534, 4.344574198308581, 3.4137247956955434, 3.3139694166725753, 2.9388631189379812, 1.9444089450290927, 1.8024824369174723, 1.6141984136195984, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/contacts.update", "apple/maps.eta", "apple/maps.route", "apple/contacts.list", "apple/clipboard.read", "apple/contacts.create", "apple/photos.search", "apple/shortcuts.run" ], "scores": [ 13.934807580105016, 12.126990278424234, 10.161465587506916, 6.896461082510534, 4.344574198308581, 3.4137247956955434, 3.3139694166725753, 2.9388631189379812, 1.9444089450290927, 1.8024824369174723, 1.6141984136195984, 1.3532076178115147, 0.7660997030790287 ], "metrics": { "false_recall": true } } } }, { "query_id": "N094", "query": "Write a new heart-rate sample into HealthKit", "category": "unsupported", "domain": "health", "intent": "capability:health-unsupported", "target_capability": "Health declarations are read-only.", "acceptable": [], "rationale": "Health declarations are read-only.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/health.statistics", "apple/clipboard.write", "apple/health.workouts.get", "apple/health.sleep.query", "apple/health.workouts.list", "builtin/WriteFile", "builtin/ScheduledTasks", "apple/alarms.create", "apple/alarms.list", "apple/location.get", "apple/maps.search", "apple/photos.albums.list", "apple/photos.get", "apple/weather.current", "builtin/Memory", "apple/calendar.events.update", "builtin/EditFile" ], "scores": [ 9, 4.5, 3.5, 3, 3, 3, 2.5, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/health.statistics", "builtin/WriteFile", "apple/clipboard.write", "apple/location.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.list", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/maps.search", "apple/photos.get", "apple/alarms.create", "apple/weather.current", "apple/photos.albums.list" ], "scores": [ 16.630978421216074, 3.8795962403736803, 3.7936376826106004, 3.348804949568758, 2.9248512842697183, 2.8055106012874256, 2.6686700803525736, 2.535933711807245, 2.499548551624379, 1.7955900028110616, 1.7955900028110616, 1.7430490933692009, 1.726212147301921, 1.6775979800585423 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/health.statistics", "apple/clipboard.write", "apple/health.workouts.get", "builtin/WriteFile", "apple/health.sleep.query", "apple/health.workouts.list", "builtin/ScheduledTasks", "apple/location.get", "apple/alarms.list", "builtin/Memory", "apple/calendar.events.update", "apple/maps.search", "apple/photos.get", "apple/alarms.create", "apple/weather.current", "apple/photos.albums.list", "builtin/EditFile" ], "scores": [ 27.6613469149855, 11.754253707105612, 7.956595300798631, 7.7622905376693065, 6.315869587661187, 6.315869587661187, 3.6984694221389542, 3.3568146346643974, 2.6585178698196947, 2.074071858218296, 1.7823784642970022, 1.779536789183193, 1.779536789183193, 1.724696020263421, 1.7071592606801997, 1.6731343036002873, 1.2442452235478334 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/health.statistics", "builtin/WriteFile", "apple/clipboard.write", "apple/health.workouts.get", "apple/health.sleep.query", "apple/health.workouts.list", "apple/alarms.create", "builtin/ScheduledTasks", "apple/location.get", "apple/contacts.create", "apple/calendar.events.create", "apple/alarms.list", "apple/reminders.create", "apple/maps.search", "apple/photos.get", "apple/weather.current", "apple/photos.albums.list", "builtin/Memory", "apple/calendar.events.update", "builtin/EditFile" ], "scores": [ 27.6613469149855, 13.981964832076965, 11.754253707105612, 7.956595300798631, 6.315869587661187, 6.315869587661187, 4.627325608764567, 3.6984694221389542, 3.3568146346643974, 2.7369467418327798, 2.6610014176998558, 2.6585178698196947, 2.5891569919465325, 1.779536789183193, 1.779536789183193, 1.7071592606801997, 1.6731343036002873, 1.6538171073149979, 1.2119692675107079, 0.8460531825275042 ], "metrics": { "false_recall": true } } } }, { "query_id": "N095", "query": "Create a recurring calendar meeting every Monday", "category": "unsupported", "domain": "calendar", "intent": "capability:calendar-unsupported", "target_capability": "Calendar create explicitly supports non-recurring events only.", "acceptable": [], "rationale": "Calendar create explicitly supports non-recurring events only.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.list", "apple/alarms.create", "apple/calendar.events.delete", "apple/calendar.list", "apple/contacts.create", "apple/reminders.create", "builtin/ScheduledTasks", "apple/links.open", "builtin/Memory", "apple/health.statistics", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/weather.daily", "builtin/WriteFile" ], "scores": [ 9.5, 5, 5, 4.5, 4, 4, 4, 4, 4, 3.5, 2, 1.5, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/calendar.list", "apple/calendar.events.list", "apple/reminders.create", "apple/contacts.create", "builtin/WriteFile", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/alarms.create", "apple/reminders.update", "builtin/Memory", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 7.710025163189024, 5.751898071884639, 4.7136616165818115, 4.70849125376238, 3.615321155408533, 3.353990223330057, 3.3063649324302986, 3.05666003761185, 3.030740607742593, 2.8131576240574634, 2.776792493451411, 2.769559846226892, 2.5703833063631247, 2.5006287346568747, 2.430546544538726, 2.1142385633003573, 1.4761555867694751, 1.3457586502029633 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.update", "apple/calendar.events.list", "apple/alarms.create", "apple/calendar.list", "apple/calendar.events.delete", "builtin/ScheduledTasks", "builtin/Memory", "builtin/WriteFile", "apple/reminders.complete", "apple/photos.get", "apple/reminders.delete", "apple/reminders.update", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 17.869349252299724, 9.571512545226692, 8.542801847641005, 8.510275502348785, 8.376948352980492, 8.29960284151429, 7.914298923247929, 7.725956296832728, 7.370475874277819, 5.15046820091138, 3.0418207860050117, 2.8774990349848864, 2.829013416224716, 2.7519669803733415, 2.609190227566039, 2.456488742268665, 1.290252793313428, 1.1713329690232368, 0.6658726147629157 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.get", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.list", "apple/calendar.events.update", "apple/alarms.create", "builtin/ScheduledTasks", "apple/calendar.list", "apple/calendar.events.delete", "builtin/WriteFile", "builtin/Memory", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/weather.daily", "apple/health.statistics", "apple/links.open" ], "scores": [ 17.75972376755636, 9.401130419780229, 8.542801847641005, 8.510275502348785, 8.29960284151429, 8.225798985347858, 7.914298923247929, 7.739625192857913, 7.725956296832728, 7.195255527716608, 5.715162977290321, 3.0418207860050117, 2.7519669803733415, 2.6528894415699615, 2.4467516364749704, 2.30355678425753, 1.290252793313428, 1.1713329690232368, 1.142400969046904 ], "metrics": { "false_recall": true } } } }, { "query_id": "N096", "query": "Read the full content of my Apple Notes documents", "category": "unsupported", "domain": "notes", "intent": "capability:notes-unsupported", "target_capability": "links.open handoff cannot read Notes content; no Notes read tool.", "acceptable": [], "rationale": "links.open handoff cannot read Notes content; no Notes read tool.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/clipboard.read", "apple/alarms.list", "apple/contacts.get", "apple/music.playback", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/calendar.events.update", "apple/clipboard.write", "apple/reminders.update", "apple/alarms.cancel", "apple/alarms.create", "apple/battery.get", "apple/calendar.events.list", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.eta", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.get", "apple/photos.search", "apple/reminders.list", "builtin/ReadFile", "apple/calendar.events.create", "apple/reminders.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.list", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.list", "apple/contacts.update", "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "builtin/ScheduledTasks", "builtin/Memory", "builtin/EditFile", "builtin/WriteFile" ], "scores": [ 10, 7, 5, 5, 5, 5, 5, 5, 5, 4.5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3.5, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2.5, 2, 1, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/contacts.get", "apple/shortcuts.run", "apple/links.open", "apple/music.playback", "apple/weather.current", "apple/clipboard.write", "apple/reminders.update", "apple/alarms.list", "apple/reminders.list", "apple/music.status", "builtin/ScheduledTasks", "apple/weather.hourly", "apple/calendar.events.update", "apple/maps.show", "apple/weather.daily", "apple/alarms.create", "builtin/ReadFile", "builtin/Memory", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/music.search", "builtin/EditFile", "apple/clipboard.read", "apple/health.workouts.get", "apple/health.workouts.list", "apple/calendar.events.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "apple/photos.get", "apple/photos.search", "apple/health.statistics" ], "scores": [ 4.729474264823359, 3.4517436328367657, 3.442277567091643, 3.2705091014017484, 3.196546683643658, 3.0869611235447083, 3.048112898315442, 2.459638929432902, 2.448649573649498, 2.38399622539939, 2.1887874416937976, 2.179936522218746, 2.1038511081773286, 2.101280785461998, 2.026318787046823, 2.01195188835184, 1.7356930089104854, 1.6208496848238496, 1.5045297512928304, 1.488613054842949, 1.41382735889783, 1.41382735889783, 1.3632427748418257, 1.3439750578113865, 1.2665935090803384, 1.2149135706002092, 1.0956951902365162, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 0.8975292682836623, 0.7350474507853814, 0.717339055840276 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/links.open", "apple/contacts.get", "apple/clipboard.read", "builtin/ReadFile", "apple/clipboard.write", "builtin/Memory", "apple/reminders.update", "apple/calendar.events.update", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "apple/reminders.list", "apple/music.playback", "builtin/ScheduledTasks", "apple/weather.current", "apple/health.workouts.get", "apple/health.workouts.list", "builtin/WriteFile", "apple/alarms.list", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/calendar.events.create", "apple/calendar.events.list", "apple/health.sleep.query", "apple/weather.hourly", "apple/weather.daily", "apple/photos.get", "builtin/EditFile", "apple/reminders.create", "apple/photos.search", "apple/health.statistics", "apple/maps.show", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/music.search", "apple/calendar.list", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.list", "apple/contacts.update", "apple/location.geocode", "apple/location.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/location.reverseGeocode", "apple/photos.albums.list" ], "scores": [ 6.899727192226153, 4.646506570007874, 4.5614996595341974, 4.557289964100901, 4.49878769001196, 3.8977289375599202, 3.80684568833329, 3.19125514778517, 2.8891502849647948, 2.6835360690060837, 2.497591979738032, 2.451233465604745, 2.340145417512436, 2.267023258912502, 1.9399169928541506, 1.7522523004403863, 1.6941561812101567, 1.6690871113261696, 1.6123492540611417, 1.5918315529410494, 1.5779828994156802, 1.5779828994156802, 1.5641726935513327, 1.561850448686806, 1.547261791678937, 1.4801456601469885, 1.4082790187257759, 1.3963295577940573, 1.3906556055226948, 1.3503569460646248, 1.2281341512218176, 1.2100028175131061, 0.7245124831044734, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6525386954849385, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "apple/links.open", "apple/contacts.get", "builtin/EditFile", "apple/clipboard.write", "apple/reminders.update", "apple/clipboard.read", "apple/reminders.list", "builtin/Memory", "apple/calendar.events.update", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "apple/alarms.list", "apple/device.get", "apple/battery.get", "apple/health.workouts.get", "apple/music.playback", "apple/health.workouts.list", "apple/photos.get", "apple/calendar.events.list", "builtin/ScheduledTasks", "apple/weather.current", "apple/calendar.events.create", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/calendar.events.get", "apple/calendar.list", "apple/photos.albums.list", "apple/reminders.create", "apple/maps.eta", "apple/weather.hourly", "apple/health.sleep.query", "apple/weather.daily", "apple/photos.search", "apple/health.statistics", "apple/maps.show", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/music.search", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.update", "apple/location.geocode", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.lists", "apple/calendar.events.delete", "apple/location.reverseGeocode" ], "scores": [ 12.510397285717172, 8.725070087423143, 8.50180892688808, 5.179900626029875, 4.791872780727982, 4.181063423600531, 3.80684568833329, 3.3762333310339603, 3.3327162267237176, 3.331714715885159, 3.19125514778517, 2.8891502849647948, 2.6835360690060837, 2.497591979738032, 2.3113487978366307, 2.3070898615156334, 2.2972894589983346, 2.2745034493078133, 2.203172972596244, 2.1704114649269393, 2.140089161738028, 2.1397600402243353, 1.9311291235927746, 1.5799603555018502, 1.5641726935513327, 1.4835039581981464, 1.457827803244451, 1.4334670795370892, 1.4247227638014857, 1.410323224313016, 1.3746858851404284, 1.3503569460646248, 1.2648609067975272, 1.2356719703878243, 1.2259372489789688, 1.1818564143530654, 1.0172802632214495, 1.0044490974914868, 0.7245124831044734, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6525386954849385, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } } } }, { "query_id": "N097", "query": "Send an SMS message to a phone number", "category": "unsupported", "domain": "messages", "intent": "capability:messages-unsupported", "target_capability": "No SMS-send capability; opening an app does not send a message.", "acceptable": [], "rationale": "No SMS-send capability; opening an app does not send a message.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/contacts.create", "apple/contacts.list", "apple/links.open", "apple/shortcuts.run", "builtin/EditFile", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile" ], "scores": [ 2, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.update" ], "scores": [ 3.0195051802203676, 2.9236734812969383 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/contacts.update", "apple/contacts.create", "apple/contacts.list", "apple/shortcuts.run", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "apple/links.open" ], "scores": [ 5.249182410278352, 3.575466308364488, 1.9444089450290927, 1.4709676577169795, 1.0922001006905449, 0.9881630284044697, 0.9518941151163509, 0.9022222210637898, 0.290204573051431 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/contacts.update", "apple/contacts.create", "apple/contacts.list", "apple/shortcuts.run", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile", "apple/links.open" ], "scores": [ 5.249182410278352, 3.575466308364488, 1.9444089450290927, 1.4709676577169795, 1.0922001006905449, 0.9881630284044697, 0.9518941151163509, 0.9022222210637898, 0.290204573051431 ], "metrics": { "false_recall": true } } } }, { "query_id": "N098", "query": "Transcribe an audio recording into text", "category": "unsupported", "domain": "audio", "intent": "capability:audio-unsupported", "target_capability": "No transcription tool in frozen catalog.", "acceptable": [], "rationale": "No transcription tool in frozen catalog.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/clipboard.write", "apple/shortcuts.run", "builtin/EditFile", "apple/clipboard.read", "apple/contacts.update", "builtin/AskUserQuestions", "builtin/ReadFile", "builtin/Memory" ], "scores": [ 1.5, 1.5, 1.5, 1, 1, 1, 1, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "builtin/ReadFile", "builtin/EditFile", "builtin/AskUserQuestions", "apple/contacts.update", "apple/shortcuts.run" ], "scores": [ 3.147473456270661, 3.0929979541428994, 2.8581499673178334, 2.7272927852161843, 2.165477147820978, 1.8905693842378057, 1.7777279238039982 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/clipboard.write", "builtin/EditFile", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ReadFile", "builtin/AskUserQuestions", "apple/contacts.update", "builtin/Memory" ], "scores": [ 4.561302494397273, 4.1306699092355466, 2.972996695597959, 2.89297795482389, 2.745409276253241, 2.062180702926565, 1.762134739525646, 1.3563273996917304 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/ReadFile", "apple/clipboard.write", "apple/shortcuts.run", "apple/clipboard.read", "builtin/WriteFile", "builtin/AskUserQuestions", "apple/contacts.update", "builtin/Memory" ], "scores": [ 6.323379252210103, 5.019844432888661, 4.2922006742543, 2.797599684974972, 2.72230178628807, 2.5040111006170407, 1.9405188352247646, 1.6581745951752167, 1.276308551475644 ], "metrics": { "false_recall": true } } } }, { "query_id": "N099", "query": "Tap arbitrary buttons in another app by screenshot coordinates", "category": "unsupported", "domain": "screen", "intent": "capability:screen-unsupported", "target_capability": "Optional phone/screen UI tools are excluded.", "acceptable": [], "rationale": "Optional phone/screen UI tools are excluded.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/alarms.create", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/music.status", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/ScheduledTasks" ], "scores": [ 5.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/links.open", "apple/location.reverseGeocode", "apple/music.status", "apple/maps.eta", "apple/shortcuts.run", "apple/alarms.create", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks" ], "scores": [ 6.619382422802759, 2.430546544538726, 2.38399622539939, 2.1395511304507666, 2.0519806739118946, 2.01195188835184, 1.9107839181010244, 1.726212147301921, 1.6934955774573934, 1.5741568020017251, 1.1554436837598263 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/links.open", "apple/location.reverseGeocode", "apple/music.status", "apple/maps.eta", "apple/shortcuts.run", "apple/alarms.create", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks" ], "scores": [ 8.299730292748695, 2.49296358785834, 2.382325764389142, 2.142951037296516, 2.0325256171929906, 1.9907674591623792, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.5495958966369607, 1.117963704064442 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/links.open", "apple/location.reverseGeocode", "apple/music.status", "apple/maps.eta", "apple/shortcuts.run", "apple/alarms.create", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks" ], "scores": [ 8.960333029776292, 2.49296358785834, 2.382325764389142, 2.142951037296516, 2.0325256171929906, 1.9907674591623792, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.5495958966369607, 1.117963704064442 ], "metrics": { "false_recall": true } } } }, { "query_id": "N100", "query": "Fetch the complete HTML content of an arbitrary website", "category": "unsupported", "domain": "web", "intent": "capability:web-unsupported", "target_capability": "Web tools disabled; links.open only hands off URL, not content retrieval.", "acceptable": [], "rationale": "Web tools disabled; links.open only hands off URL, not content retrieval.", "basis": [ "catalog.json: complete declared capability/schema inventory; Sandbox/Web/phone disabled" ], "known_diagnostic": false, "split": "dev", "methods": { "keyword": { "ranking": [ "apple/reminders.complete", "apple/clipboard.write", "apple/music.playback", "builtin/ScheduledTasks", "builtin/WriteFile", "builtin/Memory" ], "scores": [ 3, 2.5, 1, 1, 1, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/clipboard.write", "apple/reminders.complete", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 5.839841723308905, 3.819471001206962, 2.167107836671426, 1.3804900280474186 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/reminders.complete", "apple/clipboard.write", "builtin/WriteFile", "apple/music.playback", "builtin/ScheduledTasks", "builtin/Memory" ], "scores": [ 7.7622905376693065, 6.488237282381124, 2.8628436088662577, 1.720954340406063, 1.214637928883098, 1.1941549476394007 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/reminders.complete", "apple/clipboard.write", "builtin/WriteFile", "builtin/ReadFile", "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/health.workouts.get", "apple/music.playback", "builtin/ScheduledTasks", "builtin/Memory" ], "scores": [ 12.178817979410681, 6.170513015969694, 6.00543393754603, 4.500884147785318, 2.7369467418327798, 2.7369467418327798, 2.6610014176998558, 2.6610014176998558, 2.5891569919465325, 2.5891569919465325, 2.521090036132972, 2.3951563797541615, 1.5839818954898712, 1.214637928883098, 1.0991109834001507 ], "metrics": { "false_recall": true } } } }, { "query_id": "H001", "query": "Which calendars are accessible, and what are their identifiers?", "cohort": "intent_transfer", "domain": "calendar", "category": "heldout_intent", "intent": "capability:apple/calendar.list", "acceptable": [ "apple/calendar.list" ], "rationale": "Enumerates accessible calendars, not events.", "basis": [ "apple/calendar.list" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/calendar.list", "apple/calendar.events.create", "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/contacts.get", "apple/contacts.list", "apple/links.open", "apple/contacts.delete", "apple/contacts.update", "apple/photos.search", "apple/device.get", "apple/health.statistics", "apple/photos.albums.list", "apple/photos.get", "apple/reminders.get", "apple/reminders.lists", "apple/weather.daily", "builtin/AskUserQuestions", "apple/location.geocode", "apple/location.reverseGeocode", "builtin/EditFile", "builtin/ListFiles", "builtin/ReadFile", "builtin/ScheduledTasks", "builtin/WriteFile" ], "scores": [ 7, 4.5, 4.5, 4, 4, 3, 2.5, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.list", "apple/contacts.get", "apple/contacts.list", "builtin/AskUserQuestions", "apple/calendar.events.list", "apple/calendar.events.get", "apple/calendar.events.create", "apple/device.get", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.update", "apple/contacts.delete", "apple/calendar.events.delete", "apple/contacts.update", "apple/photos.get", "apple/weather.daily", "apple/photos.albums.list", "apple/health.statistics", "apple/photos.search" ], "scores": [ 14.261497917245816, 5.2136995342845385, 4.074407504940825, 3.348804949568758, 3.3063649324302986, 3.105383741638209, 3.0094803210198413, 2.7156932419630486, 2.5424564403573893, 2.458443360385633, 2.2811099463360556, 2.0237659864584026, 1.9146581234509097, 1.5741603564255942, 1.4950768918533153, 1.4761555867694751, 1.3968322222103842, 1.3457586502029633, 1.224419633898322 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.list", "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.update", "apple/contacts.get", "apple/calendar.events.delete", "builtin/AskUserQuestions", "apple/contacts.list", "apple/contacts.delete", "apple/contacts.update", "apple/reminders.get", "apple/reminders.lists", "apple/device.get", "apple/photos.search", "apple/weather.daily", "apple/photos.get", "apple/health.statistics", "apple/location.reverseGeocode", "apple/photos.albums.list", "apple/location.geocode", "builtin/ScheduledTasks", "builtin/ListFiles", "apple/links.open", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile" ], "scores": [ 17.476618937677735, 8.29960284151429, 8.117821392761574, 6.834728301526837, 5.949092692075655, 5.444248081444347, 4.55597709755589, 3.400782226857424, 3.13017557546714, 3.021484378021671, 2.155572323980997, 2.115197914250606, 2.0387645115018342, 2.0355194485073462, 1.4574864398447889, 1.290252793313428, 1.196009406912247, 1.1713329690232368, 1.144724074947355, 1.1244973289099645, 1.1184030782407894, 0.8530696926694925, 0.7340570886741702, 0.6658726147629157, 0.6641347819940883, 0.6397587973363139, 0.6063747993728439 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.list", "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.get", "apple/calendar.events.update", "apple/contacts.get", "apple/calendar.events.delete", "builtin/AskUserQuestions", "apple/contacts.list", "apple/contacts.delete", "apple/contacts.update", "apple/reminders.get", "apple/reminders.lists", "apple/device.get", "apple/photos.search", "apple/weather.daily", "apple/photos.get", "apple/health.statistics", "apple/location.reverseGeocode", "apple/links.open", "apple/photos.albums.list", "apple/location.geocode", "builtin/ScheduledTasks", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile" ], "scores": [ 17.476618937677735, 8.29960284151429, 8.117821392761574, 6.834728301526837, 5.949092692075655, 5.444248081444347, 4.55597709755589, 3.400782226857424, 3.13017557546714, 3.021484378021671, 2.155572323980997, 2.115197914250606, 2.0387645115018342, 2.0355194485073462, 1.4574864398447889, 1.290252793313428, 1.196009406912247, 1.1713329690232368, 1.144724074947355, 1.142400969046904, 1.1244973289099645, 1.1184030782407894, 0.8530696926694925, 0.7340570886741702, 0.6641347819940883, 0.6397587973363139, 0.6063747993728439 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H002", "query": "I have an eventId already; fetch that single event.", "cohort": "intent_transfer", "domain": "calendar", "category": "heldout_intent", "intent": "capability:apple/calendar.events.get", "acceptable": [ "apple/calendar.events.get" ], "rationale": "Get by existing event ID.", "basis": [ "apple/calendar.events.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.list", "apple/calendar.events.get", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/music.playback", "apple/photos.get", "apple/photos.search", "apple/reminders.create", "apple/reminders.get", "apple/reminders.list", "apple/alarms.create", "apple/alarms.list", "apple/clipboard.read", "apple/links.open", "apple/music.search", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "builtin/AskUserQuestions" ], "scores": [ 6, 6, 6, 5.5, 5, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "name_description": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.get", "apple/reminders.get", "builtin/ScheduledTasks", "apple/photos.get", "apple/reminders.create", "apple/music.playback", "apple/reminders.list", "apple/clipboard.read", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/alarms.create", "apple/photos.search", "apple/links.open" ], "scores": [ 5.477903322321855, 5.317831687177921, 4.7836961412751045, 4.459992518798011, 4.371701721545876, 2.6944644547345806, 2.3239109147965014, 2.253912805996125, 2.0606820288717214, 1.8781025023437183, 1.7229585090753954, 1.7043370142720697, 1.6861137326635875, 1.6682760259384652, 1.584464330735021, 1.5381011872642523, 1.297625056165918, 1.0010725906208502 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "bm25": { "ranking": [ "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.update", "apple/calendar.events.list", "apple/calendar.events.create", "builtin/ScheduledTasks", "apple/reminders.get", "apple/alarms.cancel", "apple/photos.get", "apple/photos.search", "apple/music.playback", "apple/reminders.create", "apple/clipboard.read", "apple/alarms.list", "apple/music.search", "apple/alarms.create", "apple/reminders.list", "apple/links.open", "apple/reminders.delete", "apple/reminders.complete", "apple/contacts.delete", "apple/contacts.get", "apple/health.workouts.get", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update" ], "scores": [ 12.49262493371049, 12.341915493427718, 10.914330493458985, 10.591484869988118, 9.916886689212527, 2.7118797719728223, 2.3660103307504636, 2.354079786875306, 2.0189831893083445, 1.7936359188262194, 1.7244033122771167, 1.7020131435723234, 1.6961419113447813, 1.6769433783228374, 1.570298655749525, 1.5219060705338434, 1.3088266447267578, 0.9724723892857147, 0.8674373781600091, 0.8524674830238934, 0.6959051917000892, 0.6959051917000892, 0.6768345461876305, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.list", "apple/calendar.events.create", "apple/reminders.get", "apple/photos.get", "apple/contacts.get", "apple/health.workouts.get", "apple/battery.get", "apple/device.get", "builtin/ScheduledTasks", "apple/location.get", "apple/alarms.cancel", "apple/photos.search", "apple/music.playback", "apple/reminders.create", "apple/clipboard.read", "apple/alarms.list", "apple/music.search", "apple/alarms.create", "apple/reminders.list", "apple/links.open", "apple/reminders.delete", "apple/reminders.complete", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update" ], "scores": [ 14.931072485374251, 12.49262493371049, 10.914330493458985, 10.591484869988118, 9.916886689212527, 4.887100366883436, 4.6799846070082, 3.356906609399945, 3.071990925941792, 2.7369467418327798, 2.7369467418327798, 2.7118797719728223, 2.5891569919465325, 2.354079786875306, 1.7936359188262194, 1.7244033122771167, 1.7020131435723234, 1.6961419113447813, 1.6769433783228374, 1.570298655749525, 1.5219060705338434, 1.3088266447267578, 0.9724723892857147, 0.8674373781600091, 0.8524674830238934, 0.6959051917000892, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H003", "query": "Look up the recorded workout with this UUID.", "cohort": "intent_transfer", "domain": "health", "category": "heldout_intent", "intent": "capability:apple/health.workouts.get", "acceptable": [ "apple/health.workouts.get" ], "rationale": "One recorded workout by UUID.", "basis": [ "apple/health.workouts.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.cancel", "apple/alarms.list", "apple/health.statistics", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.route", "apple/maps.search", "builtin/ScheduledTasks" ], "scores": [ 7, 5, 1.5, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/alarms.list", "apple/alarms.cancel", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/health.statistics" ], "scores": [ 12.395787198157493, 8.610169054383494, 2.8055106012874256, 2.6351421055956448, 2.607264453484727, 2.5938411871930867, 2.2055634157869286, 2.072598362607447, 1.656498350646591 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.cancel", "apple/location.reverseGeocode", "apple/alarms.list", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/health.statistics", "builtin/ScheduledTasks" ], "scores": [ 23.124500776633333, 14.839281442889394, 4.607001718036044, 2.877556815390035, 2.620810221347025, 2.606627671177044, 2.1935679034724593, 2.054068595663089, 1.6237979510580696, 1.1139812831763996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.cancel", "apple/location.reverseGeocode", "apple/alarms.list", "apple/location.geocode", "apple/maps.route", "apple/maps.search", "apple/health.statistics", "builtin/ScheduledTasks" ], "scores": [ 23.124500776633333, 14.839281442889394, 4.607001718036044, 2.877556815390035, 2.620810221347025, 2.606627671177044, 2.1935679034724593, 2.054068595663089, 1.6237979510580696, 1.1139812831763996 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H004", "query": "列出我可用的提醒事项清单，并告诉我默认清单。", "cohort": "intent_transfer", "domain": "reminders", "category": "heldout_intent", "intent": "capability:apple/reminders.lists", "acceptable": [ "apple/reminders.lists" ], "rationale": "List reminder containers and default, not reminder items.", "basis": [ "apple/reminders.lists" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/reminders.list", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/reminders.update", "apple/links.open", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.list", "apple/health.workouts.list", "apple/photos.albums.list" ], "scores": [ 6, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/reminders.list", "apple/reminders.complete", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.create", "apple/reminders.get", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.list", "apple/links.open" ], "scores": [ 10.355577512497662, 9.815195099979737, 9.815195099979737, 8.785658142322799, 8.785658142322799, 8.349616533617082, 8.349616533617082, 3.2646010288734075, 2.8533585020144336, 2.8533585020144336, 2.684288301093061, 2.684288301093061, 2.2791487414708236, 2.1253334848001453 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.list", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.get", "apple/reminders.update", "apple/links.open", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.list", "builtin/ListFiles" ], "scores": [ 13.970681728538613, 11.322515292618643, 9.286893001482053, 7.8363460079639236, 6.417620768581872, 6.276627057843893, 5.770999050379919, 4.106473478734701, 3.6565709667499404, 3.5222678853118445, 3.5222678853118445, 3.458749265208039, 3.458749265208039, 3.2812334478708602, 2.7369467418327798 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H005", "query": "Find my incomplete reminders whose titles contain dentist.", "cohort": "intent_transfer", "domain": "reminders", "category": "heldout_intent", "intent": "capability:apple/reminders.list", "acceptable": [ "apple/reminders.list" ], "rationale": "Title substring search among reminders.", "basis": [ "apple/reminders.list" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/reminders.list", "apple/reminders.update", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/links.open", "builtin/ScheduledTasks", "apple/alarms.create", "apple/alarms.list", "apple/music.search", "apple/calendar.events.create", "apple/calendar.events.update" ], "scores": [ 6, 5, 4.5, 4.5, 4.5, 4.5, 4, 2, 2, 1.5, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/reminders.list", "apple/reminders.get", "apple/reminders.lists", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.create", "apple/alarms.list", "apple/music.search", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 8.880321459417356, 3.612813620113822, 3.5412226376495974, 3.4062280126487074, 3.2512998801244857, 3.1371483815404533, 3.05666003761185, 2.2055634157869286, 2.072598362607447, 2.01195188835184, 1.1554436837598263 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/reminders.list", "apple/reminders.get", "apple/reminders.update", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.lists", "apple/alarms.create", "builtin/ScheduledTasks", "apple/alarms.list", "apple/music.search", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open" ], "scores": [ 13.062266605687421, 9.605636229394815, 9.451483269168433, 9.285244357925471, 9.130891830020635, 8.61598850828072, 8.138621340785264, 2.8496439328389913, 2.1947612418795974, 1.6769433783228374, 1.570298655749525, 0.9848947523202506, 0.9442576845200903, 0.70568370941594 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.list", "apple/reminders.get", "apple/reminders.update", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.lists", "apple/music.search", "apple/alarms.list", "apple/alarms.create", "builtin/ScheduledTasks", "apple/photos.search", "apple/battery.get", "apple/device.get", "apple/maps.search", "apple/contacts.get", "apple/photos.get", "apple/calendar.events.get", "apple/location.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.get", "apple/health.workouts.list", "apple/links.open", "apple/calendar.events.create", "apple/calendar.events.update" ], "scores": [ 14.435102135758815, 11.08674414579347, 9.451483269168433, 9.285244357925471, 9.130891830020635, 8.61598850828072, 8.138621340785264, 3.2755561436956095, 3.198039754338451, 2.8496439328389913, 2.1947612418795974, 1.7052574879460844, 1.607920950061708, 1.607920950061708, 1.607920950061708, 1.5633040505560998, 1.5633040505560998, 1.5210963760156138, 1.5210963760156138, 1.4431681307115642, 1.4431681307115642, 1.407123515710695, 1.407123515710695, 1.407123515710695, 1.30903966193533, 1.2107026713576772, 0.9848947523202506, 0.9442576845200903 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H006", "query": "按已知reminderId读取这一条提醒的详情。", "cohort": "intent_transfer", "domain": "reminders", "category": "heldout_intent", "intent": "capability:apple/reminders.get", "acceptable": [ "apple/reminders.get" ], "rationale": "One reminder by identifier.", "basis": [ "apple/reminders.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.lists", "apple/links.open", "apple/photos.get", "apple/calendar.events.get", "apple/contacts.get", "apple/health.workouts.get", "builtin/ScheduledTasks", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/device.get", "apple/location.get", "apple/music.playback", "apple/alarms.cancel", "apple/calendar.events.list", "apple/contacts.delete", "apple/contacts.update", "apple/photos.search", "builtin/AskUserQuestions" ], "scores": [ 9, 7, 7, 6.5, 6.5, 6.5, 5.5, 4, 3, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/reminders.get", "apple/reminders.create", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "apple/photos.get", "apple/music.playback", "apple/calendar.events.delete", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 6.307278074848403, 5.117342066483571, 4.8339042269754104, 3.5412226376495974, 3.4062280126487074, 3.2512998801244857, 3.1371483815404533, 2.253912805996125, 1.8781025023437183, 1.8016996019310052, 1.5532511243390867, 1.4803458231303015, 1.4405949110207767 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/reminders.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.update", "apple/reminders.list", "apple/reminders.lists", "apple/photos.get", "apple/contacts.get", "apple/calendar.events.get", "apple/battery.get", "apple/device.get", "apple/health.workouts.get", "apple/location.get", "apple/links.open", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.playback", "apple/calendar.events.create", "apple/calendar.events.update", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 17.78712991856387, 14.052171031348186, 13.582524669374177, 12.679467398770482, 12.539538296829436, 12.459653037819859, 12.053080531184083, 4.874560969159303, 3.551482971551047, 3.4604715978861886, 3.061358970857833, 3.061358970857833, 2.9273134386129787, 2.6757188918211545, 2.0810902666793925, 1.8960280939425802, 1.793265803487735, 1.7244033122771167, 1.3699185217230823, 1.3062125395735957, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.lists", "apple/reminders.list", "apple/reminders.update", "apple/photos.get", "apple/calendar.events.get", "apple/contacts.get", "apple/health.workouts.get", "apple/battery.get", "apple/device.get", "apple/location.get", "apple/links.open", "builtin/ReadFile", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/music.playback", "apple/calendar.events.create", "apple/calendar.events.update", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/photos.search", "apple/contacts.update", "apple/calendar.events.list" ], "scores": [ 18.936283040303643, 14.452345933873174, 13.956768525254338, 12.896922367348463, 12.765513907708382, 12.55180947308295, 12.293688367500437, 5.526702693325572, 4.225598032201792, 4.203624695717317, 3.931528999457869, 3.5772446563552007, 3.5772446563552007, 3.4408453261367575, 3.425030012588448, 2.4364052592946104, 1.8960280939425802, 1.793265803487735, 1.7244033122771167, 1.3699185217230823, 1.3062125395735957, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.5202630923778531, 0.4532555669664913, 0.37712574519671693 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H007", "query": "Erase this existing contact from the address book.", "cohort": "intent_transfer", "domain": "contacts", "category": "heldout_intent", "intent": "capability:apple/contacts.delete", "acceptable": [ "apple/contacts.delete" ], "rationale": "Deletion capability, subject to approval; not update.", "basis": [ "apple/contacts.delete" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/contacts.list", "apple/contacts.get", "apple/contacts.update", "apple/contacts.create", "apple/contacts.delete", "apple/location.geocode", "builtin/WriteFile", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.write", "apple/maps.search", "builtin/ScheduledTasks" ], "scores": [ 6, 5.5, 5, 4.5, 4.5, 1.5, 1.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.get", "builtin/WriteFile", "apple/contacts.create", "apple/contacts.delete", "apple/location.geocode", "apple/contacts.update" ], "scores": [ 9.799217513236611, 7.5079945717602055, 5.065193500893222, 3.908385007994482, 3.6211209622787783, 3.0990449327715073, 3.057540300459825 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "bm25": { "ranking": [ "apple/contacts.list", "apple/contacts.get", "apple/contacts.update", "apple/contacts.create", "apple/contacts.delete", "builtin/WriteFile", "apple/location.geocode", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.search", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 15.107524035650837, 14.674396084996033, 11.494350678543967, 11.41435524303158, 11.146694165037609, 4.802385981774975, 4.078889372840208, 1.7350573342035134, 1.4956890564914902, 1.0790560593678742, 0.8851282095813073, 0.8652613463947955 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } }, "combined": { "ranking": [ "apple/contacts.delete", "apple/contacts.list", "apple/contacts.get", "apple/contacts.update", "apple/contacts.create", "apple/calendar.events.delete", "builtin/WriteFile", "apple/reminders.delete", "apple/location.geocode", "apple/clipboard.write", "apple/maps.search", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 15.841765214689598, 15.107524035650837, 14.674396084996033, 11.494350678543967, 11.41435524303158, 6.27886177557786, 4.802385981774975, 4.268742291627459, 4.078889372840208, 1.4956890564914902, 1.0790560593678742, 0.8851282095813073, 0.8652613463947955 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H008", "query": "获取我当前的前台位置，经纬度即可。", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.get", "acceptable": [ "apple/location.get" ], "rationale": "Foreground current location acquisition.", "basis": [ "apple/location.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode" ], "scores": [ 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/location.get" ], "scores": [ 4.851812771864217, 4.851812771864217, 3.9350368504723905 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/location.get", "apple/location.geocode", "apple/location.reverseGeocode", "builtin/GetCurrentTime", "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/photos.get", "apple/calendar.events.get", "apple/reminders.get", "apple/health.workouts.get" ], "scores": [ 6.39689187998381, 5.866964571583784, 5.866964571583784, 5.255138622230446, 2.7369467418327798, 2.7369467418327798, 2.6610014176998558, 2.6610014176998558, 2.5891569919465325, 2.521090036132972, 2.3951563797541615 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H009", "query": "Turn an explicitly supplied street address into coordinates.", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.geocode", "acceptable": [ "apple/location.geocode" ], "rationale": "Forward geocoding of supplied address.", "basis": [ "apple/location.geocode" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/location.geocode", "apple/calendar.events.update", "apple/contacts.get", "apple/contacts.list", "apple/links.open", "apple/location.reverseGeocode", "apple/maps.eta", "apple/maps.show", "apple/photos.search", "apple/reminders.list", "apple/reminders.update", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/Memory", "builtin/ScheduledTasks", "builtin/WriteFile", "apple/contacts.create", "apple/maps.search" ], "scores": [ 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/WriteFile", "builtin/Memory", "apple/contacts.get", "apple/location.geocode", "apple/reminders.update", "apple/contacts.list", "apple/location.reverseGeocode", "apple/reminders.list", "apple/contacts.update", "apple/maps.eta", "apple/calendar.events.update", "apple/maps.show", "apple/weather.current", "apple/photos.search", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks", "apple/links.open" ], "scores": [ 4.350411263790229, 3.8067640926287614, 3.4456687975865097, 3.0990449327715073, 2.8055106012874256, 2.6927249482077116, 2.430546544538726, 2.253759150951459, 2.182230355500232, 2.1395511304507666, 1.9364035337351766, 1.9107839181010244, 1.726212147301921, 1.6973910453637415, 1.6934955774573934, 1.5741568020017251, 1.5480217516112227, 1.1344628597433402 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "bm25": { "ranking": [ "builtin/WriteFile", "apple/location.geocode", "builtin/Memory", "apple/contacts.update", "apple/reminders.update", "apple/contacts.get", "apple/location.reverseGeocode", "apple/reminders.list", "apple/maps.eta", "apple/contacts.list", "apple/calendar.events.update", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/photos.search", "apple/weather.daily", "apple/contacts.create", "builtin/ScheduledTasks", "apple/links.open", "apple/maps.search" ], "scores": [ 4.449912122339027, 4.078889372840208, 3.762417568958924, 3.5946697175096762, 2.8354549407264593, 2.7106579929009085, 2.49296358785834, 2.2443758259945152, 2.142951037296516, 2.0828091760528666, 1.9312512392224508, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.6656673071721346, 1.5495958966369607, 1.5233732602840573, 1.4978074273357158, 1.1020517573918405, 1.0790560593678742 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/WriteFile", "apple/location.geocode", "builtin/Memory", "apple/contacts.update", "apple/reminders.update", "apple/contacts.get", "apple/location.reverseGeocode", "apple/reminders.list", "apple/maps.eta", "apple/contacts.list", "apple/calendar.events.update", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/photos.search", "apple/weather.daily", "apple/contacts.create", "builtin/ScheduledTasks", "apple/links.open", "apple/maps.search" ], "scores": [ 4.449912122339027, 4.078889372840208, 3.762417568958924, 3.5946697175096762, 2.8354549407264593, 2.7106579929009085, 2.49296358785834, 2.2443758259945152, 2.142951037296516, 2.0828091760528666, 1.9312512392224508, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.6656673071721346, 1.5495958966369607, 1.5233732602840573, 1.4978074273357158, 1.1020517573918405, 1.0790560593678742 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H010", "query": "把给定的经纬度转换为街道地址候选。", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.reverseGeocode", "acceptable": [ "apple/location.reverseGeocode" ], "rationale": "Reverse geocode explicit coordinates.", "basis": [ "apple/location.reverseGeocode" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode" ], "scores": [ 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/location.get" ], "scores": [ 4.851812771864217, 4.851812771864217, 3.9350368504723905 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/location.get" ], "scores": [ 5.866964571583784, 5.866964571583784, 3.8077348880372774 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H011", "query": "Search Apple Maps for pizza places inside this supplied region.", "cohort": "intent_transfer", "domain": "maps", "category": "heldout_intent", "intent": "capability:apple/maps.search", "acceptable": [ "apple/maps.search" ], "rationale": "Explicit query and optional bounding region.", "basis": [ "apple/maps.search" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/maps.search", "apple/links.open", "apple/music.search", "apple/maps.route", "apple/maps.show", "apple/photos.search", "apple/maps.eta", "apple/shortcuts.run", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/calendar.events.update", "apple/contacts.update", "apple/music.playback", "apple/music.status", "apple/reminders.list", "apple/reminders.update", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/photos.get", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "builtin/ScheduledTasks" ], "scores": [ 14.5, 11.5, 8.5, 8, 8, 7, 6, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/maps.search", "apple/maps.show", "apple/music.search", "apple/links.open", "apple/maps.route", "apple/photos.search", "apple/shortcuts.run", "apple/reminders.update", "apple/maps.eta", "apple/music.status", "apple/reminders.list", "apple/contacts.update", "apple/alarms.create", "apple/weather.current", "apple/calendar.events.update", "apple/alarms.list", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks", "apple/music.playback" ], "scores": [ 13.469465234420337, 6.275929411801544, 5.142144244604115, 4.864990858969687, 4.8512698956231874, 3.5415614207970476, 3.4517436328367657, 2.8055106012874256, 2.683179802495841, 2.38399622539939, 2.253759150951459, 2.182230355500232, 2.01195188835184, 1.9552868225447846, 1.9364035337351766, 1.5045297512928304, 1.488613054842949, 1.3334393574442729, 1.2394733488050822, 1.1554436837598263, 1.1034012647303224 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.search", "apple/music.search", "apple/maps.show", "apple/maps.route", "apple/links.open", "apple/photos.search", "apple/maps.eta", "apple/reminders.update", "apple/music.status", "apple/reminders.list", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/contacts.update", "apple/alarms.create", "apple/calendar.events.update", "apple/photos.get", "apple/weather.current", "apple/alarms.list", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily", "apple/music.playback", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/photos.albums.list" ], "scores": [ 29.30490047848249, 11.523927566605556, 10.639195025696896, 9.328263390443244, 9.042498551699762, 8.80266175955023, 7.136994452378095, 3.342279461302112, 2.8891502849647948, 2.751200346570168, 2.7447224362064535, 2.6835360690060837, 2.6758414458624133, 2.497591979738032, 2.379294565401443, 1.988825859144915, 0.7085420744235738, 0.6624346778200838, 0.6606930487736644, 0.6438261189935433, 0.6337103983466377, 0.6191910771063727, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.search", "apple/maps.show", "apple/links.open", "apple/maps.route", "apple/music.search", "apple/maps.eta", "apple/photos.search", "apple/reminders.list", "apple/reminders.update", "apple/music.status", "apple/shortcuts.run", "apple/contacts.update", "apple/alarms.list", "builtin/ScheduledTasks", "apple/alarms.create", "apple/calendar.events.update", "apple/contacts.list", "apple/calendar.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/photos.get", "apple/weather.current", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily", "apple/music.playback", "apple/battery.get", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/health.sleep.query", "apple/health.workouts.get", "apple/location.reverseGeocode" ], "scores": [ 25.765826428390405, 10.639195025696896, 10.114802143912874, 9.328263390443244, 7.834315474859941, 7.136994452378095, 6.552465814882041, 4.514575685312719, 3.342279461302112, 2.8891502849647948, 2.6835360690060837, 2.6758414458624133, 2.6162476281427733, 2.565086275364292, 2.497591979738032, 2.379294565401443, 2.360540450570281, 2.31424198014557, 2.30175925617362, 2.2554607857489093, 2.129474354681442, 1.4858533205308402, 0.7085420744235738, 0.6606930487736644, 0.6438261189935433, 0.6337103983466377, 0.6191910771063727, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H012", "query": "在Apple地图里展示这一个坐标点，不规划路线。", "cohort": "intent_transfer", "domain": "maps", "category": "heldout_intent", "intent": "capability:apple/maps.show", "acceptable": [ "apple/maps.show" ], "rationale": "Foreground map pin handoff; no navigation promise.", "basis": [ "apple/maps.show" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/maps.eta", "apple/shortcuts.run", "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/music.playback", "apple/music.search", "apple/music.status", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/photos.albums.list", "apple/photos.get", "apple/photos.search", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "builtin/ScheduledTasks" ], "scores": [ 10, 7, 7, 7, 6, 5, 4.5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/music.status", "apple/maps.show", "apple/alarms.create", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/music.search", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks", "apple/music.playback" ], "scores": [ 3.4517436328367657, 3.442277567091643, 2.38399622539939, 2.101280785461998, 2.01195188835184, 1.9552868225447846, 1.5045297512928304, 1.5045297512928304, 1.488613054842949, 1.41382735889783, 1.41382735889783, 1.3334393574442729, 1.2394733488050822, 1.1554436837598263, 1.1034012647303224 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "bm25": { "ranking": [ "apple/maps.show", "apple/maps.route", "apple/maps.eta", "apple/links.open", "apple/maps.search", "apple/location.geocode", "apple/location.reverseGeocode", "apple/location.get", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks", "apple/weather.current", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/weather.hourly", "apple/weather.daily", "apple/music.playback", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/photos.get", "apple/photos.search", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/photos.albums.list" ], "scores": [ 8.586836840718355, 8.524759035433966, 8.369148878189534, 7.777954790297738, 7.117262900949356, 5.3586372924398695, 5.299856098043208, 4.441861371048043, 2.8891502849647948, 2.6835360690060837, 2.497591979738032, 1.117963704064442, 0.7085420744235738, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.6191910771063727, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.show", "apple/maps.route", "apple/maps.eta", "apple/links.open", "apple/maps.search", "apple/location.geocode", "apple/location.reverseGeocode", "apple/location.get", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks", "apple/weather.current", "apple/alarms.list", "apple/alarms.cancel", "apple/music.search", "apple/weather.hourly", "apple/weather.daily", "apple/music.playback", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/photos.get", "apple/photos.search", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/photos.albums.list" ], "scores": [ 14.591914703756935, 10.498939716882496, 10.343329559638065, 9.731828152562404, 7.68169249242564, 6.373789092159437, 6.315007897762776, 4.31455940861293, 2.8891502849647948, 2.6835360690060837, 2.497591979738032, 1.117963704064442, 0.7085420744235738, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.6191910771063727, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H013", "query": "Create one Anan-owned alarm at this future absolute timestamp.", "cohort": "intent_transfer", "domain": "alarms", "category": "heldout_intent", "intent": "capability:apple/alarms.create", "acceptable": [ "apple/alarms.create" ], "rationale": "One future absolute AlarmKit alarm.", "basis": [ "apple/alarms.create" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/alarms.create", "apple/alarms.list", "apple/alarms.cancel", "apple/calendar.events.create", "apple/contacts.create", "apple/reminders.create", "builtin/Memory", "apple/contacts.delete", "apple/music.playback", "builtin/ScheduledTasks", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.update", "apple/health.workouts.get", "apple/location.get", "apple/maps.route", "apple/music.status", "apple/photos.get", "apple/photos.search", "apple/reminders.delete", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/EditFile", "builtin/ListFiles", "builtin/WriteFile", "apple/calendar.events.list", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.list", "apple/links.open", "builtin/ReadFile" ], "scores": [ 14, 8, 7.5, 6.5, 5, 5, 2.5, 2, 2, 2, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/alarms.create", "apple/alarms.list", "apple/alarms.cancel", "apple/calendar.events.create", "apple/reminders.create", "apple/contacts.create", "apple/contacts.delete", "builtin/Memory", "builtin/WriteFile", "apple/music.playback", "builtin/ScheduledTasks", "apple/music.status", "apple/weather.current", "apple/weather.hourly", "apple/clipboard.read", "apple/weather.daily", "apple/photos.search", "builtin/ListFiles", "builtin/EditFile", "apple/contacts.update", "apple/reminders.delete", "apple/health.workouts.get", "apple/calendar.events.update", "apple/location.get", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/photos.get" ], "scores": [ 17.505672818420123, 13.74880695590194, 9.36394559387692, 5.941821909645199, 5.752186920765384, 4.700412973674526, 3.769564601432209, 3.278998653659781, 2.8131576240574634, 2.1733592660455487, 2.0672994037973105, 2.0653687015756934, 1.992517481390494, 1.95475367730173, 1.931435403678754, 1.8170043301691585, 1.6973910453637415, 1.6822078764459198, 1.5024992193814797, 1.4593071471968346, 1.4160558529896354, 1.3959771463946806, 1.340122705366102, 1.1929880582775596, 1.1248369805109821, 1.0997080487563646, 1.0526744179854908, 0.9892126698597037 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/alarms.create", "apple/alarms.list", "apple/alarms.cancel", "apple/calendar.events.create", "apple/reminders.create", "apple/contacts.create", "builtin/Memory", "apple/contacts.delete", "builtin/WriteFile", "builtin/ScheduledTasks", "apple/music.playback", "apple/music.status", "apple/calendar.events.update", "apple/clipboard.read", "builtin/ListFiles", "builtin/EditFile", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.delete", "apple/weather.current", "apple/weather.hourly", "apple/location.get", "apple/weather.daily", "apple/calendar.events.delete", "apple/clipboard.write", "apple/photos.search", "apple/maps.route", "apple/photos.get", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/health.workouts.list", "builtin/ReadFile", "apple/links.open" ], "scores": [ 32.29932092572617, 19.464530737639574, 18.15041256493431, 11.736969685854561, 11.25903231893408, 10.107616076462895, 4.138295567607377, 3.745475352988968, 2.8774990349848864, 2.495617046077265, 2.092350354055724, 2.0639215021837605, 2.0311853777436726, 1.9221483250094473, 1.651417995754508, 1.4598619112844569, 1.3840390692635096, 1.3691132429326038, 1.3691132429326038, 1.2080440814820608, 1.1839668621096326, 1.1390026644524067, 1.0965468745284472, 1.0829958778625002, 1.0444767918636064, 1.0211488049505084, 0.9971873002352808, 0.9337714661875144, 0.8641940862618136, 0.8253300840178237, 0.8161541886022222, 0.7984011807425733, 0.5185160798440496, 0.290204573051431 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/alarms.create", "apple/alarms.list", "apple/alarms.cancel", "apple/calendar.events.create", "apple/reminders.create", "apple/contacts.create", "builtin/WriteFile", "builtin/Memory", "apple/contacts.delete", "builtin/ScheduledTasks", "apple/music.playback", "apple/music.status", "apple/calendar.events.update", "apple/clipboard.read", "builtin/ListFiles", "builtin/EditFile", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.delete", "apple/weather.current", "apple/weather.hourly", "apple/location.get", "apple/weather.daily", "apple/calendar.events.delete", "apple/clipboard.write", "apple/photos.search", "apple/maps.route", "apple/photos.get", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/health.workouts.list", "builtin/ReadFile", "apple/links.open" ], "scores": [ 32.29932092572617, 19.464530737639574, 18.15041256493431, 11.736969685854561, 11.25903231893408, 10.107616076462895, 5.715162977290321, 4.138295567607377, 3.745475352988968, 2.495617046077265, 2.092350354055724, 2.0639215021837605, 2.0311853777436726, 1.9221483250094473, 1.651417995754508, 1.4598619112844569, 1.3840390692635096, 1.3691132429326038, 1.3691132429326038, 1.2080440814820608, 1.1839668621096326, 1.1390026644524067, 1.0965468745284472, 1.0829958778625002, 1.0444767918636064, 1.0211488049505084, 0.9971873002352808, 0.9337714661875144, 0.8641940862618136, 0.8253300840178237, 0.8161541886022222, 0.7984011807425733, 0.5185160798440496, 0.290204573051431 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H014", "query": "List the native alarms owned by Anan, not all Clock app alarms.", "cohort": "intent_transfer", "domain": "alarms", "category": "heldout_intent", "intent": "capability:apple/alarms.list", "acceptable": [ "apple/alarms.list" ], "rationale": "Scope is actual Anan-owned native alarms.", "basis": [ "apple/alarms.list" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/alarms.list", "apple/alarms.create", "apple/alarms.cancel", "apple/photos.albums.list", "apple/contacts.list", "apple/reminders.list", "apple/calendar.events.list", "apple/links.open", "builtin/ScheduledTasks", "apple/calendar.list", "apple/music.status", "apple/reminders.lists", "builtin/ListFiles", "apple/health.workouts.list", "apple/reminders.create", "apple/calendar.events.create", "apple/calendar.events.update", "apple/health.statistics", "apple/music.playback", "apple/music.search", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "builtin/Memory", "apple/calendar.events.delete", "apple/contacts.delete", "apple/health.sleep.query", "apple/maps.search", "apple/battery.get", "apple/calendar.events.get", "apple/clipboard.read", "apple/clipboard.write", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/maps.show", "apple/photos.get", "apple/photos.search", "apple/weather.hourly", "apple/contacts.get", "apple/contacts.update", "apple/health.workouts.get" ], "scores": [ 13, 11.5, 10, 6.5, 5.5, 5, 4.5, 4.5, 4.5, 4, 4, 4, 4, 3.5, 3.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/alarms.create", "apple/alarms.list", "apple/alarms.cancel", "apple/music.status", "apple/reminders.create", "apple/photos.albums.list", "apple/calendar.events.create", "apple/music.playback", "apple/links.open", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/weather.current", "apple/shortcuts.run", "apple/health.statistics", "apple/music.search", "apple/contacts.delete", "apple/contacts.list", "apple/reminders.lists", "builtin/ListFiles", "apple/weather.daily", "apple/calendar.list", "apple/clipboard.read", "apple/weather.hourly", "apple/calendar.events.list", "apple/health.sleep.query", "apple/reminders.list", "apple/calendar.events.get", "apple/health.workouts.list", "apple/clipboard.write", "apple/photos.get", "builtin/Memory", "apple/calendar.events.delete", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/maps.show", "apple/maps.search" ], "scores": [ 16.206642873144315, 15.468902985737248, 14.056917633410503, 7.059062570742747, 5.094014407628341, 4.618189367164939, 4.459266980646826, 4.336534109281434, 4.302112121752881, 4.278915962250155, 3.816932890175719, 3.340167481642842, 3.0313529042301885, 2.9495705863389086, 2.6300590847757603, 2.430546544538726, 2.4222149813116407, 2.1668991632120185, 1.9903435646574756, 1.9890266052597032, 1.970426045916842, 1.931435403678754, 1.9284890935447245, 1.614897263876673, 1.5629367438392368, 1.4924909939710067, 1.457997372644377, 1.3390180568934824, 1.3098389254907072, 1.2162317258779303, 1.164760090359424, 1.1248369805109821, 1.0700151212109088, 1.031117386694429, 1.018772408912019, 1.018772408912019, 0.9098422129056576, 0.8549912766967913 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "apple/alarms.list", "apple/alarms.create", "apple/alarms.cancel", "apple/music.status", "apple/photos.albums.list", "apple/links.open", "apple/reminders.create", "apple/contacts.list", "apple/calendar.events.create", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/music.playback", "builtin/ListFiles", "apple/calendar.list", "apple/calendar.events.list", "apple/weather.current", "apple/contacts.delete", "apple/shortcuts.run", "apple/health.statistics", "apple/health.workouts.list", "apple/music.search", "apple/clipboard.read", "apple/weather.hourly", "builtin/Memory", "apple/weather.daily", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/maps.search", "apple/clipboard.write", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/maps.show", "apple/contacts.get", "apple/health.workouts.get", "apple/photos.search", "apple/contacts.update" ], "scores": [ 23.982887248355464, 23.45666213102067, 23.206337190706805, 6.878952578107307, 6.4689492599974265, 5.675296302601881, 5.106798477210489, 4.59463705106631, 4.547733085603811, 4.144381718209205, 4.1228942449914925, 4.0229331586539505, 4.009263933251316, 3.9991671617531868, 3.8392072769683265, 3.8153957247318555, 3.476083825847414, 3.07067126376517, 3.0574400206235626, 2.7565809355034667, 2.6900964258574644, 2.546263086285599, 2.3942703130079734, 1.9221483250094473, 1.9116662219484515, 1.86959546469196, 1.8264794298097384, 1.6244277159640166, 1.4175588968227548, 1.1253412868673076, 1.0904891394022966, 1.0742789579478196, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.7436087609996902, 0.6009512783548976, 0.5844827580213056, 0.5822345424416788, 0.3914103754917451 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/alarms.list", "apple/alarms.create", "apple/alarms.cancel", "apple/music.status", "apple/photos.albums.list", "apple/links.open", "apple/reminders.create", "builtin/ListFiles", "apple/contacts.list", "apple/calendar.events.create", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/music.playback", "apple/calendar.list", "builtin/GetCurrentTime", "apple/calendar.events.list", "apple/weather.current", "apple/contacts.delete", "apple/shortcuts.run", "apple/health.statistics", "apple/health.workouts.list", "apple/music.search", "apple/clipboard.read", "apple/weather.hourly", "builtin/Memory", "apple/weather.daily", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/maps.search", "apple/clipboard.write", "apple/photos.get", "apple/location.geocode", "apple/location.get", "apple/battery.get", "apple/maps.eta", "apple/maps.show", "apple/contacts.get", "apple/health.workouts.get", "apple/photos.search", "apple/contacts.update" ], "scores": [ 23.745330405853952, 23.241067970017163, 22.971439145705425, 6.878952578107307, 6.4689492599974265, 6.33589903962948, 5.106798477210489, 5.039330650712083, 4.59463705106631, 4.547733085603811, 4.144381718209205, 4.1228942449914925, 4.0229331586539505, 4.009263933251316, 3.9991671617531868, 3.8153957247318555, 3.6602378602162426, 3.476083825847414, 3.07067126376517, 3.0574400206235626, 2.7565809355034667, 2.6900964258574644, 2.546263086285599, 2.3942703130079734, 1.9221483250094473, 1.9116662219484515, 1.86959546469196, 1.8264794298097384, 1.6244277159640166, 1.4175588968227548, 1.1253412868673076, 1.0904891394022966, 1.0742789579478196, 0.9930830738422539, 0.8836339964142832, 0.8493613585822458, 0.8385204047248325, 0.8385204047248325, 0.7436087609996902, 0.6009512783548976, 0.5844827580213056, 0.5822345424416788, 0.3914103754917451 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H015", "query": "取消当前列表里这个UUID对应的Anan闹钟。", "cohort": "intent_transfer", "domain": "alarms", "category": "heldout_intent", "intent": "capability:apple/alarms.cancel", "acceptable": [ "apple/alarms.cancel" ], "rationale": "Cancel one current Anan-owned alarm UUID.", "basis": [ "apple/alarms.cancel" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/alarms.cancel", "apple/alarms.list", "apple/alarms.create", "apple/health.workouts.get", "apple/clipboard.read", "apple/contacts.delete", "apple/music.playback", "apple/music.status" ], "scores": [ 4, 3.5, 2.5, 1.5, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/alarms.list", "apple/alarms.cancel", "apple/health.workouts.get", "apple/contacts.delete", "apple/music.status", "apple/clipboard.read", "apple/alarms.create", "apple/music.playback" ], "scores": [ 4.545926023696669, 4.497833837722533, 3.4945260320409735, 2.430546544538726, 2.0653687015756934, 1.931435403678754, 1.7430490933692009, 1.4013424394215754 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "apple/alarms.cancel", "apple/alarms.list", "apple/alarms.create", "apple/health.workouts.get", "apple/contacts.delete", "apple/music.status", "apple/clipboard.read", "apple/music.playback" ], "scores": [ 12.811641090668399, 9.023378975416925, 6.576508792127638, 5.558597619383658, 2.456488742268665, 2.0639215021837605, 1.9221483250094473, 1.3722784440479752 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/alarms.cancel", "apple/alarms.list", "apple/alarms.create", "apple/health.workouts.get", "builtin/GetCurrentTime", "builtin/ListFiles", "apple/contacts.delete", "apple/music.status", "apple/clipboard.read", "apple/music.playback" ], "scores": [ 18.973992074385638, 8.328936747736847, 5.99343831189088, 5.558597619383658, 5.255138622230446, 5.255138622230446, 2.456488742268665, 2.0639215021837605, 1.9221483250094473, 1.3722784440479752 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H016", "query": "What plain text is available on my clipboard?", "cohort": "intent_transfer", "domain": "clipboard", "category": "heldout_intent", "intent": "capability:apple/clipboard.read", "acceptable": [ "apple/clipboard.read" ], "rationale": "Read available foreground clipboard text.", "basis": [ "apple/clipboard.read" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/clipboard.read", "apple/clipboard.write", "apple/shortcuts.run", "apple/links.open", "builtin/EditFile", "apple/contacts.update", "apple/location.get", "apple/photos.albums.list", "apple/photos.get", "builtin/AskUserQuestions", "builtin/ReadFile", "builtin/Memory" ], "scores": [ 7, 6.5, 3.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/clipboard.read", "apple/clipboard.write", "apple/shortcuts.run", "builtin/ReadFile", "builtin/EditFile", "apple/location.get", "builtin/AskUserQuestions", "apple/photos.get", "apple/photos.albums.list", "apple/contacts.update", "apple/links.open" ], "scores": [ 13.18843483879942, 10.912339148805462, 4.22937390006104, 2.8581499673178334, 2.7272927852161843, 2.499548551624379, 2.165477147820978, 2.072598362607447, 1.9364035337351766, 1.8905693842378057, 1.3094781447112005 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "apple/shortcuts.run", "builtin/EditFile", "builtin/ReadFile", "apple/location.get", "builtin/AskUserQuestions", "apple/photos.get", "apple/photos.albums.list", "apple/contacts.update", "apple/links.open", "builtin/Memory" ], "scores": [ 20.176603863827015, 19.99408318355641, 7.65472216809506, 4.1306699092355466, 2.745409276253241, 2.5055269818647994, 2.062180702926565, 2.054068595663089, 1.9312512392224508, 1.762134739525646, 1.5149625962627569, 1.3563273996917304 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "apple/shortcuts.run", "builtin/EditFile", "builtin/ReadFile", "apple/location.get", "builtin/WriteFile", "apple/photos.get", "builtin/AskUserQuestions", "apple/photos.albums.list", "apple/contacts.update", "apple/links.open", "builtin/Memory" ], "scores": [ 19.907502043684044, 19.82340701502059, 7.479325157472073, 6.323379252210103, 5.019844432888661, 2.5055269818647994, 2.5040111006170407, 2.054068595663089, 1.9405188352247646, 1.9312512392224508, 1.6581745951752167, 1.5149625962627569, 1.276308551475644 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H017", "query": "Replace the system clipboard with this plain-text sentence.", "cohort": "intent_transfer", "domain": "clipboard", "category": "heldout_intent", "intent": "capability:apple/clipboard.write", "acceptable": [ "apple/clipboard.write" ], "rationale": "Write one local plain-text clipboard item.", "basis": [ "apple/clipboard.write" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "apple/shortcuts.run", "builtin/EditFile", "apple/contacts.update", "builtin/Memory", "apple/contacts.create", "apple/device.get", "apple/links.open", "apple/music.playback", "apple/music.status", "builtin/AskUserQuestions", "builtin/ReadFile", "builtin/ScheduledTasks" ], "scores": [ 7.5, 7, 3.5, 2.5, 2, 1.5, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "builtin/EditFile", "apple/shortcuts.run", "apple/contacts.update", "apple/device.get", "builtin/ReadFile", "apple/contacts.create", "builtin/Memory", "apple/music.status", "builtin/AskUserQuestions", "apple/music.playback", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 13.216447426676517, 13.027946198511664, 5.875329114300687, 4.22937390006104, 4.072799739738038, 3.114418360186576, 2.8581499673178334, 2.4692163806821252, 2.4404053139066892, 2.2123786740293387, 2.165477147820978, 1.6175296369719743, 1.2152122937951468, 1.0722663642489143 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "apple/shortcuts.run", "builtin/EditFile", "apple/contacts.update", "builtin/Memory", "apple/device.get", "builtin/ReadFile", "apple/contacts.create", "apple/music.status", "builtin/AskUserQuestions", "apple/music.playback", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 22.47419707025204, 19.834366233911293, 7.65472216809506, 7.342008673116693, 3.9311516648124067, 3.7683034520566503, 3.119415169082174, 2.745409276253241, 2.485854735922362, 2.2108284650669687, 2.062180702926565, 1.5839818954898712, 1.1804942158124918, 1.037484468666328 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/clipboard.write", "apple/clipboard.read", "builtin/EditFile", "apple/shortcuts.run", "apple/contacts.update", "builtin/ReadFile", "builtin/Memory", "apple/device.get", "builtin/WriteFile", "apple/calendar.events.update", "apple/contacts.create", "apple/reminders.update", "apple/music.status", "builtin/AskUserQuestions", "apple/music.playback", "apple/links.open", "builtin/ScheduledTasks" ], "scores": [ 21.898015722751715, 19.663690065375473, 11.866563732953171, 7.479325157472073, 6.091466147888526, 5.019844432888661, 3.365917500697043, 3.119415169082174, 2.5040111006170407, 2.4919310360729865, 2.485854735922362, 2.432653518515385, 2.2108284650669687, 1.9405188352247646, 1.3722784440479752, 1.1804942158124918, 1.037484468666328 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H018", "query": "告诉我本机系统名称和版本，不需要用户自定义设备名。", "cohort": "intent_transfer", "domain": "device", "category": "heldout_intent", "intent": "capability:apple/device.get", "acceptable": [ "apple/device.get" ], "rationale": "Generic system metadata; user device name excluded.", "basis": [ "apple/device.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/device.get" ], "scores": [ 1.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/device.get" ], "scores": [ 5.878033912159178 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/device.get", "builtin/ListFiles" ], "scores": [ 5.255138622230446, 5.255138622230446 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H019", "query": "Search the Apple Music catalog for a song title.", "cohort": "intent_transfer", "domain": "music", "category": "heldout_intent", "intent": "capability:apple/music.search", "acceptable": [ "apple/music.search" ], "rationale": "Catalog song search, not player status.", "basis": [ "apple/music.search" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/links.open", "apple/maps.search", "apple/music.status", "apple/photos.search", "apple/alarms.create", "apple/alarms.list", "apple/shortcuts.run", "apple/alarms.cancel", "apple/maps.route", "apple/maps.show", "apple/reminders.list", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/calendar.events.create", "apple/calendar.events.update", "apple/photos.get", "apple/reminders.create", "apple/reminders.update", "apple/battery.get", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/maps.eta", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "builtin/ScheduledTasks" ], "scores": [ 16.5, 11, 10.5, 8.5, 8, 6, 5.5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 3.5, 3.5, 3.5, 3.5, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/music.status", "apple/links.open", "apple/maps.search", "apple/alarms.create", "apple/alarms.list", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/reminders.list", "apple/maps.show", "apple/weather.current", "apple/photos.search", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 18.19142780969005, 11.567104526399174, 6.802464308650038, 6.4295167169825165, 5.142144244604115, 4.02390377670368, 3.710093167079759, 3.4517436328367657, 2.3108873675196526, 2.253759150951459, 2.101280785461998, 1.9552868225447846, 1.844170375433306, 1.5045297512928304, 1.488613054842949, 1.3334393574442729, 1.2394733488050822 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/music.status", "apple/maps.search", "apple/links.open", "apple/photos.search", "apple/alarms.create", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/alarms.list", "apple/reminders.list", "apple/photos.get", "apple/reminders.create", "apple/reminders.update", "apple/calendar.events.create", "apple/calendar.events.update", "apple/maps.show", "apple/weather.current", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/photos.albums.list" ], "scores": [ 36.120390223945556, 22.830263617249663, 15.306612685862202, 11.080369395714426, 8.701444646997857, 7.136994452378095, 5.347235912577023, 2.9379474069205345, 2.6835360690060837, 2.3393780561429214, 2.2226096457643267, 1.988825859144915, 1.5844747718301733, 1.5844747718301733, 1.4329380784992425, 1.392301010699082, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6606930487736644, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/music.status", "apple/links.open", "apple/maps.search", "apple/alarms.create", "apple/photos.search", "apple/alarms.list", "apple/reminders.list", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/contacts.list", "apple/calendar.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/health.workouts.list", "apple/reminders.create", "apple/reminders.update", "apple/photos.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/maps.show", "apple/weather.current", "apple/maps.route", "apple/alarms.cancel", "apple/weather.hourly", "apple/weather.daily", "apple/battery.get", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/health.sleep.query", "apple/health.workouts.get", "apple/location.reverseGeocode" ], "scores": [ 32.43077813219995, 22.830263617249663, 15.306612685862202, 8.815454431533118, 7.541295345622341, 5.347235912577023, 4.886798507709906, 4.2931910064656105, 3.985984984506879, 2.7583112460783727, 2.6835360690060837, 2.360540450570281, 2.31424198014557, 2.30175925617362, 2.2554607857489093, 2.129474354681442, 1.5844747718301733, 1.5844747718301733, 1.4858533205308402, 1.4329380784992425, 1.392301010699082, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6606930487736644, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H020", "query": "Geocode this address without acquiring my current location.", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.geocode", "acceptable": [ "apple/location.geocode" ], "rationale": "Explicit address geocoding does not acquire current GPS.", "basis": [ "apple/location.geocode" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/weather.current", "apple/location.get", "builtin/GetCurrentTime", "apple/maps.search", "apple/maps.eta", "apple/maps.route", "apple/maps.show", "apple/photos.search", "apple/weather.daily", "apple/weather.hourly", "apple/calendar.events.update", "builtin/EditFile", "apple/battery.get", "apple/contacts.get", "apple/contacts.list", "apple/health.statistics", "apple/music.playback", "apple/music.status", "apple/photos.get", "apple/calendar.events.create", "apple/contacts.create", "apple/contacts.update", "builtin/ScheduledTasks" ], "scores": [ 11.5, 10, 5, 4, 4, 3.5, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/weather.daily", "apple/contacts.get", "apple/weather.current", "apple/maps.eta", "apple/weather.hourly", "apple/photos.search", "apple/contacts.list", "apple/maps.route", "apple/maps.show", "apple/battery.get", "apple/location.get", "builtin/GetCurrentTime", "builtin/EditFile", "apple/music.status", "apple/photos.get", "apple/calendar.events.update", "apple/health.statistics", "apple/music.playback" ], "scores": [ 14.178091098991068, 11.748667831049778, 5.0868647684878265, 3.512741716704385, 3.4456687975865097, 3.4262070896307955, 2.951990858637635, 2.871334244517602, 2.737988306319501, 2.6927249482077116, 2.6363551582324876, 2.6363551582324876, 2.4696228887134084, 2.3227134780257406, 2.205227515488847, 1.8329870145700418, 1.3881142614424309, 1.2706206887809541, 1.187125511716171, 0.9645187186498915, 0.9418286545359099 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "apple/weather.current", "apple/location.get", "apple/maps.search", "builtin/GetCurrentTime", "apple/weather.daily", "apple/maps.eta", "apple/contacts.get", "apple/weather.hourly", "apple/photos.search", "apple/maps.route", "apple/maps.show", "apple/battery.get", "builtin/EditFile", "apple/contacts.list", "apple/calendar.events.update", "apple/contacts.create", "apple/contacts.update", "apple/music.status", "apple/photos.get", "apple/health.statistics", "apple/music.playback", "apple/calendar.events.create", "builtin/ScheduledTasks" ], "scores": [ 27.36200621662389, 22.607952752097084, 6.701063115584889, 6.117188834150327, 5.97583859305092, 5.216525260283588, 3.303509300977252, 2.8094263989532626, 2.7106579929009085, 2.7007337051119755, 2.6355247755985785, 2.4914290360423816, 2.4914290360423816, 2.4735473042818215, 2.3540016180843235, 2.0828091760528666, 1.8436860444451981, 1.5233732602840573, 1.4256527922229154, 1.3186764169813905, 1.196009406912247, 0.898812452944287, 0.8767732783360794, 0.7501397165950441, 0.31570486796627983 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/location.geocode", "apple/location.reverseGeocode", "builtin/GetCurrentTime", "apple/weather.current", "apple/location.get", "apple/maps.search", "apple/weather.daily", "apple/maps.eta", "apple/contacts.get", "apple/weather.hourly", "apple/photos.search", "apple/maps.route", "apple/maps.show", "apple/battery.get", "builtin/EditFile", "apple/contacts.list", "apple/calendar.events.update", "apple/contacts.create", "apple/contacts.update", "apple/music.status", "apple/photos.get", "apple/health.statistics", "apple/music.playback", "apple/calendar.events.create", "builtin/ScheduledTasks" ], "scores": [ 27.36200621662389, 22.607952752097084, 7.081303815323238, 6.701063115584889, 6.117188834150327, 5.97583859305092, 3.303509300977252, 2.8094263989532626, 2.7106579929009085, 2.7007337051119755, 2.6355247755985785, 2.4914290360423816, 2.4914290360423816, 2.4735473042818215, 2.3540016180843235, 2.0828091760528666, 1.8436860444451981, 1.5233732602840573, 1.4256527922229154, 1.3186764169813905, 1.196009406912247, 0.898812452944287, 0.8767732783360794, 0.7501397165950441, 0.31570486796627983 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H021", "query": "Reverse geocode these coordinates without reading GPS.", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.reverseGeocode", "acceptable": [ "apple/location.reverseGeocode" ], "rationale": "Explicit coordinates, no current location read.", "basis": [ "apple/location.reverseGeocode" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/location.reverseGeocode", "apple/location.geocode", "apple/battery.get", "apple/health.statistics", "apple/links.open", "apple/maps.eta", "apple/maps.search", "apple/maps.show", "apple/photos.search", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 11, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/location.reverseGeocode", "apple/location.geocode", "apple/health.statistics", "apple/battery.get", "apple/maps.eta", "apple/maps.search", "apple/maps.show", "apple/weather.current", "apple/photos.search", "apple/weather.hourly", "apple/weather.daily", "apple/links.open" ], "scores": [ 19.494776945331996, 10.677707825278624, 2.583955380200963, 2.4696228887134084, 2.1395511304507666, 2.072598362607447, 1.9107839181010244, 1.726212147301921, 1.6973910453637415, 1.6934955774573934, 1.5741568020017251, 1.1344628597433402 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/location.reverseGeocode", "apple/location.geocode", "apple/health.statistics", "apple/battery.get", "apple/maps.eta", "apple/maps.search", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/photos.search", "apple/weather.daily", "apple/links.open" ], "scores": [ 35.88352296437738, 19.163078504371395, 2.5329463505701764, 2.4735473042818215, 2.142951037296516, 2.054068595663089, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.6656673071721346, 1.5495958966369607, 1.1020517573918405 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/location.reverseGeocode", "apple/location.geocode", "apple/health.statistics", "apple/battery.get", "apple/maps.eta", "apple/maps.search", "apple/maps.show", "apple/weather.current", "apple/weather.hourly", "apple/photos.search", "apple/weather.daily", "apple/links.open" ], "scores": [ 35.88352296437738, 19.163078504371395, 2.5329463505701764, 2.4735473042818215, 2.142951037296516, 2.054068595663089, 1.9003916383525448, 1.7071592606801997, 1.6731343036002873, 1.6656673071721346, 1.5495958966369607, 1.1020517573918405 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H022", "query": "查找给定时区内、指定截止日期区间的未完成提醒。", "cohort": "intent_transfer", "domain": "reminders", "category": "heldout_intent", "intent": "capability:apple/reminders.list", "acceptable": [ "apple/reminders.list" ], "rationale": "Due range and timezone filters on reminder query.", "basis": [ "apple/reminders.list" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/reminders.list", "apple/links.open", "apple/alarms.list", "apple/calendar.events.list", "apple/calendar.list", "apple/contacts.list", "apple/health.workouts.list", "apple/maps.search", "apple/music.search", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/reminders.update" ], "scores": [ 3, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/reminders.list", "apple/reminders.complete", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.update", "apple/reminders.create", "apple/reminders.get", "apple/alarms.list", "apple/music.search", "apple/photos.search", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/maps.search", "apple/health.workouts.list", "apple/links.open" ], "scores": [ 5.303562255067558, 3.914459190398819, 3.914459190398819, 3.5841954611935325, 3.5841954611935325, 3.439115998171961, 3.439115998171961, 2.7182316407131206, 2.535515092349053, 2.535515092349053, 2.375815388733735, 2.375815388733735, 2.235041074941157, 2.235041074941157, 2.235041074941157, 1.8977063868338677, 1.3754065572634524 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/reminders.complete", "builtin/GetCurrentTime", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.delete", "apple/reminders.create", "apple/reminders.get", "apple/music.search", "apple/photos.search", "apple/maps.search", "apple/reminders.update", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.list", "apple/links.open" ], "scores": [ 10.65560558756447, 10.510277244460893, 6.414160958434645, 4.626892566923118, 3.9584393170736933, 3.6565709667499404, 3.588163142246323, 3.5147642278936893, 3.5147642278936893, 3.375434661720279, 3.3383455318645336, 3.246730289701211, 3.1274803458411458, 3.1274803458411458, 3.071081104659632, 3.071081104659632, 2.9134618525541662, 2.214327341230771 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } } } }, { "query_id": "H023", "query": "Read the recurring event occurrence using eventId and occurrenceStart.", "cohort": "intent_transfer", "domain": "calendar", "category": "heldout_intent", "intent": "capability:apple/calendar.events.get", "acceptable": [ "apple/calendar.events.get" ], "rationale": "Get supports occurrenceStart, unlike mutation of recurrence.", "basis": [ "apple/calendar.events.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/clipboard.read", "builtin/ReadFile", "builtin/ScheduledTasks", "apple/photos.search", "apple/photos.get", "apple/contacts.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/music.playback", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.update", "apple/alarms.list", "apple/battery.get", "apple/contacts.list", "apple/device.get", "apple/maps.eta", "apple/maps.show", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/EditFile", "builtin/Memory", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.update", "builtin/AskUserQuestions" ], "scores": [ 9, 7.5, 7.5, 7, 6, 4, 4, 3.5, 3, 2.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.list", "apple/photos.search", "apple/contacts.list", "apple/photos.get", "apple/reminders.complete", "apple/reminders.get", "apple/maps.show", "apple/reminders.delete", "builtin/ScheduledTasks", "apple/reminders.update", "apple/reminders.create", "apple/music.playback", "builtin/ReadFile", "apple/reminders.list", "builtin/Memory", "apple/contacts.get", "builtin/EditFile", "apple/clipboard.read", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/weather.hourly", "apple/weather.daily", "apple/health.statistics" ], "scores": [ 15.433142406740334, 8.271736452633325, 7.21624781152086, 6.220210238198806, 4.4792278995018915, 3.778997392237577, 3.5156165905658834, 3.1514420742797875, 2.769559846226892, 2.6944644547345806, 2.6351421055956448, 2.5703833063631247, 2.473938668954748, 2.430546544538726, 2.0606820288717214, 1.8781025023437183, 1.7356930089104854, 1.7229585090753954, 1.6208496848238496, 1.596129569605437, 1.3632427748418257, 1.3439750578113865, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 0.9551091781400716, 0.8464971647744733, 0.7868454382417407, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.list", "builtin/ReadFile", "apple/clipboard.read", "apple/photos.search", "apple/reminders.complete", "apple/contacts.list", "apple/reminders.delete", "apple/reminders.update", "builtin/ScheduledTasks", "apple/photos.get", "apple/reminders.get", "apple/contacts.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/health.sleep.query", "apple/music.playback", "apple/reminders.create", "apple/maps.show", "builtin/Memory", "apple/health.statistics", "builtin/EditFile", "apple/reminders.list", "apple/weather.current", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/weather.hourly", "apple/weather.daily", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/contacts.update" ], "scores": [ 26.843100365954946, 15.307123710432418, 13.964994414366435, 12.336417806111685, 10.437905895814577, 4.557289964100901, 4.054675138958545, 3.8020496971100908, 3.6814808992486094, 3.50097963484709, 3.4766276057260477, 3.0379985599906574, 3.0062760963057227, 2.9084882265267495, 2.3660103307504636, 2.305049297997499, 1.9810435204490249, 1.9143876060082183, 1.8435682546574144, 1.7244033122771167, 1.7020131435723234, 1.6769433783228374, 1.6111411824454678, 1.391554284613297, 1.3906556055226948, 1.3088266447267578, 1.231374918430577, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 0.9499145762410579, 0.8363195411534453, 0.7745686203791382, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.4532555669664913 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.events.list", "builtin/ScheduledTasks", "apple/contacts.list", "builtin/ReadFile", "apple/photos.get", "apple/photos.search", "apple/reminders.complete", "apple/reminders.get", "apple/reminders.delete", "apple/reminders.update", "apple/clipboard.read", "apple/contacts.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/reminders.list", "apple/device.get", "apple/battery.get", "apple/music.playback", "apple/reminders.create", "apple/maps.show", "apple/alarms.list", "apple/health.sleep.query", "apple/health.statistics", "builtin/Memory", "builtin/EditFile", "apple/location.get", "apple/photos.albums.list", "apple/calendar.list", "apple/weather.current", "apple/maps.eta", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/weather.hourly", "apple/weather.daily", "apple/contacts.update" ], "scores": [ 27.649397678130978, 15.131903363871206, 13.8138450467338, 12.226792321368317, 11.015815487352107, 5.259538952932528, 4.427622193808527, 4.201778249894578, 3.6522478304707198, 3.5911958091097227, 3.5053569245938547, 3.317013613419262, 3.3141890146349793, 2.8850666019795224, 2.8694088104583075, 2.8384433540195007, 2.5032946693164524, 2.390642889725001, 2.1903094058457304, 1.8002653409399807, 1.7904649384226818, 1.7244033122771167, 1.7020131435723234, 1.6769433783228374, 1.648914120016547, 1.522243711957446, 1.1860005645916778, 1.1401709250099572, 0.9841378926907047, 0.9766794376224938, 0.9266425589614367, 0.9034987037373633, 0.8714182810782765, 0.7580363862218745, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.5918458513942811, 0.5481460160064278, 0.4532555669664913 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H024", "query": "Fetch the distance and energy for this one workout UUID, not a workout list.", "cohort": "intent_transfer", "domain": "health", "category": "heldout_intent", "intent": "capability:apple/health.workouts.get", "acceptable": [ "apple/health.workouts.get" ], "rationale": "One workout detail includes recorded distance/energy.", "basis": [ "apple/health.workouts.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/reminders.list", "builtin/ListFiles", "apple/calendar.events.list", "apple/alarms.cancel", "apple/calendar.list", "apple/reminders.lists", "apple/alarms.create", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/health.statistics", "apple/reminders.create", "builtin/Memory", "apple/clipboard.write", "apple/location.get", "apple/maps.eta", "apple/maps.route", "apple/photos.get", "apple/calendar.events.create", "apple/contacts.delete", "apple/contacts.update", "apple/health.sleep.query", "apple/links.open", "apple/maps.search", "apple/battery.get", "apple/calendar.events.get", "apple/contacts.create", "apple/location.geocode", "apple/maps.show", "apple/music.playback", "apple/music.search", "apple/music.status", "apple/photos.search", "apple/reminders.delete", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "builtin/EditFile", "apple/contacts.get", "builtin/ReadFile" ], "scores": [ 9.5, 7.5, 6, 5.5, 5.5, 5, 5, 4.5, 4, 4, 4, 3, 3, 2.5, 2.5, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.list", "apple/alarms.cancel", "apple/maps.eta", "apple/maps.route", "builtin/ListFiles", "apple/health.statistics", "apple/alarms.create", "apple/photos.albums.list", "apple/contacts.list", "apple/clipboard.write", "apple/reminders.create", "builtin/Memory", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/location.get", "apple/photos.get", "apple/reminders.lists", "apple/calendar.list", "apple/contacts.create", "apple/calendar.events.list", "apple/health.sleep.query", "builtin/EditFile", "apple/reminders.list", "apple/contacts.update", "apple/calendar.events.get", "apple/reminders.delete", "apple/weather.current", "builtin/ScheduledTasks", "apple/contacts.delete", "apple/music.search", "apple/location.geocode", "apple/battery.get", "apple/music.status", "apple/shortcuts.run", "apple/calendar.events.create", "apple/maps.show", "apple/links.open", "apple/maps.search", "apple/music.playback", "apple/weather.daily" ], "scores": [ 17.286290376593143, 6.190465248661679, 5.376063967790552, 4.690340417606903, 3.969404983069726, 3.6878165235811355, 3.6725514411033955, 3.4625618501539934, 3.1678677398882535, 2.7028255603149027, 2.4222149813116407, 2.409546974247072, 2.398487524474808, 2.329520180718848, 2.264332115201837, 2.2496739610219643, 2.224105444971989, 2.2054443957376337, 2.1668991632120185, 1.970426045916842, 1.6696723659319328, 1.614897263876673, 1.5629367438392368, 1.5024992193814797, 1.4924909939710067, 1.4593071471968346, 1.457997372644377, 1.4160558529896354, 1.3848806590980578, 1.3760374238911792, 1.3390180568934824, 1.2162317258779303, 1.0700151212109088, 1.018772408912019, 0.9834495738143223, 0.9793722303182939, 0.9697256428343111, 0.9098422129056576, 0.8598345546612385, 0.8549912766967913, 0.7720168266239731, 0.7495532564546209 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.list", "apple/alarms.cancel", "builtin/ListFiles", "apple/contacts.list", "apple/photos.albums.list", "apple/health.statistics", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.list", "apple/maps.eta", "apple/maps.route", "apple/calendar.events.list", "builtin/ScheduledTasks", "apple/alarms.create", "apple/calendar.events.delete", "apple/reminders.create", "apple/calendar.events.update", "builtin/Memory", "apple/clipboard.write", "apple/location.get", "apple/photos.get", "apple/contacts.delete", "apple/contacts.update", "apple/health.sleep.query", "apple/contacts.create", "builtin/EditFile", "apple/reminders.delete", "apple/calendar.events.create", "apple/weather.current", "apple/calendar.events.get", "apple/maps.search", "apple/music.search", "apple/location.geocode", "apple/links.open", "apple/battery.get", "apple/music.status", "apple/maps.show", "apple/shortcuts.run", "apple/music.playback", "apple/weather.daily", "apple/contacts.get", "apple/photos.search", "builtin/ReadFile" ], "scores": [ 28.67640371592544, 13.53029774879896, 6.987601643139297, 6.96665772138221, 5.490625272722834, 4.59463705106631, 4.5572830380489755, 4.391219706857279, 4.144381718209205, 4.1228942449914925, 3.8153957247318555, 3.7938417548343706, 3.6179975215823057, 3.476083825847414, 2.7540363247667377, 2.6003024097317624, 2.5005547746852548, 2.3905680059174137, 2.2521569105226455, 2.2051837675347246, 2.118755749811426, 1.9883640230346527, 1.9268545400297683, 1.8899378890752008, 1.7754494447552547, 1.6244277159640166, 1.5973405741141093, 1.4598619112844569, 1.3691132429326038, 1.230900377922385, 1.1309556352520849, 1.1253412868673076, 1.0904891394022966, 0.9930830738422539, 0.8836339964142832, 0.8407929251848839, 0.8385204047248325, 0.8075967500940832, 0.7436087609996902, 0.724055318310476, 0.720071910007749, 0.6063450614565641, 0.6009512783548976, 0.5822345424416788, 0.5185160798440496 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/alarms.list", "apple/alarms.cancel", "builtin/ListFiles", "apple/contacts.list", "apple/photos.get", "apple/location.get", "apple/photos.albums.list", "apple/health.statistics", "apple/reminders.list", "apple/reminders.lists", "apple/calendar.list", "apple/maps.eta", "apple/calendar.events.get", "apple/maps.route", "apple/battery.get", "apple/calendar.events.list", "apple/contacts.get", "builtin/ScheduledTasks", "apple/device.get", "apple/alarms.create", "apple/reminders.get", "apple/calendar.events.delete", "apple/reminders.create", "apple/calendar.events.update", "builtin/Memory", "apple/clipboard.write", "apple/contacts.delete", "apple/contacts.update", "apple/health.sleep.query", "apple/contacts.create", "builtin/EditFile", "apple/reminders.delete", "apple/calendar.events.create", "apple/weather.current", "apple/maps.search", "apple/music.search", "apple/location.geocode", "apple/links.open", "apple/music.status", "apple/maps.show", "apple/shortcuts.run", "apple/music.playback", "apple/weather.daily", "apple/photos.search", "builtin/ReadFile" ], "scores": [ 31.071560095679594, 13.53029774879896, 6.987601643139297, 6.96665772138221, 6.6907486464665915, 4.59463705106631, 4.587855957729624, 4.577521014981185, 4.5572830380489755, 4.391219706857279, 4.144381718209205, 4.1228942449914925, 3.8153957247318555, 3.7938417548343706, 3.7144982788138403, 3.6179975215823057, 3.575467146557612, 3.476083825847414, 3.2619526960547534, 2.7540363247667377, 2.7369467418327798, 2.6003024097317624, 2.521090036132972, 2.5005547746852548, 2.3905680059174137, 2.2521569105226455, 2.2051837675347246, 2.118755749811426, 1.8899378890752008, 1.7754494447552547, 1.6244277159640166, 1.5973405741141093, 1.4598619112844569, 1.3691132429326038, 1.230900377922385, 1.1309556352520849, 1.0904891394022966, 0.9930830738422539, 0.8836339964142832, 0.8407929251848839, 0.8075967500940832, 0.7436087609996902, 0.724055318310476, 0.720071910007749, 0.6063450614565641, 0.5822345424416788, 0.5185160798440496 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H025", "query": "先拿到日历ID列表，后续查询事件要用。", "cohort": "intent_transfer", "domain": "calendar", "category": "heldout_intent", "intent": "capability:apple/calendar.list", "acceptable": [ "apple/calendar.list" ], "rationale": "Calendar identifier enumeration as a distinct preparation capability.", "basis": [ "apple/calendar.list" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/calendar.list", "apple/photos.get", "apple/reminders.get", "apple/reminders.list", "apple/contacts.get", "apple/health.workouts.get", "apple/links.open", "apple/photos.search", "apple/alarms.list", "apple/battery.get", "apple/contacts.list", "apple/device.get", "apple/health.statistics", "apple/health.workouts.list", "apple/location.get", "apple/maps.search", "apple/music.playback", "apple/music.search", "apple/photos.albums.list", "apple/reminders.create", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/contacts.delete", "apple/contacts.update", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.update", "builtin/AskUserQuestions" ], "scores": [ 3.5, 3.5, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "name_description": { "ranking": [ "apple/reminders.get", "apple/photos.get", "apple/reminders.create", "apple/music.playback", "apple/calendar.events.delete", "apple/reminders.list", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 2.6944644547345806, 2.253912805996125, 2.0606820288717214, 1.8781025023437183, 1.8016996019310052, 1.7229585090753954, 1.5532511243390867, 1.4803458231303015, 1.4405949110207767 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/calendar.events.get", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.events.create", "apple/calendar.list", "apple/reminders.get", "apple/photos.get", "apple/reminders.list", "apple/contacts.get", "apple/photos.search", "apple/health.workouts.get", "apple/music.playback", "apple/alarms.list", "apple/battery.get", "apple/device.get", "apple/reminders.create", "apple/music.search", "apple/contacts.list", "apple/location.get", "apple/photos.albums.list", "apple/maps.search", "apple/health.statistics", "apple/health.workouts.list", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/reminders.complete", "apple/links.open", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update" ], "scores": [ 5.135346201016916, 4.838423933217696, 4.469933295690713, 4.351370669303557, 4.223277023737516, 4.092807550493996, 3.695739092135412, 3.616860675660247, 2.5681127972840576, 2.2937826780519917, 2.1181405787297556, 1.9361206987449302, 1.7244033122771167, 1.7130251578825753, 1.7130251578825753, 1.7130251578825753, 1.7020131435723234, 1.5978774863519025, 1.4972349929374487, 1.4972349929374487, 1.4972349929374487, 1.4085192494009349, 1.3297287613849484, 1.1959314777411738, 0.9919551950628672, 0.8674373781600091, 0.8524674830238934, 0.799714193115399, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "combined": { "ranking": [ "apple/calendar.events.get", "builtin/ListFiles", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.get", "apple/calendar.events.create", "apple/photos.get", "apple/calendar.events.update", "apple/reminders.list", "apple/contacts.get", "apple/photos.search", "apple/health.workouts.get", "apple/music.search", "apple/battery.get", "apple/device.get", "apple/maps.search", "apple/alarms.list", "apple/location.get", "apple/contacts.list", "apple/photos.albums.list", "apple/health.statistics", "apple/health.workouts.list", "apple/music.playback", "apple/reminders.create", "apple/links.open", "builtin/ScheduledTasks", "apple/reminders.delete", "apple/reminders.complete", "apple/alarms.cancel", "apple/contacts.delete", "builtin/AskUserQuestions", "apple/reminders.update", "apple/contacts.update" ], "scores": [ 5.591886654595535, 5.255138622230446, 5.088051475629538, 4.866685649344806, 4.489559904148089, 4.373815946415517, 4.207582464028516, 4.104833783555533, 3.7981435756465824, 3.2099334578129888, 2.781755785947278, 2.735261790880257, 2.6122248622239757, 2.2149986985024035, 2.127193432565133, 2.127193432565133, 2.127193432565133, 2.046084146698569, 2.046084146698569, 1.970933026077081, 1.970933026077081, 1.8479382490914722, 1.8360589197788637, 1.7244033122771167, 1.7020131435723234, 1.3720255930646441, 0.9919551950628672, 0.8674373781600091, 0.8524674830238934, 0.6959051917000892, 0.6959051917000892, 0.6587812483989206, 0.5815098177219924, 0.4532555669664913 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.2 } } } }, { "query_id": "H026", "query": "Find at most three map place candidates in an explicit bounding rectangle.", "cohort": "intent_transfer", "domain": "maps", "category": "heldout_intent", "intent": "capability:apple/maps.search", "acceptable": [ "apple/maps.search" ], "rationale": "Bounded place search, not route directions.", "basis": [ "apple/maps.search" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/maps.search", "apple/maps.route", "apple/maps.show", "apple/maps.eta", "apple/links.open", "apple/location.geocode", "apple/location.reverseGeocode", "apple/calendar.events.create", "apple/calendar.events.update", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 9, 5, 5, 4, 2, 2, 2, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/maps.search", "apple/maps.show", "apple/location.reverseGeocode", "apple/location.geocode", "apple/maps.route", "apple/maps.eta", "apple/calendar.events.create", "apple/weather.current", "apple/weather.hourly", "apple/calendar.events.update", "apple/weather.daily" ], "scores": [ 14.084474967221087, 5.679178377632376, 5.265726297970929, 4.868439187467343, 4.8512698956231874, 4.3678382820242385, 1.3859756210546532, 1.3592000163157691, 1.3334393574442729, 1.3209217681794347, 1.2394733488050822 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.search", "apple/maps.show", "apple/maps.route", "apple/maps.eta", "apple/location.reverseGeocode", "apple/location.geocode", "apple/links.open", "apple/calendar.events.create", "apple/weather.current", "apple/calendar.events.update", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks", "apple/alarms.create" ], "scores": [ 22.103718267893512, 11.332212440451997, 10.083358610482735, 8.22862843041507, 5.297557971417435, 4.798779688417559, 1.8608819506355512, 1.3134609719514998, 1.2733950432027226, 1.2480153304315296, 1.2480153304315296, 1.1558662271255022, 1.0974633722858032, 0.8505056366425909 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/maps.search", "apple/maps.show", "apple/maps.route", "apple/maps.eta", "apple/location.reverseGeocode", "apple/location.geocode", "apple/links.open", "apple/music.search", "apple/photos.search", "apple/battery.get", "apple/device.get", "apple/contacts.get", "apple/photos.get", "apple/alarms.list", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.get", "apple/reminders.list", "apple/calendar.events.create", "apple/health.workouts.list", "apple/weather.current", "apple/calendar.events.update", "apple/weather.hourly", "apple/weather.daily", "builtin/ScheduledTasks", "apple/alarms.create" ], "scores": [ 23.71163921795522, 11.332212440451997, 10.083358610482735, 8.22862843041507, 5.297557971417435, 4.798779688417559, 2.8191757583134, 1.7052574879460844, 1.7052574879460844, 1.607920950061708, 1.607920950061708, 1.5633040505560998, 1.5633040505560998, 1.5210963760156138, 1.5210963760156138, 1.5210963760156138, 1.4811079163986542, 1.4431681307115642, 1.4431681307115642, 1.407123515710695, 1.407123515710695, 1.407123515710695, 1.3728355300713926, 1.3134609719514998, 1.30903966193533, 1.2733950432027226, 1.2480153304315296, 1.2480153304315296, 1.1558662271255022, 1.0974633722858032, 0.8505056366425909 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H027", "query": "在剪贴板写入一个空字符串文本项，不承诺清除所有格式。", "cohort": "intent_transfer", "domain": "clipboard", "category": "heldout_intent", "intent": "capability:apple/clipboard.write", "acceptable": [ "apple/clipboard.write" ], "rationale": "Empty plain-text write is supported; complete clearing not claimed.", "basis": [ "apple/clipboard.write" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/clipboard.read", "apple/clipboard.write" ], "scores": [ 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/clipboard.read", "apple/clipboard.write" ], "scores": [ 5.898701468810263, 5.898701468810263 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "apple/clipboard.read", "apple/clipboard.write", "builtin/WriteFile", "builtin/ReadFile" ], "scores": [ 6.864525244051295, 6.864525244051295, 4.388309091493972, 4.26982905438862 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H028", "query": "Read a current foreground location sample including its accuracy and sample time.", "cohort": "intent_transfer", "domain": "location", "category": "heldout_intent", "intent": "capability:apple/location.get", "acceptable": [ "apple/location.get" ], "rationale": "Current sample with accuracy/time, approximate authorization accepted.", "basis": [ "apple/location.get" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/location.get", "builtin/GetCurrentTime", "apple/clipboard.read", "apple/weather.current", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/weather.daily", "apple/weather.hourly", "builtin/ReadFile", "apple/health.statistics", "apple/battery.get", "apple/health.workouts.get", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/music.status", "apple/photos.get", "apple/photos.search", "apple/calendar.events.list", "builtin/EditFile", "builtin/ScheduledTasks", "apple/calendar.events.update", "apple/device.get", "apple/music.playback", "apple/alarms.create", "apple/health.sleep.query", "apple/health.workouts.list", "apple/reminders.list", "apple/alarms.cancel", "apple/alarms.list", "apple/calendar.events.create", "apple/calendar.events.get", "apple/contacts.get", "apple/links.open", "apple/music.search", "apple/photos.albums.list", "apple/shortcuts.run", "builtin/Memory", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 8, 8, 6, 6, 5, 5, 4, 4, 4, 4, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 2.5, 2.5, 2.5, 2, 2, 2, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/location.get", "apple/health.workouts.get", "apple/weather.daily", "apple/maps.eta", "apple/battery.get", "apple/maps.search", "builtin/GetCurrentTime", "apple/clipboard.read", "apple/weather.hourly", "apple/weather.current", "apple/music.status", "apple/maps.route", "apple/location.reverseGeocode", "apple/maps.show", "apple/health.statistics", "apple/location.geocode", "apple/photos.get", "apple/calendar.events.get", "builtin/EditFile", "apple/calendar.events.list", "apple/photos.search", "apple/device.get", "apple/music.playback", "builtin/ReadFile", "apple/reminders.list", "builtin/Memory", "apple/contacts.get", "apple/alarms.cancel", "apple/alarms.create", "apple/music.search", "apple/shortcuts.run", "apple/health.workouts.list", "apple/calendar.events.update", "apple/photos.albums.list", "apple/health.sleep.query", "builtin/ScheduledTasks", "apple/alarms.list", "apple/links.open" ], "scores": [ 13.523774692199432, 7.93889292795003, 6.470490327444576, 6.422785285694957, 5.534110809368454, 5.266089585301684, 5.194471146866832, 5.1329094937717406, 5.127900578951376, 4.667466950729668, 4.569344504495134, 4.227346908468011, 4.091710037994156, 3.9884909878640613, 3.9011744874993135, 3.8764731609557677, 3.4387706458455707, 3.3189549419353934, 3.196229789411867, 2.920870833332875, 2.8162419726968997, 2.614783540631293, 1.9334663295108596, 1.7356930089104854, 1.6257579312913266, 1.6208496848238496, 1.596129569605437, 1.5741603564255942, 1.4513293217173118, 1.2706206887809541, 1.2579808728455326, 1.2149135706002092, 1.187125511716171, 1.187125511716171, 1.082418100416025, 1.033343757933971, 0.9551091781400716, 0.802784587788313 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/location.get", "builtin/GetCurrentTime", "apple/health.workouts.get", "apple/weather.current", "apple/clipboard.read", "apple/location.geocode", "apple/location.reverseGeocode", "apple/weather.daily", "apple/battery.get", "apple/maps.eta", "apple/maps.search", "builtin/ReadFile", "apple/weather.hourly", "apple/health.statistics", "apple/music.status", "apple/maps.show", "builtin/EditFile", "apple/maps.route", "apple/calendar.events.get", "apple/photos.get", "apple/calendar.events.list", "apple/photos.search", "apple/device.get", "apple/calendar.events.update", "apple/health.workouts.list", "apple/music.playback", "apple/health.sleep.query", "builtin/ScheduledTasks", "builtin/Memory", "apple/contacts.get", "apple/alarms.create", "apple/reminders.list", "apple/calendar.events.create", "apple/music.search", "apple/shortcuts.run", "apple/photos.albums.list", "apple/alarms.cancel", "apple/alarms.list", "apple/reminders.create", "apple/reminders.update", "apple/links.open" ], "scores": [ 16.522711113826585, 10.075215604894034, 8.174663242866822, 7.932438034015465, 7.825390931876223, 7.612304029249678, 7.413375000255735, 6.215109406289789, 5.54290492555391, 5.471121207610204, 5.074405016853241, 4.557289964100901, 4.4588035617089234, 4.41289482032586, 3.916217421859314, 3.8362109343024584, 3.7446572236070184, 3.5383782189910162, 3.432161121074587, 3.3447753317737865, 2.9839324752257763, 2.712315904023117, 2.6210375903801193, 2.433204941791629, 1.8676898296507356, 1.8478442795366619, 1.7720170015982264, 1.7369835932180095, 1.6111411824454678, 1.60914410629741, 1.578316724633953, 1.5142936950371024, 1.365029145517417, 1.2592608876431348, 1.246053816444047, 1.1839668621096326, 1.0352314574515493, 0.9499145762410579, 0.8668821733731853, 0.8668821733731853, 0.779849387030044 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/location.get", "builtin/GetCurrentTime", "apple/health.workouts.get", "apple/location.geocode", "apple/weather.current", "apple/location.reverseGeocode", "apple/clipboard.read", "apple/battery.get", "apple/weather.daily", "apple/maps.eta", "apple/maps.search", "apple/calendar.events.get", "apple/weather.hourly", "apple/health.statistics", "builtin/ReadFile", "apple/photos.get", "apple/music.status", "apple/maps.show", "apple/calendar.events.list", "apple/maps.route", "builtin/EditFile", "apple/device.get", "apple/photos.search", "apple/calendar.events.update", "apple/reminders.list", "apple/health.workouts.list", "apple/contacts.get", "apple/photos.albums.list", "apple/music.playback", "apple/alarms.list", "apple/alarms.create", "apple/health.sleep.query", "builtin/ScheduledTasks", "apple/calendar.events.create", "apple/music.search", "apple/shortcuts.run", "builtin/Memory", "apple/alarms.cancel", "apple/reminders.get", "apple/contacts.list", "apple/calendar.list", "apple/reminders.create", "apple/reminders.update", "apple/links.open" ], "scores": [ 17.49939055144908, 13.547915109995394, 8.696914391734248, 7.612304029249678, 7.572481396663165, 7.413375000255735, 6.640124603375986, 6.2622114851365644, 5.98868680191708, 5.157999214992051, 5.074405016853241, 4.408840558697081, 4.21432987194976, 4.207341100304241, 4.201778249894578, 4.088534935717758, 3.916217421859314, 3.8362109343024584, 3.5618420667633055, 3.5383782189910162, 3.3381395107750285, 3.336295898954703, 2.501462016022749, 2.433204941791629, 2.395776456156075, 2.3439451133675187, 2.1425381623194113, 2.1106094210710693, 1.8478442795366619, 1.648914120016547, 1.578316724633953, 1.4506924588982582, 1.4010894578982818, 1.365029145517417, 1.2592608876431348, 1.246053816444047, 1.1401709250099572, 1.0352314574515493, 0.9510032826687982, 0.9266425589614367, 0.9034987037373633, 0.8668821733731853, 0.8668821733731853, 0.779849387030044 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H029", "query": "Find catalog songs for a nonblank term, returning only the first bounded slice.", "cohort": "intent_transfer", "domain": "music", "category": "heldout_intent", "intent": "capability:apple/music.search", "acceptable": [ "apple/music.search" ], "rationale": "First catalog slice, not subscription or playback operation.", "basis": [ "apple/music.search" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/links.open", "apple/photos.search", "apple/alarms.create", "apple/calendar.events.update", "apple/maps.show", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/alarms.list", "apple/contacts.create", "apple/contacts.update", "apple/maps.search", "apple/music.status", "apple/photos.albums.list", "apple/photos.get", "apple/weather.daily", "apple/weather.hourly", "builtin/EditFile", "apple/location.geocode", "apple/reminders.create", "apple/reminders.update", "builtin/Memory", "builtin/ReadFile" ], "scores": [ 8.5, 4.5, 2, 2, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/music.search", "apple/music.playback", "builtin/EditFile", "apple/photos.search", "apple/links.open", "apple/contacts.create", "apple/weather.hourly", "apple/weather.daily", "apple/contacts.update", "apple/maps.search", "apple/music.status", "apple/shortcuts.run", "apple/photos.albums.list", "builtin/ScheduledTasks", "apple/alarms.list", "apple/maps.show", "apple/alarms.cancel", "apple/photos.get", "apple/alarms.create", "apple/calendar.events.update" ], "scores": [ 19.787573146093006, 7.856618351737842, 3.7611809396809117, 3.511441911062646, 3.0763614287077896, 2.8908421872101764, 2.6189084366781614, 2.4343568322783837, 2.3709354335155055, 2.251823179421304, 1.892501474377823, 1.704499249357205, 1.634894371444502, 1.3894118676695326, 1.2842193286009138, 1.2842193286009138, 1.2706333365586515, 1.2067986160068875, 1.1714863806419569, 1.1274973225408556 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/music.search", "apple/music.playback", "builtin/EditFile", "apple/photos.search", "apple/links.open", "apple/weather.hourly", "apple/weather.daily", "apple/maps.show", "apple/shortcuts.run", "apple/alarms.create", "apple/contacts.create", "apple/music.status", "apple/calendar.events.update", "builtin/ScheduledTasks", "apple/photos.albums.list", "apple/contacts.update", "apple/maps.search", "builtin/ReadFile", "apple/alarms.list", "apple/alarms.cancel", "apple/photos.get", "apple/location.geocode", "apple/reminders.create", "apple/reminders.update", "builtin/Memory" ], "scores": [ 25.768679935138284, 12.407978979354315, 3.489034439062399, 3.2000608748403803, 2.929639624167228, 2.587420718259699, 2.3963745882569323, 2.303565550713148, 2.2526475848785705, 1.9997343865513135, 1.8272714856337944, 1.798233682123699, 1.6247311528579726, 1.606743874083366, 1.5512004698252941, 1.479599485461764, 1.4011872391657196, 1.2392408116133165, 1.2141942578109772, 1.2006046822662546, 1.1369779299085616, 0.9390817750686774, 0.6641879378960013, 0.6641879378960013, 0.38764824653409546 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/music.search", "apple/music.playback", "apple/photos.search", "builtin/Memory", "builtin/EditFile", "apple/maps.search", "apple/photos.albums.list", "apple/links.open", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/maps.show", "apple/shortcuts.run", "apple/alarms.create", "apple/contacts.create", "apple/music.status", "apple/calendar.events.update", "apple/battery.get", "apple/device.get", "builtin/ScheduledTasks", "apple/contacts.get", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.update", "apple/contacts.list", "apple/calendar.events.list", "apple/calendar.list", "apple/health.workouts.get", "apple/reminders.list", "apple/health.workouts.list", "builtin/ReadFile", "apple/alarms.cancel", "apple/location.geocode", "apple/reminders.create", "apple/reminders.update" ], "scores": [ 26.58207336083862, 12.407978979354315, 4.905318362786465, 4.438727002648823, 3.489034439062399, 3.009108189227428, 2.994368600536858, 2.929639624167228, 2.735290633826591, 2.7002819804646614, 2.587420718259699, 2.3963745882569323, 2.303565550713148, 2.2526475848785705, 1.9997343865513135, 1.8272714856337944, 1.798233682123699, 1.6247311528579726, 1.607920950061708, 1.607920950061708, 1.606743874083366, 1.5633040505560998, 1.5210963760156138, 1.5210963760156138, 1.4811079163986542, 1.479599485461764, 1.4431681307115642, 1.407123515710695, 1.407123515710695, 1.407123515710695, 1.3728355300713926, 1.30903966193533, 1.2392408116133165, 1.2006046822662546, 0.9390817750686774, 0.6641879378960013, 0.6641879378960013 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H030", "query": "Cancel one alarm owned by Anan; do not cancel alarms from other apps.", "cohort": "intent_transfer", "domain": "alarms", "category": "heldout_intent", "intent": "capability:apple/alarms.cancel", "acceptable": [ "apple/alarms.cancel" ], "rationale": "Cancellation scope expressly limited to Anan-owned identifiers.", "basis": [ "apple/alarms.cancel" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/alarms.cancel", "apple/alarms.create", "apple/links.open", "apple/alarms.list", "apple/music.status", "apple/music.playback", "apple/health.sleep.query", "apple/photos.albums.list", "apple/calendar.events.create", "apple/clipboard.write", "apple/contacts.delete", "apple/health.workouts.list", "apple/location.get", "apple/photos.get", "apple/reminders.create", "builtin/ScheduledTasks", "apple/contacts.list", "apple/maps.search", "apple/battery.get", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.update", "apple/health.statistics", "apple/health.workouts.get", "apple/location.geocode", "apple/maps.eta", "apple/maps.route", "apple/maps.show", "apple/music.search", "apple/reminders.delete", "apple/shortcuts.run", "apple/weather.current", "apple/weather.daily", "builtin/AskUserQuestions", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "apple/calendar.events.list", "apple/photos.search", "apple/reminders.list", "builtin/ReadFile" ], "scores": [ 11.5, 9, 7.5, 7, 4, 3, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/music.status", "apple/links.open", "apple/music.playback", "apple/health.sleep.query", "apple/reminders.create", "apple/contacts.delete", "apple/photos.albums.list", "apple/calendar.events.create", "apple/health.workouts.list", "builtin/AskUserQuestions", "apple/clipboard.write", "apple/location.get", "apple/photos.get", "apple/shortcuts.run", "apple/clipboard.read", "builtin/ListFiles", "apple/contacts.create", "builtin/ScheduledTasks", "builtin/EditFile", "apple/contacts.update", "apple/reminders.delete", "apple/health.workouts.get", "apple/weather.current", "apple/calendar.events.update", "apple/health.statistics", "apple/music.search", "builtin/Memory", "apple/calendar.events.delete", "apple/location.geocode", "apple/maps.route", "apple/battery.get", "apple/maps.eta", "apple/contacts.list", "apple/maps.show", "apple/maps.search", "apple/weather.daily" ], "scores": [ 14.869476518024427, 12.013004906518958, 9.859027880319072, 7.8168107261887965, 7.176116124476912, 4.630058413927795, 4.5493236172431635, 3.8074895348554194, 3.769564601432209, 3.676260673620342, 3.001494786876812, 2.8055106012874256, 2.499548551624379, 2.409546974247072, 2.224105444971989, 2.2054443957376337, 2.0519806739118946, 1.931435403678754, 1.6822078764459198, 1.6696723659319328, 1.6320889847269806, 1.5024992193814797, 1.4593071471968346, 1.4160558529896354, 1.3959771463946806, 1.3848806590980578, 1.340122705366102, 1.2432451371448472, 1.2162317258779303, 1.164760090359424, 1.1248369805109821, 1.0700151212109088, 1.0526744179854908, 1.018772408912019, 1.018772408912019, 0.9297239873406339, 0.9098422129056576, 0.8549912766967913, 0.7495532564546209 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/links.open", "apple/music.status", "apple/health.sleep.query", "apple/music.playback", "apple/health.workouts.list", "apple/photos.albums.list", "apple/contacts.delete", "apple/reminders.create", "apple/calendar.events.create", "builtin/AskUserQuestions", "apple/clipboard.write", "apple/shortcuts.run", "apple/location.get", "apple/photos.get", "apple/clipboard.read", "builtin/ListFiles", "apple/contacts.create", "builtin/ScheduledTasks", "builtin/EditFile", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.delete", "apple/calendar.events.update", "apple/contacts.list", "apple/weather.current", "builtin/Memory", "apple/maps.search", "apple/calendar.events.delete", "apple/health.statistics", "apple/maps.route", "apple/music.search", "apple/location.geocode", "apple/battery.get", "apple/maps.eta", "apple/maps.show", "apple/weather.daily", "builtin/ReadFile", "apple/reminders.list", "apple/calendar.events.list", "apple/photos.search" ], "scores": [ 34.044178500834825, 18.706898491459146, 16.539196394048545, 9.000291517849716, 7.636169781056127, 4.657166724218358, 4.510162420661672, 4.048044161235209, 3.9267778223731358, 3.745475352988968, 3.6370192794035736, 2.9565037305029316, 2.5383443997316246, 2.118755749811426, 2.0325256171929906, 1.9883640230346527, 1.9268545400297683, 1.9221483250094473, 1.651417995754508, 1.5973405741141093, 1.4969479167281925, 1.4598619112844569, 1.3840390692635096, 1.3691132429326038, 1.3691132429326038, 1.2739620523469748, 1.2500038811300134, 1.1309556352520849, 1.0964747816023659, 1.0904891394022966, 1.0829958778625002, 1.0093267431030897, 0.9971873002352808, 0.9930830738422539, 0.8836339964142832, 0.8385204047248325, 0.8385204047248325, 0.7436087609996902, 0.6063450614565641, 0.5185160798440496, 0.3147136525322586, 0.30990819698276545, 0.28389860771412034 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/links.open", "apple/music.status", "apple/health.sleep.query", "apple/music.playback", "apple/health.workouts.list", "apple/photos.albums.list", "apple/contacts.delete", "apple/reminders.create", "apple/calendar.events.create", "builtin/AskUserQuestions", "apple/clipboard.write", "apple/shortcuts.run", "apple/location.get", "apple/photos.get", "apple/clipboard.read", "builtin/ListFiles", "apple/contacts.create", "builtin/ScheduledTasks", "builtin/EditFile", "apple/contacts.update", "apple/health.workouts.get", "apple/reminders.delete", "apple/calendar.events.update", "apple/contacts.list", "apple/weather.current", "builtin/Memory", "apple/maps.search", "apple/calendar.events.delete", "apple/health.statistics", "apple/maps.route", "apple/music.search", "apple/location.geocode", "apple/battery.get", "apple/maps.eta", "apple/maps.show", "apple/weather.daily", "builtin/ReadFile", "apple/reminders.list", "apple/calendar.events.list", "apple/photos.search" ], "scores": [ 41.11572200800877, 18.706898491459146, 16.539196394048545, 9.660894254877313, 7.636169781056127, 4.657166724218358, 4.510162420661672, 4.048044161235209, 3.9267778223731358, 3.745475352988968, 3.6370192794035736, 2.9565037305029316, 2.5383443997316246, 2.118755749811426, 2.0325256171929906, 1.9883640230346527, 1.9268545400297683, 1.9221483250094473, 1.651417995754508, 1.5973405741141093, 1.4969479167281925, 1.4598619112844569, 1.3840390692635096, 1.3691132429326038, 1.3691132429326038, 1.2739620523469748, 1.2500038811300134, 1.1309556352520849, 1.0964747816023659, 1.0904891394022966, 1.0829958778625002, 1.0093267431030897, 0.9971873002352808, 0.9930830738422539, 0.8836339964142832, 0.8385204047248325, 0.8385204047248325, 0.7436087609996902, 0.6063450614565641, 0.5185160798440496, 0.3147136525322586, 0.30990819698276545, 0.28389860771412034 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H031", "query": "请把说明文档第41行到第60行的文字取出来。", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ReadFile", "acceptable": [ "builtin/ReadFile" ], "rationale": "UTF-8 line offset/limit reader.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile" ], "scores": [ 8.779106343473345, 3.9133926474172007, 3.8077348880372774 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H032", "query": "Peek inside plans/todo.md and show its text.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ReadFile", "acceptable": [ "builtin/ReadFile" ], "rationale": "Ordinary text contents, not just directory names.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/maps.show", "apple/clipboard.read", "apple/clipboard.write", "apple/shortcuts.run", "builtin/EditFile", "apple/calendar.events.get", "apple/contacts.update", "apple/health.workouts.get", "apple/maps.search", "builtin/AskUserQuestions", "builtin/Memory", "builtin/ReadFile" ], "scores": [ 4, 3, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.08333333333333333 } }, "name_description": { "ranking": [ "apple/clipboard.read", "apple/maps.show", "apple/calendar.events.get", "apple/health.workouts.get", "apple/clipboard.write", "builtin/ReadFile", "builtin/EditFile", "apple/maps.search", "builtin/AskUserQuestions", "apple/contacts.update", "apple/shortcuts.run" ], "scores": [ 8.5020536555419, 4.126968314047327, 3.3189549419353934, 3.1777733868287177, 3.147473456270661, 2.8581499673178334, 2.7272927852161843, 2.251823179421304, 2.165477147820978, 1.8905693842378057, 1.7777279238039982 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "bm25": { "ranking": [ "apple/maps.show", "apple/clipboard.read", "apple/clipboard.write", "builtin/EditFile", "apple/calendar.events.get", "apple/health.workouts.get", "apple/shortcuts.run", "builtin/ReadFile", "builtin/Memory", "apple/maps.search", "builtin/AskUserQuestions", "apple/contacts.update" ], "scores": [ 12.510782572457543, 8.27602485171932, 4.561302494397273, 4.1306699092355466, 3.432161121074587, 3.2721473302672357, 2.972996695597959, 2.745409276253241, 2.448760207166782, 2.2316910788332835, 2.062180702926565, 1.762134739525646 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.125 } }, "combined": { "ranking": [ "apple/maps.show", "apple/clipboard.read", "builtin/EditFile", "builtin/ReadFile", "apple/clipboard.write", "apple/calendar.events.get", "apple/health.workouts.get", "apple/shortcuts.run", "builtin/WriteFile", "apple/alarms.list", "builtin/Memory", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/maps.search", "apple/reminders.list", "apple/health.workouts.list", "builtin/AskUserQuestions", "apple/contacts.update" ], "scores": [ 7.1387720841183455, 6.828981819234072, 6.323379252210103, 5.019844432888661, 4.2922006742543, 3.432161121074587, 3.2721473302672357, 2.797599684974972, 2.5040111006170407, 2.4364052592946104, 2.368741358950696, 2.311584248804978, 2.311584248804978, 2.253850044094424, 2.253850044094424, 2.2316910788332835, 2.19892950792101, 2.09674493165168, 1.9405188352247646, 1.6581745951752167 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } } } }, { "query_id": "H033", "query": "Read just the opening lines of a workspace text document.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ReadFile", "acceptable": [ "builtin/ReadFile" ], "rationale": "ReadFile supports bounded lines.", "basis": [ "builtin/ReadFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "apple/clipboard.read", "builtin/EditFile", "apple/clipboard.write", "apple/links.open", "apple/shortcuts.run", "builtin/Memory", "builtin/ScheduledTasks", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.list", "apple/contacts.get", "apple/contacts.update", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/maps.eta", "apple/photos.get", "apple/photos.search", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/AskUserQuestions", "builtin/ListFiles", "builtin/WriteFile" ], "scores": [ 7, 5, 3, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/ReadFile", "apple/clipboard.read", "builtin/EditFile", "apple/links.open", "apple/clipboard.write", "builtin/AskUserQuestions", "apple/contacts.update", "apple/shortcuts.run", "builtin/Memory", "apple/contacts.get", "apple/health.workouts.get", "apple/weather.current", "apple/health.workouts.list", "apple/calendar.events.list", "apple/device.get", "apple/health.sleep.query", "apple/battery.get", "apple/maps.eta", "builtin/ScheduledTasks", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search", "apple/health.statistics" ], "scores": [ 9.740046841675406, 4.436973011954286, 4.09053556005801, 4.0502227497924315, 3.147473456270661, 2.165477147820978, 1.8905693842378057, 1.7777279238039982, 1.6208496848238496, 1.596129569605437, 1.2665935090803384, 1.2412598610988734, 1.2149135706002092, 1.0956951902365162, 1.082418100416025, 1.082418100416025, 1.0694589285764813, 1.0694589285764813, 1.033343757933971, 0.9551091781400716, 0.8975292682836623, 0.8464971647744733, 0.7868454382417407, 0.7350474507853814, 0.717339055840276 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/ReadFile", "apple/clipboard.read", "builtin/EditFile", "apple/clipboard.write", "apple/links.open", "apple/shortcuts.run", "builtin/Memory", "builtin/AskUserQuestions", "apple/contacts.update", "apple/contacts.get", "builtin/ListFiles", "apple/health.workouts.get", "apple/health.workouts.list", "apple/weather.current", "builtin/WriteFile", "builtin/ScheduledTasks", "apple/calendar.events.list", "apple/health.sleep.query", "apple/device.get", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search", "apple/health.statistics" ], "scores": [ 16.27280500599228, 6.947653093782435, 6.65278921593713, 4.561302494397273, 4.351885218746987, 2.972996695597959, 2.967468582137198, 2.062180702926565, 1.762134739525646, 1.60914410629741, 1.369712183433248, 1.3042089742613945, 1.2461128550311649, 1.231374918430577, 1.1937564975400883, 1.1490595548480602, 1.113807122507814, 1.0992184654999453, 1.0850070323653966, 1.0711583788400274, 1.0711583788400274, 0.9499145762410579, 0.8895050372184048, 0.8363195411534453, 0.7745686203791382, 0.721309630646165, 0.7031782969374535 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/EditFile", "builtin/WriteFile", "apple/clipboard.read", "apple/links.open", "apple/clipboard.write", "apple/shortcuts.run", "builtin/Memory", "apple/contacts.get", "builtin/AskUserQuestions", "apple/health.workouts.get", "apple/device.get", "apple/battery.get", "apple/health.workouts.list", "apple/calendar.events.list", "apple/contacts.update", "apple/alarms.list", "apple/photos.get", "builtin/ListFiles", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.list", "apple/reminders.list", "apple/weather.current", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/maps.eta", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search", "apple/health.statistics" ], "scores": [ 26.970834791894724, 12.246715734116972, 7.611160245574329, 5.591710596746378, 4.351885218746987, 4.2922006742543, 2.797599684974972, 2.416479476485601, 2.1425381623194113, 1.9405188352247646, 1.8264601231288218, 1.8002653409399807, 1.7904649384226818, 1.7223681387479477, 1.6917167140453433, 1.6581745951752167, 1.648914120016547, 1.6332646411623757, 1.369712183433248, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.8714182810782765, 0.8131654195283325, 0.7778939227999769, 0.7580363862218745, 0.5918458513942811, 0.5481460160064278, 0.5104557426457969, 0.4976245769158341 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H034", "query": "把这份纪要另存为一个新的UTF-8文档。", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/WriteFile", "acceptable": [ "builtin/WriteFile" ], "rationale": "Create a new complete UTF-8 file.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile" ], "scores": [ 2, 2 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile" ], "scores": [ 8.839979462997334, 8.700822527580458 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "bm25": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile" ], "scores": [ 9.055015459005586, 8.899824244678054 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile" ], "scores": [ 12.862750347042864, 12.813216892095255, 3.8077348880372774 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H035", "query": "Persist the whole draft under a fresh filename.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/WriteFile", "acceptable": [ "builtin/WriteFile" ], "rationale": "Whole new file, not one-segment patch.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/links.open", "builtin/ScheduledTasks", "apple/calendar.events.delete", "apple/contacts.list", "apple/maps.search", "apple/reminders.delete" ], "scores": [ 2, 1.5, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/reminders.delete", "apple/calendar.events.delete", "apple/contacts.list", "builtin/ScheduledTasks", "apple/maps.search" ], "scores": [ 3.9749726046748153, 3.157499877435286, 3.0195051802203676, 2.939466819887189, 2.776792493451411 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/reminders.delete", "apple/calendar.events.delete", "apple/contacts.list", "builtin/ScheduledTasks", "apple/maps.search", "apple/links.open" ], "scores": [ 4.034985618403735, 3.1917540893882284, 3.0069337400541807, 2.887598644067425, 2.7519669803733415, 1.5055923631327193 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ListFiles", "apple/reminders.delete", "apple/calendar.events.delete", "apple/contacts.list", "builtin/ScheduledTasks", "apple/maps.search", "apple/links.open" ], "scores": [ 5.255138622230446, 4.034985618403735, 3.1917540893882284, 3.0069337400541807, 2.887598644067425, 2.7519669803733415, 2.390644071377807 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "H036", "query": "在phone工作区落一份完整文本，不覆盖已存在文件。", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/WriteFile", "acceptable": [ "builtin/WriteFile" ], "rationale": "New-file write with no overwrite.", "basis": [ "builtin/WriteFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "builtin/EditFile", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile" ], "scores": [ 1.5, 1, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.14285714285714285 } }, "name_description": { "ranking": [ "apple/contacts.list", "apple/contacts.update" ], "scores": [ 3.0195051802203676, 2.9236734812969383 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "builtin/ListFiles", "builtin/ReadFile", "builtin/WriteFile", "builtin/EditFile" ], "scores": [ 3.4137247956955434, 1.9444089450290927, 1.6141984136195984, 1.0922001006905449, 0.9881630284044697, 0.9518941151163509, 0.9022222210637898 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/ReadFile", "builtin/ListFiles", "apple/contacts.update", "apple/contacts.list", "apple/contacts.create", "builtin/EditFile" ], "scores": [ 9.253595854027523, 9.065726970830369, 6.682895338720131, 3.4137247956955434, 1.9444089450290927, 1.6141984136195984, 0.9022222210637898 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H037", "query": "这个文件夹的第一层都放了啥？", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ListFiles", "acceptable": [ "builtin/ListFiles" ], "rationale": "One-level listing.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ListFiles" ], "scores": [ 5.255138622230446 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H038", "query": "Enumerate entries immediately under archive/.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ListFiles", "acceptable": [ "builtin/ListFiles" ], "rationale": "Directory entries, not file content.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/music.playback", "apple/music.status", "builtin/ListFiles" ], "scores": [ 1.5, 1, 1, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "apple/links.open", "apple/music.status", "apple/music.playback" ], "scores": [ 3.102197171736818, 2.8483283627959466, 1.9325766935215902 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/links.open", "apple/music.status", "apple/music.playback", "builtin/ListFiles" ], "scores": [ 3.0311160534548236, 2.5883337910366304, 1.720954340406063, 1.369712183433248 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "combined": { "ranking": [ "builtin/ListFiles", "apple/links.open", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/music.status", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/health.workouts.list", "apple/music.playback" ], "scores": [ 5.029950043649491, 3.0311160534548236, 2.761049801831932, 2.6195967225619254, 2.6195967225619254, 2.5883337910366304, 2.554169588111743, 2.554169588111743, 2.4919310360729865, 2.376130635879959, 1.720954340406063 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H039", "query": "Show directory entries without walking subfolders.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/ListFiles", "acceptable": [ "builtin/ListFiles" ], "rationale": "Listing is explicitly one level.", "basis": [ "builtin/ListFiles" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/maps.show", "apple/health.workouts.list", "apple/links.open", "apple/maps.eta", "apple/maps.route", "builtin/ListFiles", "apple/battery.get", "apple/clipboard.read", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/music.playback", "apple/music.status", "apple/photos.search", "builtin/Memory" ], "scores": [ 4, 1.5, 1.5, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0.16666666666666666 } }, "name_description": { "ranking": [ "builtin/ListFiles", "apple/maps.show", "apple/health.workouts.list", "builtin/Memory", "apple/links.open", "apple/clipboard.read", "apple/maps.eta", "apple/music.status", "apple/location.reverseGeocode", "apple/maps.route", "apple/location.geocode", "apple/battery.get", "apple/maps.search", "apple/music.playback", "apple/photos.search" ], "scores": [ 4.7220808488051, 4.126968314047327, 3.3519413045349737, 3.269566973945597, 3.102197171736818, 2.986870790172134, 2.950632574157707, 2.8483283627959466, 2.8055106012874256, 2.6351421055956448, 2.5938411871930867, 2.4696228887134084, 2.072598362607447, 1.9325766935215902, 1.6973910453637415 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "apple/maps.show", "builtin/ListFiles", "apple/maps.eta", "apple/health.workouts.list", "apple/maps.route", "builtin/Memory", "apple/links.open", "apple/clipboard.read", "apple/location.reverseGeocode", "apple/location.geocode", "apple/music.status", "apple/battery.get", "apple/maps.search", "apple/music.playback", "apple/photos.search" ], "scores": [ 12.510782572457543, 6.23669298085044, 4.766694921407074, 4.482432896478026, 4.432183792644562, 3.2314784752437844, 3.0311160534548236, 2.9725087752942105, 2.877556815390035, 2.606627671177044, 2.5883337910366304, 2.4735473042818215, 2.054068595663089, 1.720954340406063, 1.6656673071721346 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "combined": { "ranking": [ "builtin/ListFiles", "apple/maps.show", "apple/health.workouts.list", "apple/maps.eta", "apple/maps.route", "builtin/Memory", "apple/links.open", "apple/location.reverseGeocode", "apple/location.geocode", "apple/music.status", "apple/battery.get", "apple/alarms.list", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.events.list", "apple/calendar.list", "apple/reminders.list", "apple/maps.search", "apple/music.playback", "apple/clipboard.read", "apple/photos.search" ], "scores": [ 14.41048284078768, 7.1387720841183455, 6.579177828129706, 4.766694921407074, 4.432183792644562, 3.2314784752437844, 3.0311160534548236, 2.877556815390035, 2.606627671177044, 2.5883337910366304, 2.4735473042818215, 2.4364052592946104, 2.311584248804978, 2.311584248804978, 2.253850044094424, 2.253850044094424, 2.19892950792101, 2.054068595663089, 1.720954340406063, 1.6961419113447813, 1.6656673071721346 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H040", "query": "只将唯一的旧称呼换成新称呼，其他字保留。", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/EditFile", "acceptable": [ "builtin/EditFile" ], "rationale": "Unique exact text replacement.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "H041", "query": "Patch the unique occurrence of alpha to beta inside draft.md.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/EditFile", "acceptable": [ "builtin/EditFile" ], "rationale": "Exact segment patch.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/calendar.events.get", "builtin/ScheduledTasks", "builtin/EditFile", "builtin/Memory" ], "scores": [ 2, 1.5, 1.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "apple/calendar.events.get", "builtin/ScheduledTasks" ], "scores": [ 4.765146137190348, 2.939466819887189 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/calendar.events.get", "builtin/ScheduledTasks", "builtin/EditFile", "apple/links.open", "builtin/Memory" ], "scores": [ 7.89510911690497, 2.887598644067425, 1.6244842064350702, 1.5055923631327193, 1.0924328074750516 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "apple/calendar.events.get", "builtin/ScheduledTasks", "apple/links.open", "builtin/EditFile", "builtin/Memory" ], "scores": [ 7.89510911690497, 2.887598644067425, 2.390644071377807, 1.6244842064350702, 1.0924328074750516 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } } } }, { "query_id": "H042", "query": "Correct one exact passage after checking the current document.", "cohort": "wording_transfer", "domain": "files", "category": "heldout_wording", "intent": "capability:builtin/EditFile", "acceptable": [ "builtin/EditFile" ], "rationale": "Single-match edit after read.", "basis": [ "builtin/EditFile" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/weather.current", "builtin/GetCurrentTime", "apple/music.playback", "builtin/EditFile", "apple/clipboard.write", "apple/shortcuts.run", "builtin/Memory", "apple/alarms.create", "apple/calendar.events.create", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.update", "apple/location.get", "apple/maps.route", "apple/maps.show", "apple/reminders.delete", "apple/links.open", "apple/maps.search", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/health.statistics", "apple/health.workouts.get", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.eta", "apple/music.status", "apple/photos.albums.list", "apple/photos.get", "apple/reminders.complete", "apple/reminders.create", "apple/reminders.update", "apple/weather.daily", "apple/weather.hourly", "builtin/ListFiles", "builtin/ReadFile" ], "scores": [ 4, 4, 3, 3, 2.5, 2.5, 2.5, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1.5, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 1, "rr": 0.25 } }, "name_description": { "ranking": [ "builtin/Memory", "apple/shortcuts.run", "builtin/EditFile", "apple/contacts.create", "apple/reminders.delete", "apple/contacts.delete", "apple/clipboard.write", "apple/contacts.update", "apple/music.playback", "apple/calendar.events.create", "apple/location.get", "apple/alarms.create", "apple/maps.show", "apple/maps.route", "builtin/GetCurrentTime", "apple/photos.albums.list", "apple/reminders.complete", "builtin/ListFiles", "apple/weather.current", "apple/location.reverseGeocode", "apple/weather.daily", "apple/reminders.update", "apple/location.geocode", "apple/reminders.create", "apple/maps.eta", "apple/links.open", "apple/health.workouts.get", "apple/music.status", "apple/calendar.events.update", "apple/maps.search", "apple/weather.hourly", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/photos.get", "apple/health.statistics" ], "scores": [ 3.8161957723429456, 3.36524482061552, 3.3354862339515217, 3.142467501818258, 3.058319788472353, 2.8919377804719337, 2.8572611340284704, 2.667225790916384, 2.6091882667383026, 2.5786791220069745, 2.5765501963347868, 2.5579637252529066, 2.5050534000910902, 2.3368937465864046, 2.205227515488847, 2.1038511081773286, 1.769521394478326, 1.6822078764459198, 1.668971403398956, 1.6335467459150792, 1.5605468552419268, 1.552919723578451, 1.5102993475823525, 1.492041006854217, 1.4379715520033696, 1.4227132918780445, 1.3959771463946806, 1.3881142614424309, 1.340122705366102, 1.2067986160068875, 1.1381819434781144, 1.1248369805109821, 1.0430656701972452, 1.0415379820610877, 0.9892126698597037, 0.9645187186498915 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "bm25": { "ranking": [ "builtin/GetCurrentTime", "apple/weather.current", "builtin/EditFile", "apple/clipboard.write", "builtin/Memory", "apple/shortcuts.run", "apple/contacts.create", "apple/reminders.delete", "apple/contacts.delete", "apple/contacts.update", "apple/calendar.events.create", "apple/location.get", "apple/alarms.create", "apple/music.playback", "apple/maps.show", "apple/maps.route", "apple/maps.search", "apple/reminders.complete", "apple/photos.albums.list", "builtin/ListFiles", "apple/location.reverseGeocode", "apple/reminders.update", "apple/reminders.create", "apple/weather.daily", "apple/location.geocode", "apple/maps.eta", "apple/health.workouts.get", "builtin/ScheduledTasks", "apple/music.status", "apple/links.open", "apple/calendar.events.update", "apple/calendar.events.delete", "apple/weather.hourly", "apple/alarms.cancel", "apple/photos.get", "apple/health.statistics", "builtin/ReadFile" ], "scores": [ 5.216525260283588, 5.045382794218522, 4.71608575043257, 3.9912559184101504, 3.7928099458394398, 3.7028871156742187, 3.0800598660438996, 3.0361716154865928, 2.858481269108505, 2.5846437515297644, 2.533360060741697, 2.5258740119747944, 2.490588381072274, 2.4736184666799077, 2.4283885156219545, 2.211381558046258, 2.144331977579361, 1.807507345290181, 1.6731343036002873, 1.651417995754508, 1.592799090577726, 1.569494658388202, 1.503500956553208, 1.4663815769404227, 1.4428330874025772, 1.369169802552464, 1.3691132429326038, 1.3310139740131004, 1.3186764169813905, 1.3124837602688564, 1.2739620523469748, 1.0829958778625002, 1.0689954759741835, 0.9860265225742033, 0.9337714661875144, 0.898812452944287, 0.5185160798440496 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.3333333333333333 } }, "combined": { "ranking": [ "builtin/EditFile", "builtin/GetCurrentTime", "apple/contacts.update", "apple/weather.current", "apple/reminders.update", "apple/calendar.events.update", "builtin/ReadFile", "apple/clipboard.write", "builtin/WriteFile", "builtin/Memory", "apple/shortcuts.run", "apple/contacts.create", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.create", "apple/location.get", "apple/alarms.create", "apple/music.playback", "apple/maps.show", "apple/maps.route", "apple/maps.search", "apple/reminders.complete", "apple/photos.albums.list", "builtin/ListFiles", "apple/location.reverseGeocode", "apple/reminders.create", "apple/weather.daily", "apple/location.geocode", "apple/maps.eta", "apple/health.workouts.get", "builtin/ScheduledTasks", "apple/music.status", "apple/links.open", "apple/calendar.events.delete", "apple/weather.hourly", "apple/alarms.cancel", "apple/photos.get", "apple/health.statistics" ], "scores": [ 13.495192093905914, 7.081303815323238, 6.10707230177324, 5.045382794218522, 4.924341668685451, 4.710558066784085, 4.326250967881327, 3.9912559184101504, 3.9133926474172007, 3.7928099458394398, 3.7028871156742187, 3.0800598660438996, 3.0361716154865928, 2.858481269108505, 2.533360060741697, 2.5258740119747944, 2.490588381072274, 2.4736184666799077, 2.4283885156219545, 2.211381558046258, 2.144331977579361, 1.807507345290181, 1.6731343036002873, 1.651417995754508, 1.592799090577726, 1.503500956553208, 1.4663815769404227, 1.4428330874025772, 1.369169802552464, 1.3691132429326038, 1.3310139740131004, 1.3186764169813905, 1.3124837602688564, 1.0829958778625002, 1.0689954759741835, 0.9860265225742033, 0.9337714661875144, 0.898812452944287 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H043", "query": "以后每次推荐餐馆都考虑我吃素。", "cohort": "wording_transfer", "domain": "memory", "category": "heldout_wording", "intent": "capability:builtin/Memory", "acceptable": [ "builtin/Memory" ], "rationale": "Persist a lasting user preference, not a one-off note.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "H044", "query": "What lasting preferences did you store about me?", "cohort": "wording_transfer", "domain": "memory", "category": "heldout_wording", "intent": "capability:builtin/Memory", "acceptable": [ "builtin/Memory" ], "rationale": "Inspect lasting user memory.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "builtin/Memory" ], "scores": [ 1 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "name_description": { "ranking": [ "builtin/Memory" ], "scores": [ 3.8067640926287614 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/Memory" ], "scores": [ 3.762417568958924 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/Memory" ], "scores": [ 13.195777563159385 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H045", "query": "把已保存的偏好里那条旧习惯忘掉。", "cohort": "wording_transfer", "domain": "memory", "category": "heldout_wording", "intent": "capability:builtin/Memory", "acceptable": [ "builtin/Memory" ], "rationale": "Delete within lasting memory scope.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/WriteFile", "builtin/Memory" ], "scores": [ 5.109318025896343, 4.716679997100231 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H046", "query": "Change my persistent profile from tea to coffee.", "cohort": "wording_transfer", "domain": "memory", "category": "heldout_wording", "intent": "capability:builtin/Memory", "acceptable": [ "builtin/Memory" ], "rationale": "Update lasting memory through exact edit.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/alarms.cancel", "apple/calendar.events.list", "apple/calendar.events.update", "apple/contacts.list", "apple/maps.search", "apple/maps.show", "apple/music.playback", "apple/photos.albums.list", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.list", "apple/reminders.update", "builtin/ScheduledTasks" ], "scores": [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [ "apple/reminders.delete", "apple/reminders.complete", "apple/maps.show", "apple/alarms.cancel", "apple/maps.search", "apple/calendar.events.update", "apple/reminders.update", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/photos.albums.list", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 3.2234801375618343, 2.443914691723882, 2.3962861849916948, 2.3709354335155055, 2.251823179421304, 2.146531974526627, 2.1447626117230607, 1.934298977912546, 1.7229585090753954, 1.7229585090753954, 1.4803458231303015, 1.2365724396619533, 0.8833160037757247 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/reminders.delete", "apple/reminders.complete", "apple/maps.show", "apple/alarms.cancel", "apple/maps.search", "apple/reminders.update", "apple/calendar.events.update", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/photos.albums.list", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 3.2721473302672357, 2.4963777043542335, 2.383253378845514, 2.3565793918572813, 2.2316910788332835, 2.167654522960934, 2.1423881224858845, 1.9662731002711833, 1.7157851251886742, 1.7157851251886742, 1.4764069862460605, 1.2109257920942886, 0.8546632305151596 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/Memory", "apple/calendar.events.update", "apple/reminders.update", "apple/reminders.delete", "apple/reminders.complete", "apple/maps.show", "builtin/EditFile", "apple/alarms.cancel", "apple/maps.search", "apple/contacts.update", "apple/calendar.events.list", "apple/contacts.list", "apple/reminders.list", "apple/photos.albums.list", "apple/music.playback", "builtin/ScheduledTasks" ], "scores": [ 7.740405830599973, 3.8737846731377363, 3.8496555889810375, 3.011713912930352, 2.227533323467932, 2.1935679034724593, 2.174019538423646, 2.1690169252867606, 2.054068595663089, 2.011124386571428, 1.754517694278828, 1.5310057190472472, 1.5310057190472472, 1.3174071195747592, 1.08051660188761, 0.7626213064612145 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H047", "query": "列一下长期记忆文件的目录。", "cohort": "wording_transfer", "domain": "memory", "category": "heldout_wording", "intent": "capability:builtin/Memory", "acceptable": [ "builtin/Memory" ], "rationale": "Memory view enumerates its own namespace.", "basis": [ "builtin/Memory" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "builtin/ListFiles", "builtin/Memory", "builtin/WriteFile", "builtin/ReadFile" ], "scores": [ 10.845833860260033, 9.433359994200462, 3.9133926474172007, 3.8077348880372774 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } } } }, { "query_id": "H048", "query": "本机现在的当地日期和UTC偏移各是什么？", "cohort": "wording_transfer", "domain": "clock", "category": "heldout_wording", "intent": "capability:builtin/GetCurrentTime", "acceptable": [ "builtin/GetCurrentTime" ], "rationale": "Current device time metadata.", "basis": [ "builtin/GetCurrentTime" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/photos.search", "builtin/GetCurrentTime", "apple/alarms.create", "apple/calendar.events.create", "apple/calendar.events.list", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.list", "builtin/ScheduledTasks" ], "scores": [ 1.5, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5 ], "metrics": { "hit1": 0, "hit3": 1, "hit5": 1, "rr": 0.5 } }, "name_description": { "ranking": [ "builtin/GetCurrentTime", "apple/photos.search" ], "scores": [ 4.03298001739044, 2.274103269717958 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "bm25": { "ranking": [ "builtin/GetCurrentTime", "apple/photos.search", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/alarms.create", "apple/health.workouts.list", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 2.242201989857617, 2.1112914553016133, 1.0168546914547099, 0.9711253308414564, 0.9603285058573691, 0.9493895120856125, 0.9394394143714464, 0.9293319871823952, 0.8909874565784706, 0.23158550835899408 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } }, "combined": { "ranking": [ "builtin/GetCurrentTime", "apple/photos.search", "apple/health.sleep.query", "apple/calendar.events.list", "apple/health.statistics", "apple/alarms.create", "apple/health.workouts.list", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/ScheduledTasks" ], "scores": [ 12.75247923431851, 2.1112914553016133, 1.0168546914547099, 0.9711253308414564, 0.9603285058573691, 0.9493895120856125, 0.9394394143714464, 0.9293319871823952, 0.8909874565784706, 0.23158550835899408 ], "metrics": { "hit1": 1, "hit3": 1, "hit5": 1, "rr": 1.0 } } } }, { "query_id": "H049", "query": "我还没定方案，你先让我选一个。", "cohort": "wording_transfer", "domain": "questions", "category": "heldout_wording", "intent": "capability:builtin/AskUserQuestions", "acceptable": [ "builtin/AskUserQuestions" ], "rationale": "Collect missing user choice.", "basis": [ "builtin/AskUserQuestions" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "H050", "query": "下周起每周五帮我查一次天气，允许延迟。", "cohort": "wording_transfer", "domain": "scheduling", "category": "heldout_wording", "intent": "capability:builtin/ScheduledTasks", "acceptable": [ "builtin/ScheduledTasks" ], "rationale": "Best-effort weekly read-only task, not a calendar event.", "basis": [ "builtin/ScheduledTasks" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 1.5, 1.5, 1.5 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "name_description": { "ranking": [], "scores": [], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "bm25": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 5.26032566896794, 5.26032566896794, 5.26032566896794 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } }, "combined": { "ranking": [ "apple/weather.current", "apple/weather.daily", "apple/weather.hourly" ], "scores": [ 6.121624994499776, 6.121624994499776, 6.121624994499776 ], "metrics": { "hit1": 0, "hit3": 0, "hit5": 0, "rr": 0 } } } }, { "query_id": "H051", "query": "Send an email directly, without relying on a preconfigured shortcut.", "cohort": "unsupported", "domain": "mail", "category": "unsupported", "intent": "unsupported:mail", "acceptable": [], "rationale": "No mail-send tool; URL handoff alone does not send mail.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/shortcuts.run", "apple/contacts.update", "apple/battery.get", "apple/contacts.list", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/photos.search", "builtin/GetCurrentTime", "apple/contacts.create" ], "scores": [ 5, 1.5, 1, 1, 1, 1, 1, 1, 1, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/shortcuts.run", "builtin/GetCurrentTime", "apple/contacts.list", "apple/contacts.update", "apple/location.reverseGeocode", "apple/location.geocode", "apple/battery.get", "apple/maps.search", "apple/photos.search" ], "scores": [ 5.804482858439773, 4.695607595388778, 3.0195051802203676, 2.9236734812969383, 2.8055106012874256, 2.5938411871930867, 2.4696228887134084, 2.072598362607447, 1.6973910453637415 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/shortcuts.run", "builtin/GetCurrentTime", "apple/contacts.update", "apple/location.reverseGeocode", "apple/contacts.list", "apple/location.geocode", "apple/battery.get", "apple/contacts.create", "apple/maps.search", "apple/photos.search" ], "scores": [ 20.059153084421567, 4.848644661805273, 4.707832142000271, 2.877556815390035, 2.681514028354969, 2.606627671177044, 2.4735473042818215, 2.226124140055592, 2.054068595663089, 1.6656673071721346 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/shortcuts.run", "builtin/GetCurrentTime", "apple/contacts.update", "apple/location.reverseGeocode", "apple/contacts.list", "apple/location.geocode", "apple/battery.get", "apple/contacts.create", "apple/maps.search", "apple/photos.search" ], "scores": [ 20.059153084421567, 4.848644661805273, 4.707832142000271, 2.877556815390035, 2.681514028354969, 2.606627671177044, 2.4735473042818215, 2.226124140055592, 2.054068595663089, 1.6656673071721346 ], "metrics": { "false_recall": true } } } }, { "query_id": "H052", "query": "Write the full body of a new Apple Notes note without a saved shortcut.", "cohort": "unsupported", "domain": "notes", "category": "unsupported", "intent": "unsupported:notes", "acceptable": [], "rationale": "No Notes content-write capability.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/clipboard.write", "apple/calendar.events.update", "apple/health.statistics", "apple/maps.search", "apple/music.playback", "apple/calendar.events.create", "apple/reminders.update", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.delete", "apple/contacts.delete", "apple/contacts.get", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.route", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/photos.search", "apple/reminders.list", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/reminders.create", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.list", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/location.get", "apple/maps.eta", "apple/photos.albums.list", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "builtin/WriteFile", "builtin/ScheduledTasks", "builtin/Memory", "builtin/EditFile" ], "scores": [ 11, 10, 7.5, 6, 5.5, 5, 5, 4.5, 4.5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 1.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/shortcuts.run", "apple/health.statistics", "builtin/WriteFile", "apple/clipboard.write", "apple/calendar.events.update", "apple/maps.search", "apple/links.open", "apple/contacts.get", "apple/reminders.update", "apple/location.reverseGeocode", "apple/location.geocode", "builtin/ScheduledTasks", "apple/battery.get", "apple/reminders.list", "apple/music.status", "apple/music.playback", "apple/health.workouts.get", "apple/maps.show", "apple/contacts.delete", "apple/health.workouts.list", "apple/alarms.create", "apple/weather.current", "apple/health.sleep.query", "apple/calendar.events.delete", "apple/photos.search", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/calendar.events.create", "apple/music.search", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 10.73643073318075, 5.096808095761029, 3.8795962403736803, 3.7936376826106004, 3.5006833303877127, 3.486425721505277, 3.442277567091643, 3.133344695217922, 3.048112898315442, 2.8055106012874256, 2.5938411871930867, 2.535933711807245, 2.4696228887134084, 2.448649573649498, 2.38399622539939, 2.270212561002062, 2.109852852396264, 2.101280785461998, 2.0237659864584026, 2.0237659864584026, 2.01195188835184, 1.9552868225447846, 1.8030590716561463, 1.7000568511749237, 1.6973910453637415, 1.5045297512928304, 1.5045297512928304, 1.488613054842949, 1.4656245762044289, 1.41382735889783, 1.3334393574442729, 1.2394733488050822 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/shortcuts.run", "apple/clipboard.write", "builtin/WriteFile", "apple/health.statistics", "apple/links.open", "apple/calendar.events.update", "apple/reminders.update", "apple/location.reverseGeocode", "builtin/Memory", "apple/location.geocode", "apple/contacts.get", "apple/calendar.events.create", "apple/battery.get", "apple/music.status", "apple/maps.search", "apple/health.workouts.get", "apple/contacts.delete", "apple/health.workouts.list", "apple/alarms.create", "apple/reminders.list", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/photos.search", "apple/calendar.events.delete", "apple/music.playback", "apple/reminders.create", "builtin/EditFile", "apple/maps.show", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/music.search", "apple/weather.hourly", "apple/weather.daily", "apple/calendar.list", "apple/clipboard.read", "apple/contacts.create", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/location.get", "apple/maps.eta", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.get", "apple/calendar.events.list", "apple/photos.albums.list" ], "scores": [ 23.418124448414066, 12.261078227681264, 7.7622905376693065, 6.9262043446524775, 6.899727192226153, 6.366749206601937, 3.80684568833329, 3.325600141569027, 3.1665046656933473, 3.113452191752697, 3.037362463710464, 3.030342954605759, 2.980371824857474, 2.8891502849647948, 2.7066072911480274, 2.620554873679487, 2.552190972780245, 2.5237801294602042, 2.497591979738032, 2.451233465604745, 2.33260163294754, 2.279087933793983, 2.172491827747787, 2.1665432842336214, 1.7618025795499144, 1.3503569460646248, 1.2442452235478334, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/shortcuts.run", "builtin/WriteFile", "apple/clipboard.write", "apple/links.open", "apple/health.statistics", "apple/calendar.events.update", "apple/calendar.events.create", "apple/alarms.create", "apple/reminders.create", "apple/reminders.update", "apple/location.reverseGeocode", "apple/contacts.create", "apple/location.geocode", "apple/contacts.get", "apple/battery.get", "apple/music.status", "builtin/Memory", "apple/maps.search", "apple/health.workouts.get", "apple/contacts.delete", "apple/health.workouts.list", "apple/reminders.list", "builtin/ScheduledTasks", "apple/health.sleep.query", "apple/photos.search", "apple/calendar.events.delete", "apple/music.playback", "builtin/EditFile", "apple/maps.show", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/music.search", "apple/weather.hourly", "apple/weather.daily", "apple/calendar.list", "apple/clipboard.read", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/location.get", "apple/maps.eta", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.lists", "apple/calendar.events.get", "apple/calendar.events.list", "apple/photos.albums.list" ], "scores": [ 23.418124448414066, 13.981964832076965, 12.261078227681264, 8.50180892688808, 6.9262043446524775, 5.796340009815642, 5.691344372305615, 5.400221568239177, 3.9395139380111575, 3.80684568833329, 3.325600141569027, 3.2437712624084325, 3.113452191752697, 3.037362463710464, 2.980371824857474, 2.8891502849647948, 2.746249914790049, 2.7066072911480274, 2.620554873679487, 2.552190972780245, 2.5237801294602042, 2.451233465604745, 2.33260163294754, 2.279087933793983, 2.172491827747787, 2.1665432842336214, 1.7618025795499144, 0.8460531825275042, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } } } }, { "query_id": "H053", "query": "Crop the actual pixels of an image in my photo library.", "cohort": "unsupported", "domain": "photos", "category": "unsupported", "intent": "unsupported:photos", "acceptable": [], "rationale": "Photo tools return metadata, not pixel editing.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/photos.albums.list", "apple/photos.get", "apple/photos.search", "apple/links.open", "apple/alarms.list", "apple/clipboard.read", "apple/contacts.get", "apple/location.get", "apple/music.search", "apple/reminders.complete", "apple/reminders.update" ], "scores": [ 6, 5.5, 5.5, 2, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/photos.albums.list", "apple/photos.get", "apple/photos.search", "apple/reminders.complete", "apple/contacts.get", "apple/reminders.update", "apple/music.search", "apple/location.get", "apple/clipboard.read", "apple/alarms.list" ], "scores": [ 8.916301297207283, 6.931552832697999, 4.612209023906652, 3.4732645251781626, 3.133344695217922, 3.048112898315442, 2.776792493451411, 2.7156932419630486, 2.4221849112268674, 2.3962861849916948 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.get", "apple/location.get", "apple/music.search", "apple/clipboard.read", "apple/alarms.list", "apple/links.open" ], "scores": [ 14.505360056824461, 13.693604438727558, 12.621460711060156, 3.5478243783800933, 3.0806466293353036, 2.9209278227470183, 2.7221886479398227, 2.454140563572183, 2.410538121601221, 2.383253378845514, 1.1237743813923031 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/reminders.complete", "apple/reminders.update", "apple/contacts.get", "apple/location.get", "apple/music.search", "apple/clipboard.read", "apple/alarms.list", "apple/links.open" ], "scores": [ 14.505360056824461, 13.693604438727558, 12.621460711060156, 3.5478243783800933, 3.0806466293353036, 2.9209278227470183, 2.7221886479398227, 2.454140563572183, 2.410538121601221, 2.383253378845514, 1.7843771184199018 ], "metrics": { "false_recall": true } } } }, { "query_id": "H054", "query": "Create a playlist in my Apple Music library.", "cohort": "unsupported", "domain": "music", "category": "unsupported", "intent": "unsupported:music", "acceptable": [], "rationale": "Catalog/player tools do not create library playlists.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/music.search", "apple/alarms.create", "apple/links.open", "apple/music.playback", "apple/music.status", "apple/calendar.events.create", "apple/contacts.create", "apple/reminders.create", "apple/shortcuts.run", "apple/alarms.cancel", "apple/alarms.list", "apple/maps.route", "apple/maps.search", "apple/maps.show", "apple/photos.albums.list", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "apple/photos.search", "apple/battery.get", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.get", "apple/location.reverseGeocode", "apple/maps.eta", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "builtin/ScheduledTasks", "builtin/Memory", "builtin/WriteFile" ], "scores": [ 9, 8, 8, 8, 8, 7, 7, 7, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3.5, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2.5, 1.5, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/music.search", "apple/music.status", "apple/music.playback", "apple/alarms.create", "apple/photos.albums.list", "apple/shortcuts.run", "apple/links.open", "builtin/ScheduledTasks", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "builtin/WriteFile", "builtin/Memory", "apple/maps.show", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/weather.hourly", "apple/weather.daily" ], "scores": [ 8.290566206862781, 6.802464308650038, 4.652314829197242, 4.512580623008715, 3.761822691870858, 3.4517436328367657, 3.442277567091643, 3.222743087557137, 3.05666003761185, 3.030740607742593, 2.9403271227683874, 2.8131576240574634, 2.1142385633003573, 2.101280785461998, 1.9552868225447846, 1.5045297512928304, 1.5045297512928304, 1.488613054842949, 1.41382735889783, 1.3334393574442729, 1.2394733488050822 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/music.status", "apple/music.search", "apple/music.playback", "apple/alarms.create", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/links.open", "apple/photos.albums.list", "builtin/ScheduledTasks", "builtin/Memory", "builtin/WriteFile", "apple/shortcuts.run", "apple/photos.search", "apple/maps.show", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/weather.hourly", "apple/weather.daily", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode" ], "scores": [ 15.306612685862202, 15.142765250817362, 12.043938200335763, 10.411890902985961, 9.049626368216657, 9.017100022924438, 8.438698097842202, 5.4330080963263025, 3.796273661253868, 3.4167632220625808, 3.0418207860050117, 2.8774990349848864, 2.6835360690060837, 1.50741032852536, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/music.status", "apple/music.search", "apple/music.playback", "apple/alarms.create", "apple/reminders.create", "apple/contacts.create", "apple/calendar.events.create", "apple/links.open", "builtin/WriteFile", "apple/photos.albums.list", "builtin/ScheduledTasks", "builtin/Memory", "apple/shortcuts.run", "apple/photos.search", "apple/maps.show", "apple/weather.current", "apple/alarms.list", "apple/maps.route", "apple/alarms.cancel", "apple/maps.search", "apple/weather.hourly", "apple/weather.daily", "apple/battery.get", "apple/calendar.list", "apple/clipboard.read", "apple/clipboard.write", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.list", "apple/contacts.update", "apple/device.get", "apple/health.statistics", "apple/location.geocode", "apple/location.get", "apple/maps.eta", "apple/photos.get", "apple/reminders.complete", "apple/reminders.delete", "apple/reminders.get", "apple/reminders.list", "apple/reminders.lists", "apple/reminders.update", "apple/calendar.events.delete", "apple/calendar.events.get", "apple/calendar.events.list", "apple/calendar.events.update", "apple/health.sleep.query", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode" ], "scores": [ 15.306612685862202, 15.142765250817362, 12.043938200335763, 10.411890902985961, 9.049626368216657, 9.017100022924438, 8.438698097842202, 6.140473524781067, 5.715162977290321, 3.796273661253868, 3.4167632220625808, 3.0418207860050117, 2.6835360690060837, 1.50741032852536, 0.7245124831044734, 0.7085420744235738, 0.6624346778200838, 0.6624346778200838, 0.6606930487736644, 0.6525386954849385, 0.6438261189935433, 0.6337103983466377, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.5068245205756526, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184, 0.44804332617899184 ], "metrics": { "false_recall": true } } } }, { "query_id": "H055", "query": "Merge duplicate contacts into one atomically.", "cohort": "unsupported", "domain": "contacts", "category": "unsupported", "intent": "unsupported:contacts", "acceptable": [], "rationale": "No contact merge/transaction tool.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/contacts.create", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.get", "apple/contacts.list", "apple/calendar.events.create", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.delete", "apple/calendar.events.update", "apple/clipboard.write", "apple/health.workouts.get", "apple/location.get", "apple/maps.route", "apple/music.playback", "apple/photos.get", "apple/reminders.delete", "builtin/EditFile", "builtin/ListFiles", "builtin/Memory", "builtin/ReadFile", "builtin/ScheduledTasks" ], "scores": [ 7, 5.5, 5.5, 4.5, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.5, 0.5 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/contacts.create", "apple/contacts.delete", "apple/contacts.update", "apple/contacts.get", "apple/calendar.events.create", "apple/contacts.list", "builtin/ListFiles", "builtin/EditFile", "apple/reminders.delete", "apple/health.workouts.get", "apple/calendar.events.update", "apple/location.get", "builtin/Memory", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/alarms.create", "apple/music.playback" ], "scores": [ 13.293332730941948, 4.960139019172261, 4.51684744765666, 4.062325774173696, 3.691816622635003, 3.5908759744630157, 1.6822078764459198, 1.5024992193814797, 1.4160558529896354, 1.3959771463946806, 1.340122705366102, 1.1929880582775596, 1.164760090359424, 1.1248369805109821, 1.0997080487563646, 1.0526744179854908, 1.0415379820610877, 0.9892126698597037, 0.9602672350864689, 0.7720168266239731 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/contacts.list", "apple/calendar.events.create", "builtin/ListFiles", "builtin/EditFile", "apple/health.workouts.get", "apple/reminders.delete", "apple/calendar.events.update", "apple/location.get", "builtin/Memory", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/alarms.create", "builtin/ScheduledTasks", "apple/music.playback", "builtin/ReadFile" ], "scores": [ 19.255585861620656, 12.435680775757913, 11.963738092095124, 11.45273695558456, 9.52373522475088, 3.6470825885392095, 1.651417995754508, 1.4598619112844569, 1.3691132429326038, 1.3691132429326038, 1.2739620523469748, 1.1390026644524067, 1.0964747816023659, 1.0829958778625002, 1.0444767918636064, 0.9971873002352808, 0.9860265225742033, 0.9337714661875144, 0.904995019691811, 0.8652613463947955, 0.720071910007749, 0.5185160798440496 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/contacts.update", "apple/contacts.list", "apple/calendar.events.create", "builtin/ListFiles", "builtin/EditFile", "apple/health.workouts.get", "apple/reminders.delete", "apple/calendar.events.update", "apple/location.get", "builtin/Memory", "apple/calendar.events.delete", "apple/clipboard.write", "apple/maps.route", "apple/alarms.cancel", "apple/photos.get", "apple/alarms.create", "builtin/ScheduledTasks", "apple/music.playback", "builtin/ReadFile" ], "scores": [ 19.255585861620656, 12.435680775757913, 11.963738092095124, 11.45273695558456, 9.52373522475088, 3.6470825885392095, 1.651417995754508, 1.4598619112844569, 1.3691132429326038, 1.3691132429326038, 1.2739620523469748, 1.1390026644524067, 1.0964747816023659, 1.0829958778625002, 1.0444767918636064, 0.9971873002352808, 0.9860265225742033, 0.9337714661875144, 0.904995019691811, 0.8652613463947955, 0.720071910007749, 0.5185160798440496 ], "metrics": { "false_recall": true } } } }, { "query_id": "H056", "query": "Delete recorded samples from HealthKit.", "cohort": "unsupported", "domain": "health", "category": "unsupported", "intent": "unsupported:health", "acceptable": [], "rationale": "Health tools are read-only.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/health.statistics", "apple/health.workouts.get", "apple/calendar.events.delete", "apple/contacts.delete", "apple/health.workouts.list", "apple/reminders.delete", "apple/health.sleep.query", "builtin/ScheduledTasks", "builtin/Memory", "apple/alarms.create", "apple/alarms.list", "apple/location.get", "apple/maps.search", "apple/photos.albums.list", "apple/photos.get", "apple/weather.current" ], "scores": [ 6, 4.5, 4, 4, 4, 4, 3, 3, 1.5, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/health.workouts.get", "apple/health.workouts.list", "apple/health.statistics", "apple/reminders.delete", "apple/contacts.delete", "apple/location.get", "apple/calendar.events.delete", "builtin/ScheduledTasks", "apple/alarms.list", "apple/health.sleep.query", "builtin/Memory", "apple/maps.search", "apple/photos.get", "apple/alarms.create", "apple/weather.current", "apple/photos.albums.list" ], "scores": [ 6.843461487360058, 6.564232463902723, 6.1558051469255535, 3.752882783565409, 3.6211209622787783, 3.348804949568758, 3.224841952905041, 3.039850038541701, 2.6686700803525736, 2.499548551624379, 2.4404053139066892, 1.7955900028110616, 1.7955900028110616, 1.7430490933692009, 1.726212147301921, 1.6775979800585423 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "apple/health.sleep.query", "builtin/ScheduledTasks", "apple/location.get", "builtin/Memory", "apple/alarms.list", "apple/maps.search", "apple/photos.get", "apple/alarms.create", "apple/weather.current", "apple/photos.albums.list" ], "scores": [ 12.350559040948626, 11.991580919202367, 10.171116368037218, 10.156196128710354, 9.97993715650646, 8.698195352050329, 6.315869587661187, 4.639135584460305, 3.3568146346643974, 3.1123028295377932, 2.6585178698196947, 1.779536789183193, 1.779536789183193, 1.724696020263421, 1.7071592606801997, 1.6731343036002873 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/reminders.delete", "apple/contacts.delete", "apple/calendar.events.delete", "apple/health.sleep.query", "builtin/ScheduledTasks", "apple/location.get", "builtin/Memory", "apple/alarms.list", "apple/maps.search", "apple/photos.get", "apple/alarms.create", "apple/weather.current", "apple/photos.albums.list" ], "scores": [ 12.350559040948626, 11.991580919202367, 10.171116368037218, 10.156196128710354, 9.97993715650646, 8.698195352050329, 6.315869587661187, 4.639135584460305, 3.3568146346643974, 3.1123028295377932, 2.6585178698196947, 1.779536789183193, 1.779536789183193, 1.724696020263421, 1.7071592606801997, 1.6731343036002873 ], "metrics": { "false_recall": true } } } }, { "query_id": "H057", "query": "Install an App Store application automatically.", "cohort": "unsupported", "domain": "apps", "category": "unsupported", "intent": "unsupported:apps", "acceptable": [], "rationale": "No application-install tool.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/links.open", "apple/music.status", "apple/alarms.create", "apple/shortcuts.run", "builtin/ScheduledTasks" ], "scores": [ 4, 2, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/music.status", "apple/links.open", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 6.102768339235191, 5.484919563059419, 2.0519806739118946, 2.01195188835184, 1.1554436837598263 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/links.open", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 6.81878799462161, 6.09849214263024, 2.0325256171929906, 1.9907674591623792, 1.117963704064442 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/links.open", "apple/music.status", "apple/shortcuts.run", "apple/alarms.create", "builtin/ScheduledTasks" ], "scores": [ 7.479390731649208, 6.09849214263024, 2.0325256171929906, 1.9907674591623792, 1.117963704064442 ], "metrics": { "false_recall": true } } } }, { "query_id": "H058", "query": "Read my saved passwords from the system Keychain.", "cohort": "unsupported", "domain": "security", "category": "unsupported", "intent": "unsupported:security", "acceptable": [], "rationale": "No password or Keychain-read capability.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/clipboard.read", "builtin/ReadFile", "builtin/ScheduledTasks", "apple/device.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/shortcuts.run", "apple/alarms.list", "apple/battery.get", "apple/calendar.events.create", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/calendar.events.update", "apple/contacts.create", "apple/contacts.delete", "apple/contacts.get", "apple/links.open", "apple/maps.eta", "apple/music.playback", "apple/music.status", "apple/photos.get", "apple/photos.search", "apple/weather.current", "apple/weather.daily", "apple/weather.hourly", "builtin/EditFile", "builtin/Memory" ], "scores": [ 5, 4, 2.5, 2, 2, 2, 2, 2, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/device.get", "apple/clipboard.read", "apple/health.workouts.get", "apple/health.workouts.list", "apple/health.sleep.query", "apple/contacts.create", "apple/music.status", "builtin/ScheduledTasks", "apple/contacts.delete", "apple/health.statistics", "builtin/ReadFile", "apple/calendar.events.delete", "builtin/Memory", "apple/contacts.get", "apple/shortcuts.run", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/EditFile", "apple/weather.current", "apple/links.open", "apple/music.playback", "apple/calendar.events.list", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search" ], "scores": [ 4.196836460602601, 3.412887254507381, 3.376446361476602, 3.238679557058612, 2.8854771720721715, 2.4692163806821252, 2.2123786740293387, 2.1056101221828856, 2.0237659864584026, 1.9122605853203791, 1.7356930089104854, 1.7000568511749237, 1.6208496848238496, 1.596129569605437, 1.4802042419042116, 1.4656245762044289, 1.3968322222103842, 1.3632427748418257, 1.2412598610988734, 1.2152122937951468, 1.1668112962717396, 1.0956951902365162, 1.0694589285764813, 1.0694589285764813, 0.9551091781400716, 0.8975292682836623, 0.8464971647744733, 0.7868454382417407, 0.7350474507853814 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/clipboard.read", "builtin/ReadFile", "apple/device.get", "apple/health.workouts.get", "apple/health.workouts.list", "apple/health.sleep.query", "apple/contacts.create", "apple/music.status", "builtin/ScheduledTasks", "apple/shortcuts.run", "apple/contacts.delete", "apple/health.statistics", "apple/calendar.events.delete", "builtin/Memory", "apple/contacts.get", "apple/calendar.events.create", "apple/calendar.events.update", "builtin/EditFile", "apple/weather.current", "apple/links.open", "apple/music.playback", "apple/calendar.events.list", "apple/battery.get", "apple/maps.eta", "apple/alarms.list", "apple/photos.get", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search" ], "scores": [ 6.113639216088124, 4.557289964100901, 4.204422201447571, 3.476720521761889, 3.3218496583123773, 2.9302630731149364, 2.485854735922362, 2.2108284650669687, 2.1865440235143883, 2.1464029527033963, 2.0453664522045925, 1.8745112659606904, 1.7184999580546296, 1.6111411824454678, 1.60914410629741, 1.4661702610544265, 1.393115594519765, 1.3906556055226948, 1.231374918430577, 1.1804942158124918, 1.1426115024435417, 1.113807122507814, 1.0711583788400274, 1.0711583788400274, 0.9499145762410579, 0.8895050372184048, 0.8363195411534453, 0.7745686203791382, 0.721309630646165 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/clipboard.read", "apple/device.get", "builtin/ReadFile", "apple/health.workouts.get", "apple/health.workouts.list", "apple/health.sleep.query", "apple/contacts.create", "apple/music.status", "apple/shortcuts.run", "apple/contacts.get", "apple/contacts.delete", "builtin/ScheduledTasks", "apple/battery.get", "apple/calendar.events.delete", "apple/calendar.events.list", "apple/health.statistics", "apple/alarms.list", "apple/photos.get", "apple/calendar.events.create", "apple/calendar.events.update", "apple/links.open", "apple/music.playback", "builtin/Memory", "builtin/EditFile", "apple/calendar.events.get", "apple/location.get", "apple/reminders.get", "apple/contacts.list", "apple/photos.albums.list", "apple/calendar.list", "apple/reminders.list", "apple/weather.current", "apple/maps.eta", "apple/weather.hourly", "apple/weather.daily", "apple/photos.search" ], "scores": [ 4.928372887587887, 4.919680510022155, 4.201778249894578, 3.9989716706293166, 3.79810494202916, 2.608938530414968, 2.485854735922362, 2.2108284650669687, 2.1464029527033963, 2.1425381623194113, 2.0453664522045925, 1.8506498881946607, 1.7904649384226818, 1.7184999580546296, 1.6917167140453433, 1.6689575459390709, 1.648914120016547, 1.6332646411623757, 1.4661702610544265, 1.393115594519765, 1.1804942158124918, 1.1426115024435417, 1.1401709250099572, 0.9841378926907047, 0.9766794376224938, 0.9766794376224938, 0.9510032826687982, 0.9266425589614367, 0.9266425589614367, 0.9034987037373633, 0.8814827611189724, 0.8714182810782765, 0.7580363862218745, 0.5918458513942811, 0.5481460160064278, 0.5104557426457969 ], "metrics": { "false_recall": true } } } }, { "query_id": "H059", "query": "Export the full-resolution photo binary using only these tools.", "cohort": "unsupported", "domain": "photos", "category": "unsupported", "intent": "unsupported:photos", "acceptable": [], "rationale": "Photo tools expose metadata, not image binary export.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/links.open", "apple/music.playback", "builtin/ScheduledTasks", "apple/alarms.cancel", "apple/alarms.create", "apple/alarms.list", "apple/calendar.events.update", "apple/contacts.get", "apple/contacts.list", "apple/health.statistics", "apple/maps.show", "apple/music.search", "apple/music.status", "apple/shortcuts.run", "builtin/AskUserQuestions", "builtin/Memory" ], "scores": [ 5.5, 5, 4.5, 4, 1.5, 1.5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/links.open", "apple/photos.albums.list", "builtin/AskUserQuestions", "apple/photos.get", "apple/contacts.list", "apple/photos.search", "apple/contacts.get", "apple/health.statistics", "apple/music.status", "apple/music.search", "apple/shortcuts.run", "builtin/ScheduledTasks", "apple/alarms.list", "apple/maps.show", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.update", "apple/music.playback" ], "scores": [ 4.990505175014953, 4.6855218686035975, 4.496259069511381, 4.410034562998602, 3.5156165905658834, 3.327619538120083, 3.133344695217922, 2.583955380200963, 1.892501474377823, 1.7166803960897803, 1.704499249357205, 1.3894118676695326, 1.2842193286009138, 1.2842193286009138, 1.2706333365586515, 1.1714863806419569, 1.1274973225408556, 0.9418286545359099 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/links.open", "builtin/AskUserQuestions", "apple/contacts.list", "apple/contacts.get", "apple/health.statistics", "apple/music.status", "apple/music.playback", "apple/music.search", "apple/shortcuts.run", "builtin/ScheduledTasks", "builtin/Memory", "apple/alarms.list", "apple/maps.show", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.update" ], "scores": [ 11.932491159521803, 9.79832129670893, 9.372310851658964, 5.945925645198203, 4.542870945438329, 3.50097963484709, 2.9209278227470183, 2.5329463505701764, 1.798233682123699, 1.621653562440418, 1.62154593790355, 1.609381947431756, 1.606743874083366, 1.4800810540091471, 1.2141942578109772, 1.2141942578109772, 1.2006046822662546, 1.1019391803305796, 1.0689954759741835 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/photos.get", "apple/photos.albums.list", "apple/photos.search", "apple/links.open", "builtin/AskUserQuestions", "apple/contacts.list", "apple/contacts.get", "apple/health.statistics", "apple/music.status", "apple/music.playback", "apple/music.search", "apple/shortcuts.run", "builtin/ScheduledTasks", "builtin/Memory", "apple/alarms.list", "apple/maps.show", "apple/alarms.cancel", "apple/alarms.create", "apple/calendar.events.update" ], "scores": [ 11.932491159521803, 9.79832129670893, 9.372310851658964, 6.606528382225802, 4.542870945438329, 3.50097963484709, 2.9209278227470183, 2.5329463505701764, 1.798233682123699, 1.621653562440418, 1.62154593790355, 1.609381947431756, 1.606743874083366, 1.4800810540091471, 1.2141942578109772, 1.2141942578109772, 1.2006046822662546, 1.1019391803305796, 1.0689954759741835 ], "metrics": { "false_recall": true } } } }, { "query_id": "H060", "query": "Send a WhatsApp message without a configured shortcut or UI controller.", "cohort": "unsupported", "domain": "messages", "category": "unsupported", "intent": "unsupported:messages", "acceptable": [], "rationale": "No message-send capability; opening an app is insufficient.", "basis": [ "frozen catalog declarations: no matching executable capability; unknown shortcuts are not assumed" ], "known_diagnostic": false, "split": "comparison", "methods": { "keyword": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/alarms.create", "apple/battery.get", "apple/health.sleep.query", "apple/health.statistics", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.geocode", "apple/location.reverseGeocode", "apple/maps.search", "apple/music.status", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 6, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ], "metrics": { "false_recall": true } }, "name_description": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/health.sleep.query", "apple/location.geocode", "apple/battery.get", "apple/music.status", "apple/maps.search", "apple/alarms.create", "apple/health.statistics", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 7.856463532351667, 3.442277567091643, 3.1777733868287177, 3.048112898315442, 2.8055106012874256, 2.7156932419630486, 2.5938411871930867, 2.4696228887134084, 2.38399622539939, 2.072598362607447, 2.01195188835184, 1.7997415466286588, 1.6973910453637415, 1.1554436837598263 ], "metrics": { "false_recall": true } }, "bm25": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/health.sleep.query", "apple/location.geocode", "apple/battery.get", "apple/music.status", "apple/maps.search", "apple/alarms.create", "apple/health.statistics", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 22.09167870161456, 4.834503377416997, 3.2721473302672357, 3.1263892000981492, 2.877556815390035, 2.757843902510372, 2.606627671177044, 2.4735473042818215, 2.382325764389142, 2.054068595663089, 1.9907674591623792, 1.7642134293155032, 1.6656673071721346, 1.117963704064442 ], "metrics": { "false_recall": true } }, "combined": { "ranking": [ "apple/shortcuts.run", "apple/links.open", "apple/health.workouts.get", "apple/health.workouts.list", "apple/location.reverseGeocode", "apple/health.sleep.query", "apple/location.geocode", "apple/battery.get", "apple/music.status", "apple/maps.search", "apple/alarms.create", "apple/health.statistics", "apple/photos.search", "builtin/ScheduledTasks" ], "scores": [ 22.09167870161456, 5.495106114444595, 3.2721473302672357, 3.1263892000981492, 2.877556815390035, 2.757843902510372, 2.606627671177044, 2.4735473042818215, 2.382325764389142, 2.054068595663089, 1.9907674591623792, 1.7642134293155032, 1.6656673071721346, 1.117963704064442 ], "metrics": { "false_recall": true } } } } ], "dev": { "keyword": { "positive_n": 90, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.6111111111111112, "hit3": 0.7333333333333333, "hit5": 0.7777777777777778, "rr": 0.6761111111111111 }, "capability_groups": 36, "capability_group_mean": { "hit1": 0.6847222222222222, "hit3": 0.8447751322751322, "hit5": 0.8950396825396825, "rr": 0.7672398589065256 } }, "name_description": { "positive_n": 90, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.5666666666666667, "hit3": 0.6333333333333333, "hit5": 0.6888888888888889, "rr": 0.6152197802197802 }, "capability_groups": 36, "capability_group_mean": { "hit1": 0.6189814814814815, "hit3": 0.7158068783068783, "hit5": 0.8037698412698413, "rr": 0.6862684603756032 } }, "bm25": { "positive_n": 90, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.6, "hit3": 0.7333333333333333, "hit5": 0.7666666666666667, "rr": 0.6698989898989899 }, "capability_groups": 36, "capability_group_mean": { "hit1": 0.6729497354497354, "hit3": 0.8447751322751322, "hit5": 0.8672619047619048, "rr": 0.7598334535834536 } }, "combined": { "positive_n": 90, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.7888888888888889, "hit3": 0.9111111111111111, "hit5": 0.9777777777777777, "rr": 0.8594444444444443 }, "capability_groups": 36, "capability_group_mean": { "hit1": 0.8092592592592592, "hit3": 0.9362433862433862, "hit5": 0.9944444444444444, "rr": 0.8792989417989419 } } }, "comparison": { "keyword": { "positive_n": 50, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.46, "hit3": 0.58, "hit5": 0.7, "rr": 0.5591904761904761 }, "capability_groups": 27, "capability_group_mean": { "hit1": 0.5012345679012346, "hit3": 0.6246913580246913, "hit5": 0.754320987654321, "rr": 0.6036155202821869 } }, "name_description": { "positive_n": 50, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.42, "hit3": 0.5, "hit5": 0.58, "rr": 0.478 }, "capability_groups": 27, "capability_group_mean": { "hit1": 0.4395061728395062, "hit3": 0.519753086419753, "hit5": 0.6308641975308643, "rr": 0.5046296296296297 } }, "bm25": { "positive_n": 50, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.48, "hit3": 0.7, "hit5": 0.74, "rr": 0.5948333333333333 }, "capability_groups": 27, "capability_group_mean": { "hit1": 0.5382716049382716, "hit3": 0.754320987654321, "hit5": 0.8037037037037037, "rr": 0.6501028806584362 } }, "combined": { "positive_n": 50, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": 0.68, "hit3": 0.84, "hit5": 0.9, "rr": 0.7706666666666666 }, "capability_groups": 27, "capability_group_mean": { "hit1": 0.7123456790123457, "hit3": 0.8506172839506172, "hit5": 0.8938271604938272, "rr": 0.7882716049382716 } } }, "cohorts": { "intent_transfer": { "keyword": { "positive_n": 30, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.7, "hit3": 0.8333333333333334, "hit5": 0.9333333333333333, "rr": 0.7938888888888889 }, "capability_groups": 19, "capability_group_mean": { "hit1": 0.6842105263157895, "hit3": 0.7894736842105263, "hit5": 0.9210526315789473, "rr": 0.7745614035087719 } }, "name_description": { "positive_n": 30, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.5666666666666667, "hit3": 0.6333333333333333, "hit5": 0.7666666666666667, "rr": 0.63 }, "capability_groups": 19, "capability_group_mean": { "hit1": 0.5263157894736842, "hit3": 0.6052631578947368, "hit5": 0.7631578947368421, "rr": 0.6013157894736842 } }, "bm25": { "positive_n": 30, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.7, "hit3": 0.9333333333333333, "hit5": 0.9666666666666667, "rr": 0.8177777777777778 }, "capability_groups": 19, "capability_group_mean": { "hit1": 0.6842105263157895, "hit3": 0.9210526315789473, "hit5": 0.9736842105263158, "rr": 0.8043859649122806 } }, "combined": { "positive_n": 30, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.8, "hit3": 0.9666666666666667, "hit5": 1.0, "rr": 0.8844444444444445 }, "capability_groups": 19, "capability_group_mean": { "hit1": 0.8157894736842105, "hit3": 0.9736842105263158, "hit5": 1.0, "rr": 0.8956140350877192 } } }, "unsupported": { "keyword": { "positive_n": 0, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": null, "hit3": null, "hit5": null, "rr": null }, "capability_groups": 0, "capability_group_mean": { "hit1": null, "hit3": null, "hit5": null, "rr": null } }, "name_description": { "positive_n": 0, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": null, "hit3": null, "hit5": null, "rr": null }, "capability_groups": 0, "capability_group_mean": { "hit1": null, "hit3": null, "hit5": null, "rr": null } }, "bm25": { "positive_n": 0, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": null, "hit3": null, "hit5": null, "rr": null }, "capability_groups": 0, "capability_group_mean": { "hit1": null, "hit3": null, "hit5": null, "rr": null } }, "combined": { "positive_n": 0, "negative_n": 10, "negative_false_recall": 10, "bm25": { "hit1": null, "hit3": null, "hit5": null, "rr": null }, "capability_groups": 0, "capability_group_mean": { "hit1": null, "hit3": null, "hit5": null, "rr": null } } }, "wording_transfer": { "keyword": { "positive_n": 20, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.1, "hit3": 0.2, "hit5": 0.35, "rr": 0.20714285714285716 }, "capability_groups": 8, "capability_group_mean": { "hit1": 0.06666666666666667, "hit3": 0.23333333333333334, "hit5": 0.35833333333333334, "rr": 0.1976190476190476 } }, "name_description": { "positive_n": 20, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.2, "hit3": 0.3, "hit5": 0.3, "rr": 0.25 }, "capability_groups": 8, "capability_group_mean": { "hit1": 0.23333333333333334, "hit3": 0.31666666666666665, "hit5": 0.31666666666666665, "rr": 0.275 } }, "bm25": { "positive_n": 20, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.15, "hit3": 0.35, "hit5": 0.4, "rr": 0.26041666666666663 }, "capability_groups": 8, "capability_group_mean": { "hit1": 0.19166666666666665, "hit3": 0.35833333333333334, "hit5": 0.39999999999999997, "rr": 0.28368055555555555 } }, "combined": { "positive_n": 20, "negative_n": 0, "negative_false_recall": 0, "bm25": { "hit1": 0.5, "hit3": 0.65, "hit5": 0.75, "rr": 0.6 }, "capability_groups": 8, "capability_group_mean": { "hit1": 0.4666666666666667, "hit3": 0.5583333333333333, "hit5": 0.6416666666666666, "rr": 0.5333333333333333 } } } }, "provenance": { "base": "c58660846b6551c8ba40dfb645c515d30b4c6f90", "freeze_sha256": "73bb777ba8965213fc581d7cae5761a95905d91fb8dfba9071c774ea3f1159ef", "raw_sha256": "bc648db88170e616d844adb62764c8e3bed5afeda779799eb166c8db3eb2673c", "reuse_source_sha256": "c74cef65b2cb4733a815c3c6e38abd4fa5979712c76623587daf58ab37849847", "status": "fixed revealed comparison, not new blind test" } }