From 2ac6a4c53d0d7efd30daa051c1756b6bb15537e7 Mon Sep 17 00:00:00 2001 From: dyabel Date: Sat, 2 Mar 2024 16:25:20 +0000 Subject: [PATCH] clean --- README.md | 3 +- anytool/prompt_template.py | 5 +- misc/unsolvable.json | 1624 +++++-------------- preprocess/extract_category_tool_details.py | 2 - rapidapi_key_list_example.json | 2 +- scripts/main.py | 16 +- 6 files changed, 426 insertions(+), 1226 deletions(-) diff --git a/README.md b/README.md index b218355..2742ad3 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,7 @@ Fill out the [form](https://docs.google.com/forms/d/e/1FAIpQLSdqHypmYanWU8ZhuUcr **ToolBench** Download the ToolBench data using the following link: [Google Drive](https://drive.google.com/drive/folders/1yBUQ732mPu-KclJnuQELEhtKakdXFc3J) or [Tsinghua Cloud](https://cloud.tsinghua.edu.cn/f/c9e50625743b40bfbe10/). - -The file structure is as follows: +Decompress the data.zip and the file structure is as follows: ``` ├── /data/ │ ├── /instruction/ diff --git a/anytool/prompt_template.py b/anytool/prompt_template.py index 63b83c2..b17dd7f 100644 --- a/anytool/prompt_template.py +++ b/anytool/prompt_template.py @@ -306,9 +306,8 @@ Please check whether the given task solvable with following rules: 1. If the `query` provide invalid information (e.g. invalid email address or phone number), return "Unsolvable" 2. If the `query` needs more information to solve (e.g. the target restaurant name in a navigation task), return "Unsolvable" 3. If you are unable to draw a conclusion, return "Unsure" -5. Otherwise, return "Solvable" +4. Otherwise, return "Solvable" Remember, you should assume you have all the tools to solve the query but you do not need to answer the query at this time. You must call the Finish function at one step. -""" -# 4. If the query is illegal or unethical or sensitive, return "Unsure" \ No newline at end of file +""" \ No newline at end of file diff --git a/misc/unsolvable.json b/misc/unsolvable.json index 36421e3..0e425da 100644 --- a/misc/unsolvable.json +++ b/misc/unsolvable.json @@ -1,49 +1,107 @@ [ 29, + 43, + 52, + 55, + 75, + 83, + 455, + 456, + 457, + 459, + 460, 1236, - 3995, - 4117, - 4144, - 4153, - 4155, - 4179, - 4266, - 4273, - 4286, - 4366, - 6504, - 6511, - 9718, - 9719, - 12642, - 18268, - 23444, - 26698, - 26701, - 29592, - 29844, - 29917, - 38164, - 84839, - 59890, - 63190, - 67087, - 68553, - 71583, - 82434, - 84935, + 1617, + 1844, 1983, 1984, 1985, 1986, 1989, 1991, + 2144, + 2213, + 2399, + 2412, + 2433, + 2701, + 3323, + 3432, + 3482, + 3494, + 3510, + 3534, + 3645, + 3672, + 3770, + 3786, + 3794, + 3910, + 3922, + 3929, + 3990, + 3995, + 4006, + 4031, + 4095, + 4117, + 4144, + 4153, + 4155, + 4176, + 4179, + 4242, + 4266, + 4271, + 4273, + 4279, + 4280, + 4282, + 4283, + 4285, + 4286, + 4288, + 4366, + 4505, + 5695, + 6148, + 6504, + 6511, + 6677, + 6736, + 6959, + 7257, + 7468, + 7497, + 7903, + 7971, + 7989, + 8025, + 8129, + 9332, + 9335, + 9336, + 9337, + 9339, + 9340, 9341, 9343, + 9344, 9345, 9346, 9349, + 9718, + 9719, + 9792, + 9834, + 9921, + 9957, + 9959, + 9984, + 10160, + 10221, 10898, + 11448, 11644, 11645, 11646, @@ -51,166 +109,19 @@ 11648, 11649, 11650, - 14485, - 14489, - 14939, - 14947, - 18982, - 20022, - 20023, - 20024, - 20026, - 20027, - 20028, - 20029, - 20030, - 455, - 456, - 457, - 459, - 460, - 4279, - 4280, - 4282, - 4283, - 4285, - 4286, - 9332, - 9335, - 9336, - 9337, - 9339, - 9340, + 12034, 12094, 12095, - 14938, - 14939, - 14947, - 14948, - 14950, - 23236, - 23239, - 23240, - 2399, - 6148, - 6736, - 7989, - 8025, - 9921, + 12296, + 12471, + 12507, + 12634, + 12642, + 12742, + 12768, + 12773, + 12894, 13095, - 15511, - 23163, - 32617, - 32652, - 32807, - 33330, - 33889, - 34282, - 34823, - 35112, - 36068, - 40436, - 41142, - 41806, - 42351, - 45775, - 47301, - 49991, - 52534, - 52734, - 55721, - 56226, - 56514, - 58096, - 59266, - 60255, - 62568, - 62757, - 63567, - 67007, - 71402, - 74098, - 74322, - 77471, - 77514, - 82701, - 83742, - 83819, - 21464, - 23486, - 16184, - 13780, - 25125, - 1617, - 5695, - 6677, - 7903, - 7971, - 8129, - 9792, - 10221, - 14198, - 15058, - 15335, - 16700, - 16899, - 17978, - 18761, - 20358, - 24313, - 24777, - 25152, - 25164, - 25365, - 26542, - 32177, - 34696, - 35382, - 38414, - 48950, - 49529, - 53959, - 58412, - 64662, - 66927, - 68228, - 72092, - 72773, - 74702, - 77903, - 85289, - 85759, - 85860, - 87540, - 87714, - 9984, - 7497, - 3922, - 3510, - 2213, - 10160, - 17038, - 9344, - 21596, - 43, - 55, - 75, - 83, - 3432, - 3482, - 3494, - 3534, - 3645, - 3672, - 3786, - 3910, - 3929, - 4006, - 4095, - 4176, - 4242, - 4271, - 4288, 13345, 13354, 13384, @@ -225,256 +136,339 @@ 13592, 13639, 13699, + 13722, 13758, 13778, + 13780, + 13826, 13838, + 13951, 13963, 14036, 14078, - 14293, - 14456, - 14460, - 14595, - 14618, - 29601, - 29606, - 33431, - 33457, - 33701, - 42534, - 42882, - 43505, - 43663, - 43734, - 44010, - 58738, - 62159, - 62232, - 62261, - 71325, - 71363, - 71387, - 71675, - 71756, - 71760, - 71838, - 71980, - 72000, - 72406, - 72465, - 72590, - 72827, - 1844, - 7257, - 11448, - 12034, - 12507, - 12634, - 12894, - 15439, - 15929, - 17233, - 19207, - 19850, - 21584, - 22032, - 22262, - 26341, - 26837, - 27543, - 28662, - 29044, - 29346, - 29499, - 30501, - 32084, - 33950, - 34056, - 34667, - 34789, - 34980, - 35139, - 42365, - 45688, - 47748, - 49339, - 50058, - 50866, - 51054, - 51287, - 54246, - 54793, - 56101, - 56133, - 56636, - 57384, - 58028, - 63875, - 65457, - 65459, - 65521, - 65607, - 67969, - 68308, - 73991, - 74318, - 75958, - 76034, - 76228, - 76230, - 77800, - 79476, - 79640, - 79644, - 79645, - 79968, - 83220, - 83236, - 84575, - 84585, - 84677, - 85051, - 86555, - 87064, - 42802, - 55323, - 49066, - 7497, - 60484, - 87632, - 79215, - 72285, - 67966, - 14198, - 13826, - 32728, - 7903, - 2412, - 52, - 72348, - 43136, - 79019, - 73574, - 26104, - 25719, - 15656, - 15541, - 12773, - 7468, - 9834, - 12471, - 3323, - 86291, - 50406, - 48258, - 47532, - 18736, - 14846, - 12768, - 12742, - 9959, - 9957, - 4505, - 29778, - 34811, - 37847, - 41983, - 43364, - 43375, - 46322, - 46676, - 46760, - 51809, - 51817, - 54604, - 54640, - 54839, - 58606, - 62870, - 62960, - 65185, - 65190, - 71638, - 71823, - 80449, - 80500, - 84916, - 2701, - 12296, - 13722, - 22234, - 25687, - 28229, - 28240, - 28946, - 34946, - 44066, - 45371, - 51600, - 53924, - 54421, - 56049, - 56737, - 62126, - 64386, - 65584, - 66451, - 79741, - 83931, - 85530, - 87569, - 54739, - 2144, - 6959, - 23248, - 24146, - 29059, - 29291, - 29322, - 38494, - 40054, - 41389, - 43821, - 49267, - 60837, - 72373, - 74737, - 77901, - 81581, - 82314, - 84845, - 85152, - 3770, - 3794, - 3990, - 4031, - 13951, 14096, 14117, 14185, + 14198, + 14293, 14384, + 14456, + 14460, + 14485, + 14489, + 14595, + 14618, + 14846, + 14938, + 14939, + 14947, + 14948, + 14950, + 15058, + 15335, + 15439, + 15511, + 15541, + 15656, + 15929, + 16184, + 16196, + 16700, + 16899, + 17038, + 17233, + 17978, + 18268, + 18736, + 18761, + 18982, + 19207, + 19850, + 20022, + 20023, + 20024, + 20026, + 20027, + 20028, + 20029, + 20030, + 20358, + 21464, + 21584, + 21596, + 22032, + 22234, + 22262, + 23163, + 23236, + 23239, + 23240, + 23248, + 23444, + 23486, + 24146, + 24313, + 24777, + 25125, + 25152, + 25164, + 25365, + 25687, + 25719, + 26104, + 26341, + 26542, + 26698, + 26701, + 26837, + 27543, + 28229, + 28240, + 28662, + 28751, + 28946, + 29044, + 29059, + 29291, + 29322, + 29346, + 29499, + 29592, + 29601, + 29606, + 29778, + 29844, + 29917, + 30501, + 32084, + 32177, + 32617, + 32652, + 32728, + 32807, + 33330, 33372, + 33431, + 33457, + 33701, 33716, + 33889, + 33950, + 34056, + 34282, + 34667, + 34696, + 34789, + 34811, + 34823, + 34946, + 34980, + 35112, + 35139, + 35382, + 36068, + 37847, + 38164, + 38414, + 38494, + 40054, + 40436, + 41142, + 41389, + 41806, + 41983, + 42351, + 42365, + 42534, 42701, + 42802, + 42882, 42885, 42957, 43076, 43102, + 43136, 43146, 43316, + 43364, 43368, + 43375, + 43505, + 43663, + 43734, + 43821, + 44010, 44033, 44040, + 44066, 44080, 44260, + 45371, + 45688, + 45775, + 46322, + 46676, + 46760, + 47301, + 47532, + 47748, + 48258, + 48950, + 49066, + 49267, + 49339, + 49529, + 49991, + 50058, + 50406, + 50866, + 51054, + 51287, + 51600, + 51809, + 51817, + 52534, + 52734, + 53924, + 53959, + 54246, + 54421, + 54604, + 54640, + 54739, + 54793, + 54839, + 55323, + 55721, + 56049, + 56101, + 56133, + 56226, + 56514, + 56636, + 56737, + 57384, + 58028, + 58096, + 58412, + 58606, + 58738, + 59266, + 59890, + 60255, + 60484, + 60837, + 62126, + 62159, + 62232, + 62261, + 62568, + 62757, + 62870, + 62960, + 63190, + 63567, + 63875, + 64386, + 64662, + 65185, + 65190, + 65457, + 65459, + 65521, + 65584, + 65607, + 66451, + 66927, + 67007, + 67087, + 67966, + 67969, + 68228, + 68308, + 68553, + 71325, + 71363, + 71387, + 71402, 71501, + 71583, + 71638, + 71675, + 71756, + 71760, + 71823, + 71838, + 71980, + 72000, + 72092, + 72285, + 72348, + 72373, + 72406, + 72465, 72585, + 72590, 72618, + 72773, + 72827, 72892, + 73574, + 73991, + 74098, + 74318, + 74322, + 74702, + 74737, + 75958, + 76034, + 76228, + 76230, + 77471, + 77514, + 77800, + 77901, + 77903, + 79019, + 79215, + 79476, + 79640, + 79644, + 79645, 79681, - 16196, - 2433, - 28751, + 79741, + 79968, + 80449, + 80500, + 81581, + 82314, + 82434, + 82701, + 83220, + 83236, + 83742, + 83819, + 83931, + 84575, + 84585, + 84677, + 84839, + 84845, + 84916, + 84935, + 85051, + 85152, + 85289, + 85530, + 85759, + 85860, + 86291, + 86555, + 87064, + 87540, + 87569, + 87632, + 87714, 1000000, 1000001, 1000003, @@ -551,17 +545,21 @@ 1000298, 1000299, 1000310, + 1000311, 1000313, 1000315, 1000318, 1000321, + 1000322, 1000324, 1000334, + 1000335, 1000343, 1000347, 1000352, 1000356, 1000362, + 1000363, 1000367, 1000369, 1000371, @@ -574,10 +572,12 @@ 1000396, 1000398, 1000401, + 1000402, 1000404, 1000407, 1000412, 1000419, + 1000420, 1000421, 1000426, 1000433, @@ -585,830 +585,44 @@ 1000442, 1000444, 1000454, + 1000455, 1000456, 1000458, 1000460, + 1000461, 1000471, 1000473, 1000483, + 1000484, 1000485, + 1000486, 1000487, 1000494, + 1000495, 1000497, 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, 1000505, 1000508, 1000511, 1000513, 1000515, + 1000516, 1000517, 1000519, - 1000521, - 1000528, - 1000530, - 1000533, - 1000536, - 1000539, - 1000486, - 1000516, 1000520, - 1000562, - 1000560, - 1000558, - 1000555, - 1000553, - 1000552, - 1000547, - 1000544, + 1000521, 1000522, - 1000000, - 1000001, - 1000003, - 1000004, - 1000008, - 1000018, - 1000028, - 1000031, - 1000034, - 1000038, - 1000039, - 1000042, - 1000044, - 1000046, - 1000047, - 1000048, - 1000059, - 1000060, - 1000062, - 1000072, - 1000078, - 1000083, - 1000084, - 1000087, - 1000088, - 1000089, - 1000092, - 1000099, - 1000102, - 1000125, - 1000133, - 1000139, - 1000155, - 1000161, - 1000162, - 1000163, - 1000171, - 1000177, - 1000188, - 1000190, - 1000191, - 1000192, - 1000196, - 1000199, - 1000200, - 1000205, - 1000210, - 1000213, - 1000215, - 1000217, - 1000223, - 1000233, - 1000234, - 1000235, - 1000241, - 1000247, - 1000250, - 1000251, - 1000253, - 1000257, - 1000258, - 1000259, - 1000261, - 1000267, - 1000268, - 1000273, - 1000275, - 1000279, - 1000288, - 1000289, - 1000291, - 1000293, - 1000296, - 1000298, - 1000299, - 1000310, - 1000313, - 1000315, - 1000318, - 1000321, - 1000324, - 1000334, - 1000343, - 1000347, - 1000352, - 1000356, - 1000362, - 1000367, - 1000369, - 1000371, - 1000374, - 1000376, - 1000379, - 1000384, - 1000387, - 1000394, - 1000396, - 1000398, - 1000401, - 1000404, - 1000407, - 1000412, - 1000419, - 1000421, - 1000426, - 1000433, - 1000435, - 1000442, - 1000444, - 1000454, - 1000456, - 1000458, - 1000460, - 1000471, - 1000473, - 1000483, - 1000485, - 1000487, - 1000494, - 1000497, - 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, - 1000505, - 1000508, - 1000511, - 1000513, - 1000515, - 1000517, - 1000519, - 1000521, 1000528, 1000530, 1000533, 1000536, 1000539, - 1000486, - 1000516, - 1000520, - 1000562, - 1000560, - 1000558, - 1000555, - 1000553, - 1000552, - 1000547, 1000544, - 1000522, - 1000000, - 1000001, - 1000003, - 1000004, - 1000008, - 1000018, - 1000028, - 1000031, - 1000034, - 1000038, - 1000039, - 1000042, - 1000044, - 1000046, - 1000047, - 1000048, - 1000059, - 1000060, - 1000062, - 1000072, - 1000078, - 1000083, - 1000084, - 1000087, - 1000088, - 1000089, - 1000092, - 1000099, - 1000102, - 1000125, - 1000133, - 1000139, - 1000155, - 1000161, - 1000162, - 1000163, - 1000171, - 1000177, - 1000188, - 1000190, - 1000191, - 1000192, - 1000196, - 1000199, - 1000200, - 1000205, - 1000210, - 1000213, - 1000215, - 1000217, - 1000223, - 1000233, - 1000234, - 1000235, - 1000241, - 1000247, - 1000250, - 1000251, - 1000253, - 1000257, - 1000258, - 1000259, - 1000261, - 1000267, - 1000268, - 1000273, - 1000275, - 1000279, - 1000288, - 1000289, - 1000291, - 1000293, - 1000296, - 1000298, - 1000299, - 1000310, - 1000313, - 1000315, - 1000318, - 1000321, - 1000324, - 1000334, - 1000343, - 1000347, - 1000352, - 1000356, - 1000362, - 1000367, - 1000369, - 1000371, - 1000374, - 1000376, - 1000379, - 1000384, - 1000387, - 1000394, - 1000396, - 1000398, - 1000401, - 1000404, - 1000407, - 1000412, - 1000419, - 1000421, - 1000426, - 1000433, - 1000435, - 1000442, - 1000444, - 1000454, - 1000456, - 1000458, - 1000460, - 1000471, - 1000473, - 1000483, - 1000485, - 1000487, - 1000494, - 1000497, - 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, - 1000505, - 1000508, - 1000511, - 1000513, - 1000515, - 1000517, - 1000519, - 1000521, - 1000528, - 1000530, - 1000533, - 1000536, - 1000539, - 1000486, - 1000516, - 1000520, - 1000562, - 1000560, - 1000558, - 1000555, - 1000553, - 1000552, 1000547, - 1000544, - 1000522, - 1000000, - 1000001, - 1000003, - 1000004, - 1000008, - 1000018, - 1000028, - 1000031, - 1000034, - 1000038, - 1000039, - 1000042, - 1000044, - 1000046, - 1000047, - 1000048, - 1000059, - 1000060, - 1000062, - 1000072, - 1000078, - 1000083, - 1000084, - 1000087, - 1000088, - 1000089, - 1000092, - 1000099, - 1000102, - 1000125, - 1000133, - 1000139, - 1000155, - 1000161, - 1000162, - 1000163, - 1000171, - 1000177, - 1000188, - 1000190, - 1000191, - 1000192, - 1000196, - 1000199, - 1000200, - 1000205, - 1000210, - 1000213, - 1000215, - 1000217, - 1000223, - 1000233, - 1000234, - 1000235, - 1000241, - 1000247, - 1000250, - 1000251, - 1000253, - 1000257, - 1000258, - 1000259, - 1000261, - 1000267, - 1000268, - 1000273, - 1000275, - 1000279, - 1000288, - 1000289, - 1000291, - 1000293, - 1000296, - 1000298, - 1000299, - 1000310, - 1000313, - 1000315, - 1000318, - 1000321, - 1000324, - 1000334, - 1000343, - 1000347, - 1000352, - 1000356, - 1000362, - 1000367, - 1000369, - 1000371, - 1000374, - 1000376, - 1000379, - 1000384, - 1000387, - 1000394, - 1000396, - 1000398, - 1000401, - 1000404, - 1000407, - 1000412, - 1000419, - 1000421, - 1000426, - 1000433, - 1000435, - 1000442, - 1000444, - 1000454, - 1000456, - 1000458, - 1000460, - 1000471, - 1000473, - 1000483, - 1000485, - 1000487, - 1000494, - 1000497, - 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, - 1000505, - 1000508, - 1000511, - 1000513, - 1000515, - 1000517, - 1000519, - 1000521, - 1000528, - 1000530, - 1000533, - 1000536, - 1000539, - 1000486, - 1000516, - 1000520, - 1000562, - 1000560, - 1000558, - 1000555, - 1000553, 1000552, - 1000547, - 1000544, - 1000522, - 1000000, - 1000001, - 1000003, - 1000004, - 1000008, - 1000018, - 1000028, - 1000031, - 1000034, - 1000038, - 1000039, - 1000042, - 1000044, - 1000046, - 1000047, - 1000048, - 1000059, - 1000060, - 1000062, - 1000072, - 1000078, - 1000083, - 1000084, - 1000087, - 1000088, - 1000089, - 1000092, - 1000099, - 1000102, - 1000125, - 1000133, - 1000139, - 1000155, - 1000161, - 1000162, - 1000163, - 1000171, - 1000177, - 1000188, - 1000190, - 1000191, - 1000192, - 1000196, - 1000199, - 1000200, - 1000205, - 1000210, - 1000213, - 1000215, - 1000217, - 1000223, - 1000233, - 1000234, - 1000235, - 1000241, - 1000247, - 1000250, - 1000251, - 1000253, - 1000257, - 1000258, - 1000259, - 1000261, - 1000267, - 1000268, - 1000273, - 1000275, - 1000279, - 1000288, - 1000289, - 1000291, - 1000293, - 1000296, - 1000298, - 1000299, - 1000310, - 1000313, - 1000315, - 1000318, - 1000321, - 1000324, - 1000334, - 1000343, - 1000347, - 1000352, - 1000356, - 1000362, - 1000367, - 1000369, - 1000371, - 1000374, - 1000376, - 1000379, - 1000384, - 1000387, - 1000394, - 1000396, - 1000398, - 1000401, - 1000404, - 1000407, - 1000412, - 1000419, - 1000421, - 1000426, - 1000433, - 1000435, - 1000442, - 1000444, - 1000454, - 1000456, - 1000458, - 1000460, - 1000471, - 1000473, - 1000483, - 1000485, - 1000487, - 1000494, - 1000497, - 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, - 1000505, - 1000508, - 1000511, - 1000513, - 1000515, - 1000517, - 1000519, - 1000521, - 1000528, - 1000530, - 1000533, - 1000536, - 1000539, - 1000486, - 1000516, - 1000520, - 1000562, - 1000560, - 1000558, - 1000555, 1000553, - 1000552, - 1000547, - 1000544, - 1000522, - 1000000, - 1000001, - 1000003, - 1000004, - 1000008, - 1000018, - 1000028, - 1000031, - 1000034, - 1000038, - 1000039, - 1000042, - 1000044, - 1000046, - 1000047, - 1000048, - 1000059, - 1000060, - 1000062, - 1000072, - 1000078, - 1000083, - 1000084, - 1000087, - 1000088, - 1000089, - 1000092, - 1000099, - 1000102, - 1000125, - 1000133, - 1000139, - 1000155, - 1000161, - 1000162, - 1000163, - 1000171, - 1000177, - 1000188, - 1000190, - 1000191, - 1000192, - 1000196, - 1000199, - 1000200, - 1000205, - 1000210, - 1000213, - 1000215, - 1000217, - 1000223, - 1000233, - 1000234, - 1000235, - 1000241, - 1000247, - 1000250, - 1000251, - 1000253, - 1000257, - 1000258, - 1000259, - 1000261, - 1000267, - 1000268, - 1000273, - 1000275, - 1000279, - 1000288, - 1000289, - 1000291, - 1000293, - 1000296, - 1000298, - 1000299, - 1000310, - 1000313, - 1000315, - 1000318, - 1000321, - 1000324, - 1000334, - 1000343, - 1000347, - 1000352, - 1000356, - 1000362, - 1000367, - 1000369, - 1000371, - 1000374, - 1000376, - 1000379, - 1000384, - 1000387, - 1000394, - 1000396, - 1000398, - 1000401, - 1000404, - 1000407, - 1000412, - 1000419, - 1000421, - 1000426, - 1000433, - 1000435, - 1000442, - 1000444, - 1000454, - 1000456, - 1000458, - 1000460, - 1000471, - 1000473, - 1000483, - 1000485, - 1000487, - 1000494, - 1000497, - 1000503, - 1000311, - 1000322, - 1000335, - 1000363, - 1000402, - 1000420, - 1000455, - 1000461, - 1000484, - 1000495, - 1000505, - 1000508, - 1000511, - 1000513, - 1000515, - 1000517, - 1000519, - 1000521, - 1000528, - 1000530, - 1000533, - 1000536, - 1000539, - 1000486, - 1000516, - 1000520, - 1000562, - 1000560, - 1000558, 1000555, - 1000553, - 1000552, - 1000547, - 1000544, - 1000522 + 1000558, + 1000560, + 1000562 ] \ No newline at end of file diff --git a/preprocess/extract_category_tool_details.py b/preprocess/extract_category_tool_details.py index 72bc324..f2963f9 100644 --- a/preprocess/extract_category_tool_details.py +++ b/preprocess/extract_category_tool_details.py @@ -27,6 +27,4 @@ def extract_tool_data(): tool_data[root.split('/')[-1]][tool_name] = {"tool_description": tool_description} return tool_data tool_data = extract_tool_data() -print(tool_data.keys()) json.dump(tool_data, open("category_tool_details.json", "w", encoding='utf-8'), indent=4) -# json.dump(tool_data, open("category_tool_details_add_nonfree.json", "w", encoding='utf-8'), indent=4) diff --git a/rapidapi_key_list_example.json b/rapidapi_key_list_example.json index 0637a08..e8089d5 100644 --- a/rapidapi_key_list_example.json +++ b/rapidapi_key_list_example.json @@ -1 +1 @@ -[] \ No newline at end of file +["xxxxx"] \ No newline at end of file diff --git a/scripts/main.py b/scripts/main.py index b4a2c3f..cb91059 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -899,9 +899,9 @@ if __name__ == "__main__": os.makedirs(output_dir, exist_ok=True) os.makedirs('output', exist_ok=True) success_cnt = 0 - pass_cnt = 0 unsolvable_task_cnt = 0 unsolvable_list = json.load(open('misc/unsolvable.json', 'r', encoding='utf-8')) + json.dump(sorted(list(set(unsolvable_list))), open('misc/unsolvable.json', 'w', encoding='utf-8'), indent=4) total_cnt = 0 query_data_all = json.load(open(query_path, 'r', encoding='utf-8')) for query_data in query_data_all: @@ -930,11 +930,9 @@ if __name__ == "__main__": assign_results['stop'] = [] ts = time.time() resumed_agents = [] - print(query_id, query, file=open(f'{output_dir}/query.txt', 'a', encoding='utf-8')) if not args.include_unsolvable and int(query_id) in unsolvable_list: unsolvable_task_cnt += 1 - print(unsolvable_task_cnt) - print('Unsolvable human', unsolvable_task_cnt, pass_cnt, success_cnt, total_cnt, file=open(f'{output_dir}/success_cnt.txt', 'a', encoding='utf-8')) + print('Unsolvable human', unsolvable_task_cnt, success_cnt, total_cnt, file=open(f'{output_dir}/success_cnt.txt', 'a', encoding='utf-8')) continue total_cnt += 1 task_solvable = 'Solvable' @@ -945,8 +943,6 @@ if __name__ == "__main__": solved = assign_results['solved'] check_solved = assign_results['check_solved'] last_solve_time = assign_results['last_solve_time'] - if solved: - pass_cnt += 1 if args.recheck_solved: check_solved, reason, _ = check_solved_toolbench(f'{output_dir}/{query_id}_{last_solve_time}_DFS_woFilter_w2.json', assign_results['query_id']) assign_results['check_solved'] = check_solved @@ -1007,8 +1003,6 @@ if __name__ == "__main__": break cnt += 1 failed_reason = None - print(len(global_api_list), file=open(f'{output_dir}/api_list_len.txt', 'a', encoding='utf-8')) - print(global_api_list, file=open(f'{output_dir}/api_list.txt', 'a', encoding='utf-8')) print('#'*100) assign_results['api_list'].append(deepcopy(global_api_list)) if stop or not flag or all([agent.finish_search for agent in agents]) and len(global_api_list) > 0: @@ -1045,11 +1039,9 @@ if __name__ == "__main__": if origin_api in global_api_list: api_list_to_prune.append(origin_api) print(colored(api_list_to_prune, 'red')) - print(len(api_list_to_prune)) remove_apis(api_list_to_prune) if len(global_api_list) >= max_api_number: break - # print(api_list_to_prune, file=open(f'{output_dir}/prune_api_list.txt', 'a', encoding='utf-8')) stop = False else: assert status != 'The current api list can solve the query.' @@ -1108,8 +1100,6 @@ if __name__ == "__main__": success_cnt += 1 else: print(output_dir, 'failed', file=open(f'{output_dir}/failed.txt', 'a', encoding='utf-8')) - if solved: - pass_cnt += 1 assign_results['loop_times'] = cnt assign_results['last_solve_time'] = last_solve_time if 'messages' in solve_data: @@ -1144,7 +1134,7 @@ if __name__ == "__main__": assign_results['check_solved'] = check_solved json.dump(assign_results, open(f'{output_dir}/{query_id}.json', 'w', encoding='utf-8'), indent=4) print(check_solved, total_tokens, time.time() - ts, query_path, file=open(f'{output_dir}/time.txt', 'a', encoding='utf-8')) - print(query_id, task_solvable, cnt, check_solved, unsolvable_task_cnt, pass_cnt, success_cnt, total_cnt, success_cnt/total_cnt, file=open(f'{output_dir}/success_cnt.txt', 'a', encoding='utf-8')) + print(query_id, check_solved, success_cnt, total_cnt, success_cnt/total_cnt, file=open(f'{output_dir}/success_cnt.txt', 'a', encoding='utf-8')) except Exception as e: print(e) continue \ No newline at end of file