Make sure to sort only list of strings in unsorted_list responses.

test_of_time_accuracy.py

@@ -215,7 +215,7 @@ class TestOfTimeAccuracy(evaluate.Metric):
             otherwise returns data unchanged
         """
         if isinstance(data, dict) and "unordered_list" in data:
-            return sorted(data["unordered_list"])
+            return sorted([item for item in data["unordered_list"] if isinstance(item, str)])
         return data
 
     @staticmethod
@@ -279,11 +279,10 @@ class TestOfTimeAccuracy(evaluate.Metric):
         if data is None or not isinstance(data, dict):
             return data
 
-        # Process the first key if it's a list field
-        if data:
-            first_key = next(iter(data.keys()))
-            if first_key in ["ordered_list", "unordered_list"] and isinstance(data[first_key], list):
-                data[first_key] = [item.lower() for item in data[first_key] if isinstance(item, str)]
+        # Process list fields regardless of key order
+        for key in ["ordered_list", "unordered_list"]:
+            if key in data and isinstance(data[key], list):
+                data[key] = [item.lower() for item in data[key] if isinstance(item, str)]
 
         return data
 

tests/test_arithmetic_scoring.py

@@ -9,6 +9,7 @@ arithmetic_test_cases = {
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["berlin","london"]\n}\n```',
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "malformed_unordered_list": ["Berlin","London"]\n}\n```',
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": "Berlin"\n}\n```',
+        '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["Berlin", {"unordered_list": ["London", "Berlin"]}]\n}\n```',
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
         '\n```json\n{\n  "explanation": "Step 1: Determine the time it takes the robot to carry a single box. The robot takes 4 hours, 34 minutes, and 30 seconds to carry 2 boxes. We divide this time by 2 to find the time per box.\\n- Hours: 4 / 2 = 2 hours\\n- Minutes: 34 / 2 = 17 minutes\\n- Seconds: 30 / 2 = 15 seconds\\nSo, it takes the robot 2 hours, 17 minutes, and 15 seconds to carry one box.\\n\\nStep 2: Calculate the total time to carry 25 boxes. We multiply the time per box by the total number of boxes (25).\\n- Total Hours: 2 hours/box * 25 boxes = 50 hours\\n- Total Minutes: 17 minutes/box * 25 boxes = 425 minutes\\n- Total Seconds: 15 seconds/box * 25 boxes = 375 seconds\\n\\nStep 3: Convert the calculated time into the standard H:M:S format by carrying over excess seconds and minutes.\\n- Convert seconds to minutes: 375 seconds is equal to 6 minutes and 15 seconds (since 375 / 60 = 6 with a remainder of 15). We add the 6 minutes to our minutes total.\\n- New total: 50 hours, (425 + 6) minutes, 15 seconds -> 50 hours, 431 minutes, 15 seconds.\\n- Convert minutes to hours: 431 minutes is equal to 7 hours and 11 minutes (since 431 / 60 = 7 with a remainder of 11). We add the 7 hours to our hours total.\\n- New total: (50 + 7) hours, 11 minutes, 15 seconds -> 57 hours, 11 minutes, 15 seconds.\\n\\nThe final time is 57 hours, 11 minutes, and 15 seconds.",\n  "H": 57,\n  "M": 11,\n  "S": 15\n}\n```',
         '{"explanation": "some explanation", "age": "3319"}',
@@ -21,13 +22,14 @@ arithmetic_test_cases = {
         '{"unordered_list": ["London", "Berlin"]}',
         '{"unordered_list": ["London", "Berlin"]}',
         '{"unordered_list": ["London", "Berlin"]}',
+        '{"unordered_list": ["London", "Berlin"]}',
         '{"answer": "2005-04-07"}',
         '{"H": 57.0, "M": 11.0, "S": 15.0}',
         '{"answer": 3319}',
         '{"answer": 6}',
         "{'hours': 6, 'minutes': 2, 'seconds': 58}",
     ],
-    "result": {"accuracy": 7 / 10},
+    "result": {"accuracy": 7 / 11},
     "per_item_accuracy": [
         True,
         True,
@@ -35,6 +37,7 @@ arithmetic_test_cases = {
         False,
         False,
         False,
+        False,
         True,
         True,
         True,