Add more directed type casting for arithmetic subset.

test_of_time_accuracy.py

@@ -120,11 +120,41 @@ class TestOfTimeAccuracy(evaluate.Metric):
         except (ValueError, SyntaxError):
             return None
 
-    def _sort_unordered_list(self, d):
+    @staticmethod
+    def _sort_unordered_list(d):
         if isinstance(d, dict) and "unordered_list" in d:
             return sorted(d["unordered_list"])
         return d
 
+    @staticmethod
+    def _cast_prediction(reference: dict, prediction: dict) -> None | dict:
+        """
+        Casts the values in the prediction dictionary to match the types
+        of the values in the reference dictionary.
+        """
+        casted_prediction = {}
+
+        try:
+            for ref_key, ref_value in reference.items():
+                if ref_key not in prediction:
+                    return None
+
+                reference_type = type(ref_value)
+                pred_value = prediction[ref_key]
+
+                # Special safeguard: Python allows list("abc") -> ['a', 'b', 'c'].
+                # We don't want to turn strings into character lists.
+                if reference_type == list and not isinstance(pred_value, list):
+                    return None
+
+                # This handles int("123") -> 123, float(12) -> 12.0, str(100) -> "100"
+                casted_prediction[ref_key] = reference_type(pred_value)
+
+            return casted_prediction
+
+        except (ValueError, TypeError):
+            return None
+
     def _compute(
         self,
         predictions,
@@ -135,22 +165,28 @@ class TestOfTimeAccuracy(evaluate.Metric):
         """Returns the scores"""
         predictions = [self._extract_first_json_object(p) for p in predictions]
         if subset == "semantic":
+            # Semantic subset's answers are not JSON objects.
+            # Expected answers are always in "answer" field.
             predictions = [self._get_answer(p) for p in predictions]
         elif subset == "arithmetic":
+            # Arithmetic subset's answers are JSON objects.
+            # Answer fields vary. Thus, remove explanation field.
             predictions = [self._pop_explanation(p) for p in predictions]
             references = [self._parse_label(r) for r in references]
         else:
             raise ValueError(f"Invalid subset: {subset}")
         accuracy = []
-        for i, j in zip(predictions, references):
-            if subset == "arithmetic" and "unordered_list" in j:
-                i = self._sort_unordered_list(i)
-                j = self._sort_unordered_list(j)
+        for pred, ref in zip(predictions, references):
+            if subset == "arithmetic":
+                pred = self._cast_prediction(ref, pred)
+                if "unordered_list" in ref:
+                    pred = self._sort_unordered_list(pred)
+                    ref = self._sort_unordered_list(ref)
             if subset == "semantic":
-                i = str(i)
-                j = str(j)
+                pred = str(pred)
+                ref = str(ref)
             accuracy.append(
-                i == j
+                pred == ref
             )  # Semantic subset answer JSON somestimes has int as value. Label is string.
         if return_average:
             return {"accuracy": sum(accuracy) / len(accuracy)}

tests.py → tests/test_arithmetic_scoring.py

@@ -8,30 +8,19 @@ arithmetic_test_cases = {
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["Berlin","London"]\n}\n```',
         '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "malformed_unordered_list": ["Berlin","London"]\n}\n```',
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
-        '\n```json\n{\n  "explanation": "Step 1: Determine the time it takes the robot to carry a single box. The robot takes 4 hours, 34 minutes, and 30 seconds to carry 2 boxes. We divide this time by 2 to find the time per box.\\n- Hours: 4 / 2 = 2 hours\\n- Minutes: 34 / 2 = 17 minutes\\n- Seconds: 30 / 2 = 15 seconds\\nSo, it takes the robot 2 hours, 17 minutes, and 15 seconds to carry one box.\\n\\nStep 2: Calculate the total time to carry 25 boxes. We multiply the time per box by the total number of boxes (25).\\n- Total Hours: 2 hours/box * 25 boxes = 50 hours\\n- Total Minutes: 17 minutes/box * 25 boxes = 425 minutes\\n- Total Seconds: 15 seconds/box * 25 boxes = 375 seconds\\n\\nStep 3: Convert the calculated time into the standard H:M:S format by carrying over excess seconds and minutes.\\n- Convert seconds to minutes: 375 seconds is equal to 6 minutes and 15 seconds (since 375 / 60 = 6 with a remainder of 15). We add the 6 minutes to our minutes total.\\n- New total: 50 hours, (425 + 6) minutes, 15 seconds -> 50 hours, 431 minutes, 15 seconds.\\n- Convert minutes to hours: 431 minutes is equal to 7 hours and 11 minutes (since 431 / 60 = 7 with a remainder of 11). We add the 7 hours to our hours total.\\n- New total: (50 + 7) hours, 11 minutes, 15 seconds -> 57 hours, 11 minutes, 15 seconds.\\n\\nThe final time is 57 hours, 11 minutes, and 15 seconds.",\n  "H": 57,\n  "M": 11,\n  "S": 15\n}\n```'
+        '\n```json\n{\n  "explanation": "Step 1: Determine the time it takes the robot to carry a single box. The robot takes 4 hours, 34 minutes, and 30 seconds to carry 2 boxes. We divide this time by 2 to find the time per box.\\n- Hours: 4 / 2 = 2 hours\\n- Minutes: 34 / 2 = 17 minutes\\n- Seconds: 30 / 2 = 15 seconds\\nSo, it takes the robot 2 hours, 17 minutes, and 15 seconds to carry one box.\\n\\nStep 2: Calculate the total time to carry 25 boxes. We multiply the time per box by the total number of boxes (25).\\n- Total Hours: 2 hours/box * 25 boxes = 50 hours\\n- Total Minutes: 17 minutes/box * 25 boxes = 425 minutes\\n- Total Seconds: 15 seconds/box * 25 boxes = 375 seconds\\n\\nStep 3: Convert the calculated time into the standard H:M:S format by carrying over excess seconds and minutes.\\n- Convert seconds to minutes: 375 seconds is equal to 6 minutes and 15 seconds (since 375 / 60 = 6 with a remainder of 15). We add the 6 minutes to our minutes total.\\n- New total: 50 hours, (425 + 6) minutes, 15 seconds -> 50 hours, 431 minutes, 15 seconds.\\n- Convert minutes to hours: 431 minutes is equal to 7 hours and 11 minutes (since 431 / 60 = 7 with a remainder of 11). We add the 7 hours to our hours total.\\n- New total: (50 + 7) hours, 11 minutes, 15 seconds -> 57 hours, 11 minutes, 15 seconds.\\n\\nThe final time is 57 hours, 11 minutes, and 15 seconds.",\n  "H": 57,\n  "M": 11,\n  "S": 15\n}\n```',
     ],
     "references": [
         '{"answer": "352 BC"}',
         '{"unordered_list": ["London", "Berlin"]}',
         '{"unordered_list": ["London", "Berlin"]}',
-        "{'answer': '2005-04-07'}",
-        '{"H": 57.0, "M": 11.0, "S": 15.0}'
+        '{"answer": "2005-04-07"}',
+        '{"H": 57.0, "M": 11.0, "S": 15.0}',
     ],
-    "result": {"accuracy": 3/5},
+    "result": {"accuracy": 3 / 5},
     "per_item_accuracy": [True, True, False, False, True],
 }
 
-semantic_test_cases = {
-    "predictions": [
-        '{"explanation": First, we need to find the third occurrence of E33 being the R53 of E22. We can see that it happened from 1959 to 1962, then from 1967 to 1968, and then from 1982 to 1984. The third occurrence happened from 1982 to 1984. We can then compute the duration by subtracting the start time from the end time.", "answer": 2}',
-        ' "To find the duration, we need to find the start and end time when E97 was the R71 of E67. From the given facts, we can see that E97 was the R71 of E67 from 1961 to 1961, and also from 1964 to 1964. We need to find the first occurrence, which is from 1961 to 1961.", "answer": 1}',
-        '{"explanation": "To find when E92 stopped being the R88 of E11, we need to look at the temporal facts where E92 was the R88 of E11 and find the end time. We see that E92 was the R88 of E11 from 1982 to 1985, and there is no other fact that indicates E92 stopped being the R88 of E11 before 1985. However, we also see that E92 was the R17 of E42 from 1986 to 1992, and E92 was the R88 of E42 from 1977 to 1979, but this is irrelevant to the question. Therefore, E92 stopped being the R88 of E11 in 1985.", "answer": 1985}',
-    ],
-    "references": ["2", "0", "1985"],
-    "result": {"accuracy": 1 / 3},
-    "per_item_accuracy": [False, False, True],
-}
-
 
 def test_arithmetic_accuracy():
     metric = TestOfTimeAccuracy()
@@ -43,16 +32,6 @@ def test_arithmetic_accuracy():
     assert results == arithmetic_test_cases["result"]
 
 
-def test_semantic_accuracy():
-    metric = TestOfTimeAccuracy()
-    results = metric.compute(
-        predictions=semantic_test_cases["predictions"],
-        references=semantic_test_cases["references"],
-        subset="semantic",
-    )
-    assert results == semantic_test_cases["result"]
-
-
 def test_per_item_arithmetic_accuracy():
     metric = TestOfTimeAccuracy()
     results = metric.compute(
@@ -64,17 +43,6 @@ def test_per_item_arithmetic_accuracy():
     assert results["accuracy"] == arithmetic_test_cases["per_item_accuracy"]
 
 
-def test_per_item_semantic_accuracy():
-    metric = TestOfTimeAccuracy()
-    results = metric.compute(
-        predictions=semantic_test_cases["predictions"],
-        references=semantic_test_cases["references"],
-        subset="semantic",
-        return_average=False,
-    )
-    assert results["accuracy"] == semantic_test_cases["per_item_accuracy"]
-
-
 def test_invalid_subset():
     metric = TestOfTimeAccuracy()
     with pytest.raises(ValueError):

tests/test_arithmetic_type_casting.py

@@ -0,0 +1,186 @@
+# Stastics about expected data type per key
+##################################################################################################
+# label_key                      label_value_type
+# answer                         <class 'str'>                                                 600
+#                                <class 'int'>                                                 300
+# unordered_list                 <class 'list'>                                                250
+# date                           <class 'str'>                                                 150
+# day, time                      <class 'str'>, <class 'str'>                                  123
+# H, M, S                        <class 'float'>, <class 'float'>, <class 'float'>             100
+# ordered_list                   <class 'list'>                                                100
+# A, B, C                        <class 'int'>, <class 'int'>, <class 'int'>                    50
+# X, Y, Z                        <class 'float'>, <class 'float'>, <class 'float'>              50
+#                                <class 'int'>, <class 'int'>, <class 'int'>                    50
+# hours, minutes, seconds        <class 'int'>, <class 'int'>, <class 'int'>                    50
+# hours, minutes                 <class 'int'>, <class 'int'>                                   15
+# days, hours, minutes, seconds  <class 'int'>, <class 'int'>, <class 'int'>, <class 'int'>     12
+##################################################################################################
+
+from test_arithmetic_scoring import TestOfTimeAccuracy
+
+
+def test_answer_type_casting():
+    references_answer_key = [
+        {"answer": "352 BC"},
+        {"answer": 1032},
+    ]
+    predictions_answer_key = [
+        {"answer": "352 BC"},
+        {"answer": "1032"},
+    ]
+    for ref, pred in zip(references_answer_key, predictions_answer_key):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_unordered_list_type_casting():
+    references_unordered_list_key = [{"unordered_list": ["Kyle", "Jason", "Joe"]}]
+    predictions_unordered_list_key = [{"unordered_list": ["Kyle", "Jason", "Joe"]}]
+    for ref, pred in zip(references_unordered_list_key, predictions_unordered_list_key):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_date_type_casting():
+    references_date_key = [{"date": "12/11/2011"}]
+    predictions_date_key = [{"date": "12/11/2011"}]
+    for ref, pred in zip(references_date_key, predictions_date_key):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_day_time_type_casting():
+    references_day_time_keys = [{"day": "+2", "time": "21:44:10"}]
+    predictions_day_time_keys = [{"day": "+2", "time": "21:44:10"}]
+    for ref, pred in zip(references_day_time_keys, predictions_day_time_keys):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_hms_type_casting():
+    references_hms_keys = [
+        {"H": 2.0, "M": 13.0, "S": 30.0},
+        {"H": 2.0, "M": 13.0, "S": 30.0},
+        {"H": 2.0, "M": 13.0, "S": 30.0},
+        {"H": 2.0, "M": 13.0, "S": 30.0},
+    ]
+    predictions_hms_keys = [
+        {"H": 2, "M": 13, "S": 30},
+        {"H": 2.0, "M": 13.0, "S": 30.0},
+        {"H": "2", "M": "13", "S": "30"},
+        {"H": "2.0", "M": "13.0", "S": "30.0"},
+    ]
+    for ref, pred in zip(references_hms_keys, predictions_hms_keys):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_ordered_list_type_casting():
+    references_ordered_list_key = [
+        {"ordered_list": ["Joe", "Jenny", "Jason", "Dan", "Kyle"]},
+    ]
+    predictions_ordered_list_key = [
+        {"ordered_list": ["Joe", "Jenny", "Jason", "Dan", "Kyle"]},
+    ]
+    for ref, pred in zip(references_ordered_list_key, predictions_ordered_list_key):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+# TODO: Check if I should treat float strings differently, e.g., int(float("18.0"))
+def test_abc_type_casting():
+    references_abc_keys = [
+        {"A": 80, "B": 22, "C": 20},
+        {"A": 80, "B": 22, "C": 20},
+        {"A": 80, "B": 22, "C": 20},
+        # {"A": 80, "B": 22, "C": 20},
+    ]
+    predictions_abc_keys = [
+        {"A": 80, "B": 22, "C": 20},
+        {"A": 80.0, "B": 22.0, "C": 20.0},
+        {"A": "80", "B": "22", "C": "20"},
+        # {"A": "80.0", "B": "22.0", "C": "20.0"},
+    ]
+    for ref, pred in zip(references_abc_keys, predictions_abc_keys):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_xyz_type_casting():
+    references_xyz_keys = [
+        {"X": 2.0, "Y": 24.0, "Z": 44.0},
+        {"X": 2, "Y": 4, "Z": 36},
+        {"X": 2.0, "Y": 24.0, "Z": 44.0},
+        {"X": 2, "Y": 4, "Z": 36},
+        {"X": 2.0, "Y": 24.0, "Z": 44.0},
+        {"X": 2, "Y": 4, "Z": 36},
+    ]
+    predictions_xyz_keys = [
+        {"X": 2.0, "Y": 24.0, "Z": 44.0},
+        {"X": 2, "Y": 4, "Z": 36},
+        {"X": "2.0", "Y": "24.0", "Z": "44.0"},
+        {"X": "2", "Y": "4", "Z": "36"},
+        {"X": 2, "Y": 24, "Z": 44},
+        {"X": 2.0, "Y": 4.0, "Z": 36.0},
+    ]
+    for ref, pred in zip(references_xyz_keys, predictions_xyz_keys):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_hours_minutes_seconds_type_casting():
+    references_hours_minutes_seconds_keys = [
+        {"hours": 17, "minutes": 48, "seconds": 51},
+        {"hours": 17, "minutes": 48, "seconds": 51},
+        {"hours": 17, "minutes": 48, "seconds": 51},
+        # {"hours": 17, "minutes": 48, "seconds": 51},
+    ]
+    predictions_hours_minutes_seconds_keys = [
+        {"hours": 17, "minutes": 48, "seconds": 51},
+        {"hours": 17.0, "minutes": 48.0, "seconds": 51.0},
+        {"hours": "17", "minutes": "48", "seconds": "51"},
+        # {"hours": "17.0", "minutes": "48.0", "seconds": "51.0"},
+    ]
+    for ref, pred in zip(
+        references_hours_minutes_seconds_keys, predictions_hours_minutes_seconds_keys
+    ):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_hours_minutes_type_casting():
+    references_hours_minutes_keys = [
+        {"hours": 5, "minutes": 0},
+        {"hours": 5, "minutes": 0},
+        {"hours": 5, "minutes": 0},
+        # {"hours": 5, "minutes": 0},
+    ]
+    predictions_hours_minutes_keys = [
+        {"hours": 5, "minutes": 0},
+        {"hours": 5.0, "minutes": 0.0},
+        {"hours": "5", "minutes": "0"},
+        # {"hours": "5.0", "minutes": "0.0"},
+    ]
+    for ref, pred in zip(references_hours_minutes_keys, predictions_hours_minutes_keys):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast
+
+
+def test_days_hours_minutes_seconds_type_casting():
+    references_days_hours_minutes_seconds_keys = [
+        {"days": 1, "hours": 12, "minutes": 45, "seconds": 0},
+        {"days": 1, "hours": 12, "minutes": 45, "seconds": 0},
+        {"days": 1, "hours": 12, "minutes": 45, "seconds": 0},
+        # {"days": 1, "hours": 12, "minutes": 45, "seconds": 0},
+    ]
+    predictions_days_hours_minutes_seconds_keys = [
+        {"days": 1, "hours": 12, "minutes": 45, "seconds": 0},
+        {"days": 1.0, "hours": 12.0, "minutes": 45.0, "seconds": 0.0},
+        {"days": "1", "hours": "12", "minutes": "45", "seconds": "0"},
+        # {"days": "1.0", "hours": "12.0", "minutes": "45.0", "seconds": "0.0"},
+    ]
+    for ref, pred in zip(
+        references_days_hours_minutes_seconds_keys,
+        predictions_days_hours_minutes_seconds_keys,
+    ):
+        pred_cast = TestOfTimeAccuracy._cast_prediction(ref, pred)
+        assert ref == pred_cast

tests/test_semantic_scoring.py

@@ -0,0 +1,33 @@
+from test_of_time_accuracy import TestOfTimeAccuracy
+
+semantic_test_cases = {
+    "predictions": [
+        '{"explanation": First, we need to find the third occurrence of E33 being the R53 of E22. We can see that it happened from 1959 to 1962, then from 1967 to 1968, and then from 1982 to 1984. The third occurrence happened from 1982 to 1984. We can then compute the duration by subtracting the start time from the end time.", "answer": 2}',
+        ' "To find the duration, we need to find the start and end time when E97 was the R71 of E67. From the given facts, we can see that E97 was the R71 of E67 from 1961 to 1961, and also from 1964 to 1964. We need to find the first occurrence, which is from 1961 to 1961.", "answer": 1}',
+        '{"explanation": "To find when E92 stopped being the R88 of E11, we need to look at the temporal facts where E92 was the R88 of E11 and find the end time. We see that E92 was the R88 of E11 from 1982 to 1985, and there is no other fact that indicates E92 stopped being the R88 of E11 before 1985. However, we also see that E92 was the R17 of E42 from 1986 to 1992, and E92 was the R88 of E42 from 1977 to 1979, but this is irrelevant to the question. Therefore, E92 stopped being the R88 of E11 in 1985.", "answer": 1985}',
+    ],
+    "references": ["2", "0", "1985"],
+    "result": {"accuracy": 1 / 3},
+    "per_item_accuracy": [False, False, True],
+}
+
+
+def test_per_item_semantic_accuracy():
+    metric = TestOfTimeAccuracy()
+    results = metric.compute(
+        predictions=semantic_test_cases["predictions"],
+        references=semantic_test_cases["references"],
+        subset="semantic",
+        return_average=False,
+    )
+    assert results["accuracy"] == semantic_test_cases["per_item_accuracy"]
+
+
+def test_semantic_accuracy():
+    metric = TestOfTimeAccuracy()
+    results = metric.compute(
+        predictions=semantic_test_cases["predictions"],
+        references=semantic_test_cases["references"],
+        subset="semantic",
+    )
+    assert results == semantic_test_cases["result"]