push

kaushiks_criteria.py

@@ -28,27 +28,40 @@ year={2020}
 
 # TODO: Add description of the module here
 _DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
+Evaluate structured output formatting for generated text.
+- considers header / column / tag / key names
+- DOES NOT consider the cell / row values
+
+Formats:
+    - [] Custom
+    - [] Markdown tables
+    - [] HTML tables
+    - [] JSON
+    - [] XML
+    - [] CSV / TSV 
 """
 
 
 # TODO: Add description of the arguments of the module here
 _KWARGS_DESCRIPTION = """
-Calculates how good are predictions given some references, using certain scores
+Calculates how well the `structure` of the predictions matches the `structure` of the references.
 Args:
-    predictions: list of predictions to score. Each predictions
+    predictions: list of strings to score. Each predictions
         should be a string with tokens separated by spaces.
     references: list of reference for each prediction. Each
         reference should be a string with tokens separated by spaces.
+    invariance: bool, whether to consider the order of the columns / tags / keys.
 Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
+    kaushiks_criteria: kaushiks_criteria score.
 Examples:
     Examples should be written in doctest format, and should illustrate how
     to use the function.
 
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
+    >>> my_new_module = evaluate.load("DoctorSlimm/kaushiks_criteria")
+    >>> results = my_new_module.compute(
+            references=['<table><tr><td>1</td><td>2</td></tr></table>'],
+            predictions=['<table><tr><td>1</td><td>2</td></tr></table>']
+        )
     >>> print(results)
     {'accuracy': 1.0}
 """
@@ -71,8 +84,8 @@ class kaushiks_criteria(evaluate.Metric):
             inputs_description=_KWARGS_DESCRIPTION,
             # This defines the format of each prediction and reference
             features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
+                'predictions': datasets.Value('string'),
+                'references': datasets.Value('string'),
             }),
             # Homepage of the module for documentation
             homepage="http://module.homepage",