Spaces:
Runtime error
Runtime error
initial commit
Browse files- roc_auc_macro.py +29 -23
roc_auc_macro.py
CHANGED
@@ -15,47 +15,55 @@
|
|
15 |
|
16 |
import evaluate
|
17 |
import datasets
|
|
|
18 |
|
19 |
|
20 |
# TODO: Add BibTeX citation
|
21 |
_CITATION = """\
|
22 |
@InProceedings{huggingface:module,
|
23 |
-
title = {
|
24 |
-
authors={
|
25 |
-
year={
|
26 |
}
|
27 |
"""
|
28 |
|
29 |
# TODO: Add description of the module here
|
30 |
_DESCRIPTION = """\
|
31 |
-
This
|
|
|
|
|
32 |
"""
|
33 |
|
34 |
|
35 |
# TODO: Add description of the arguments of the module here
|
36 |
_KWARGS_DESCRIPTION = """
|
37 |
-
|
|
|
|
|
|
|
38 |
Args:
|
39 |
-
|
40 |
-
|
|
|
|
|
41 |
references: list of reference for each prediction. Each
|
42 |
-
reference should be
|
|
|
43 |
Returns:
|
44 |
-
|
45 |
-
another_score: description of the second score,
|
46 |
Examples:
|
47 |
Examples should be written in doctest format, and should illustrate how
|
48 |
to use the function.
|
49 |
|
50 |
-
>>>
|
51 |
-
>>> results = my_new_module.compute(
|
|
|
|
|
|
|
52 |
>>> print(results)
|
53 |
-
{'
|
54 |
"""
|
55 |
|
56 |
-
# TODO: Define external resources urls if needed
|
57 |
-
BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
|
58 |
-
|
59 |
|
60 |
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
|
61 |
class roc_auc_macro(evaluate.Metric):
|
@@ -71,14 +79,14 @@ class roc_auc_macro(evaluate.Metric):
|
|
71 |
inputs_description=_KWARGS_DESCRIPTION,
|
72 |
# This defines the format of each prediction and reference
|
73 |
features=datasets.Features({
|
74 |
-
'
|
75 |
'references': datasets.Value('int64'),
|
76 |
}),
|
77 |
# Homepage of the module for documentation
|
78 |
-
homepage="
|
79 |
# Additional links to the codebase or references
|
80 |
-
codebase_urls=["
|
81 |
-
reference_urls=["
|
82 |
)
|
83 |
|
84 |
def _download_and_prepare(self, dl_manager):
|
@@ -88,8 +96,6 @@ class roc_auc_macro(evaluate.Metric):
|
|
88 |
|
89 |
def _compute(self, predictions, references):
|
90 |
"""Returns the scores"""
|
91 |
-
# TODO: Compute the different scores of the module
|
92 |
-
accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
|
93 |
return {
|
94 |
-
"
|
95 |
}
|
|
|
15 |
|
16 |
import evaluate
|
17 |
import datasets
|
18 |
+
from sklearn.metrics import roc_auc_score
|
19 |
|
20 |
|
21 |
# TODO: Add BibTeX citation
|
22 |
_CITATION = """\
|
23 |
@InProceedings{huggingface:module,
|
24 |
+
title = {Roc AUC Macro},
|
25 |
+
authors={Mike Frantz},
|
26 |
+
year={2022}
|
27 |
}
|
28 |
"""
|
29 |
|
30 |
# TODO: Add description of the module here
|
31 |
_DESCRIPTION = """\
|
32 |
+
This module makes scikit-learn's roc_auc_score with average='macro'
|
33 |
+
available as a metric in the hub for use in multi-class or multi-label
|
34 |
+
classification.
|
35 |
"""
|
36 |
|
37 |
|
38 |
# TODO: Add description of the arguments of the module here
|
39 |
_KWARGS_DESCRIPTION = """
|
40 |
+
Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores.
|
41 |
+
Calculates metrics for each label, and find their unweighted mean. This does not take label imbalance into account.
|
42 |
+
(FROM https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html)
|
43 |
+
|
44 |
Args:
|
45 |
+
scores: An array of shape (n_samples, n_classes) of probability estimates.
|
46 |
+
In the multi-class case, the probabilities must sum to 1 across classes.
|
47 |
+
In the multi-label case, the probabilities, each output corresponds
|
48 |
+
to a binary decision for a particular label.
|
49 |
references: list of reference for each prediction. Each
|
50 |
+
reference should be an array of shape (n_sampls, n_classes) with values
|
51 |
+
of 0 or 1.
|
52 |
Returns:
|
53 |
+
roc_auc_macro
|
|
|
54 |
Examples:
|
55 |
Examples should be written in doctest format, and should illustrate how
|
56 |
to use the function.
|
57 |
|
58 |
+
>>> roc_auc_macro = evaluate.load("mgfrantz/roc_auc_macro")
|
59 |
+
>>> results = my_new_module.compute(
|
60 |
+
references=[[1, 0, 1], [0, 1, 0]],
|
61 |
+
scores=[[.8, .2, .9], [.1, .8, .6]]
|
62 |
+
)
|
63 |
>>> print(results)
|
64 |
+
{'roc_auc_macro': 1.0}
|
65 |
"""
|
66 |
|
|
|
|
|
|
|
67 |
|
68 |
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
|
69 |
class roc_auc_macro(evaluate.Metric):
|
|
|
79 |
inputs_description=_KWARGS_DESCRIPTION,
|
80 |
# This defines the format of each prediction and reference
|
81 |
features=datasets.Features({
|
82 |
+
'scores': datasets.Value('float64'),
|
83 |
'references': datasets.Value('int64'),
|
84 |
}),
|
85 |
# Homepage of the module for documentation
|
86 |
+
homepage="N/A",
|
87 |
# Additional links to the codebase or references
|
88 |
+
codebase_urls=["N/A"],
|
89 |
+
reference_urls=["N/A"]
|
90 |
)
|
91 |
|
92 |
def _download_and_prepare(self, dl_manager):
|
|
|
96 |
|
97 |
def _compute(self, predictions, references):
|
98 |
"""Returns the scores"""
|
|
|
|
|
99 |
return {
|
100 |
+
"roc_auc_macro": roc_auc_score(references, scores, average='macro'),
|
101 |
}
|