Spaces:
Runtime error
Runtime error
fix module type = 'metric'
Browse files- lingo_judge_metric.py +5 -0
lingo_judge_metric.py
CHANGED
|
@@ -39,9 +39,12 @@ Returns:
|
|
| 39 |
class LingoJudgeMetric(evaluate.Metric):
|
| 40 |
def _info(self):
|
| 41 |
return evaluate.MetricInfo(
|
|
|
|
|
|
|
| 42 |
description=_DESCRIPTION,
|
| 43 |
citation=_CITATION,
|
| 44 |
inputs_description=_KWARGS_DESCRIPTION,
|
|
|
|
| 45 |
features=datasets.Features(
|
| 46 |
{
|
| 47 |
"questions": datasets.Value("string"),
|
|
@@ -49,6 +52,7 @@ class LingoJudgeMetric(evaluate.Metric):
|
|
| 49 |
"references": datasets.Sequence(datasets.Value("string")),
|
| 50 |
}
|
| 51 |
),
|
|
|
|
| 52 |
)
|
| 53 |
|
| 54 |
def _download_and_prepare(self, dl_manager):
|
|
@@ -56,4 +60,5 @@ class LingoJudgeMetric(evaluate.Metric):
|
|
| 56 |
self.scorer = LingoJudge().eval().to(self.device)
|
| 57 |
|
| 58 |
def _compute(self, questions, predictions, references):
|
|
|
|
| 59 |
return self.scorer.compute(questions, references, predictions)
|
|
|
|
| 39 |
class LingoJudgeMetric(evaluate.Metric):
|
| 40 |
def _info(self):
|
| 41 |
return evaluate.MetricInfo(
|
| 42 |
+
# This is the description that will appear on the modules page.
|
| 43 |
+
module_type="metric",
|
| 44 |
description=_DESCRIPTION,
|
| 45 |
citation=_CITATION,
|
| 46 |
inputs_description=_KWARGS_DESCRIPTION,
|
| 47 |
+
# This defines the format of each question, prediction, and reference.
|
| 48 |
features=datasets.Features(
|
| 49 |
{
|
| 50 |
"questions": datasets.Value("string"),
|
|
|
|
| 52 |
"references": datasets.Sequence(datasets.Value("string")),
|
| 53 |
}
|
| 54 |
),
|
| 55 |
+
reference_urls=["https://github.com/wayveai/LingoQA"],
|
| 56 |
)
|
| 57 |
|
| 58 |
def _download_and_prepare(self, dl_manager):
|
|
|
|
| 60 |
self.scorer = LingoJudge().eval().to(self.device)
|
| 61 |
|
| 62 |
def _compute(self, questions, predictions, references):
|
| 63 |
+
"""Returns the scores"""
|
| 64 |
return self.scorer.compute(questions, references, predictions)
|