diff --git a/src/evaluate/evaluator/__init__.py b/src/evaluate/evaluator/__init__.py
index a2fe4be8..7510d5ea 100644
--- a/src/evaluate/evaluator/__init__.py
+++ b/src/evaluate/evaluator/__init__.py
@@ -103,10 +103,12 @@ def check_task(task: str) -> Dict:
     """
     if task in TASK_ALIASES:
         task = TASK_ALIASES[task]
-    if not check_pipeline_task(task):
-        raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.")
-    if task in SUPPORTED_EVALUATOR_TASKS.keys() and task in SUPPORTED_PIPELINE_TASKS.keys():
+    if task in SUPPORTED_EVALUATOR_TASKS:
         return SUPPORTED_EVALUATOR_TASKS[task]
+    try:
+        check_pipeline_task(task)
+    except KeyError as e:
+        raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") from e
     raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.")
 
 
diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py
index 259b5c7b..1e3eeac4 100644
--- a/tests/test_evaluator.py
+++ b/tests/test_evaluator.py
@@ -921,6 +921,16 @@ def setUp(self):
         self.pipe = DummyText2TextGenerationPipeline()
         self.evaluator = evaluator("text2text-generation")
 
+    def test_text2text_evaluator_tasks_can_be_created(self):
+        for task, expected_class_name in [
+            ("text2text-generation", "Text2TextGenerationEvaluator"),
+            ("summarization", "SummarizationEvaluator"),
+            ("translation", "TranslationEvaluator"),
+        ]:
+            e = evaluator(task)
+            self.assertEqual(type(e).__name__, expected_class_name)
+            self.assertEqual(e.task, task)
+
     def test_pipe_init(self):
         results = self.evaluator.compute(
             model_or_pipeline=self.pipe,