Fix typing errors detected by a newer version of mypy (stanford-crfm#1841)

Author: yifanmai

scripts/data_overlap/load_documents.py

@@ -136,4 +136,4 @@ def get_raw_document_iterator(file_path: str) -> Iterator[str]:
 
 def get_custom_document_iterator(file_path: str) -> Iterator[str]:
     """Define your own document reading method"""
-    pass
+    raise NotImplementedError()

setup.cfg

@@ -174,6 +174,10 @@ ignore = E203,E231,E731,W503,W605
 # Settings for Mypy: static type checker for Python 3
 [mypy]
 ignore_missing_imports = True
+# TODO(#1831): Change this to True
+check_untyped_defs = False
+# TODO(#1831): Change this to True
+disallow_untyped_defs = False
 
 [tool:pytest]
 addopts =

src/helm/benchmark/augmentations/contrast_sets_perturbation.py

@@ -83,4 +83,4 @@ def apply(self, instance: Instance, seed: Optional[int] = None) -> Instance:
         )
 
     def perturb(self, text: str, rng: Random) -> str:  # we need this since parent method is abstract
-        pass
+        raise NotImplementedError("Should never be called since apply() was overridden")

src/helm/benchmark/augmentations/gender_perturbation.py

@@ -99,7 +99,7 @@ def __init__(
         source_class: str,
         target_class: str,
         mapping_file_path: Optional[str] = None,
-        mapping_file_genders: List[str] = None,
+        mapping_file_genders: Optional[List[str]] = None,
         bidirectional: bool = False,
     ):
         """Initialize the gender perturbation.

src/helm/benchmark/run.py

@@ -49,10 +49,12 @@ def run_entries_to_run_specs(
             adapter_spec: AdapterSpec = run_spec.adapter_spec
             if max_eval_instances is not None and adapter_spec.max_eval_instances is None:
                 adapter_spec = replace(adapter_spec, max_eval_instances=max_eval_instances)
-            if num_train_trials is not None or adapter_spec.max_train_instances == 0:
-                adapter_spec = replace(
-                    adapter_spec, num_train_trials=1 if adapter_spec.max_train_instances == 0 else num_train_trials
-                )
+
+            if adapter_spec.max_train_instances == 0:
+                adapter_spec = replace(adapter_spec, num_train_trials=1)
+            elif num_train_trials is not None:
+                adapter_spec = replace(adapter_spec, num_train_trials=num_train_trials)
+
             run_spec = replace(run_spec, adapter_spec=adapter_spec)
 
             # Append groups

src/helm/benchmark/run_expander.py

@@ -67,35 +67,6 @@ def sanitize(value):
         ]
 
 
-class ReplaceRunSpecValueRunExpander(RunExpander):
-    """
-    Replace a single field (e.g., max_train_instances) with a list of values (e.g., 0, 1, 2).
-    """
-
-    def __init__(self, value):
-        """
-        `value` is either the actual value to use or a lookup into the values dict.
-        """
-        self.name = type(self).name
-        if value in type(self).values_dict:
-            self.values = type(self).values_dict[value]
-        else:
-            self.values = [value]
-
-    def expand(self, run_spec: RunSpec) -> List[RunSpec]:
-        def sanitize(value):
-            return str(value).replace("/", "_")
-
-        return [
-            replace(
-                run_spec,
-                name=f"{run_spec.name},{self.name}={sanitize(value)}",
-                metrics=value,
-            )
-            for value in self.values
-        ]
-
-
 class InstructionsRunExpander(RunExpander):
     """
     Set the instructions of the prompt.
@@ -530,7 +501,7 @@ def gender(
     source_class: str,
     target_class: str,
     mapping_file_path: Optional[str] = None,
-    mapping_file_genders: Tuple[str] = None,
+    mapping_file_genders: Optional[Tuple[str]] = None,
     bidirectional: bool = False,
 ) -> PerturbationSpec:
     return PerturbationSpec(

src/helm/common/cache.py

@@ -1,4 +1,5 @@
-from abc import ABC, abstractmethod
+from abc import abstractmethod
+import contextlib
 from dataclasses import dataclass
 import json
 from typing import Dict, Callable, Generator, Iterable, Optional, Tuple
@@ -103,19 +104,13 @@ def cache_stats_key(self) -> str:
         return self.main.cache_stats_key
 
 
-class KeyValueStore(ABC):
+class KeyValueStore(contextlib.AbstractContextManager):
     """Key value store that persists writes."""
 
     @property
     def path(self):
         return self._path
 
-    def __enter__(self) -> "KeyValueStore":
-        pass
-
-    def __exit__(self, exc_type, exc_value, traceback) -> None:
-        pass
-
     @abstractmethod
     def contains(self, key: Dict) -> bool:
         pass
@@ -150,11 +145,9 @@ def __init__(self, path: str):
 
     def __enter__(self) -> "_SqliteKeyValueStore":
         self._sqlite_dict.__enter__()
-        super().__enter__()
         return self
 
     def __exit__(self, exc_type, exc_value, traceback) -> None:
-        super().__exit__(exc_type, exc_value, traceback)
         self._sqlite_dict.__exit__(exc_type, exc_value, traceback)
 
     def contains(self, key: Dict) -> bool:
@@ -204,11 +197,10 @@ def __init__(self, uri: str, collection_name: str):
         super().__init__()
 
     def __enter__(self) -> "_MongoKeyValueStore":
-        super().__enter__()
         return self
 
     def __exit__(self, exc_type, exc_value, traceback) -> None:
-        super().__exit__(exc_type, exc_value, traceback)
+        return
 
     def _canonicalize_key(self, key: Dict) -> SON:
         serialized = json.dumps(key, sort_keys=True)

src/helm/proxy/clients/ai21_client.py

@@ -129,7 +129,7 @@ def parse_token(raw: Dict, first: bool) -> Token:
                 top_logprobs=top_logprobs,
             )
 
-        def parse_sequence(raw: Dict, first: bool, finish_reason: Dict = None) -> Sequence:
+        def parse_sequence(raw: Dict, first: bool, finish_reason: Optional[Dict] = None) -> Sequence:
             text = raw["text"]
             tokens = [parse_token(token, first and i == 0) for i, token in enumerate(raw["tokens"])]
             logprob = sum(token.logprob for token in tokens)

src/helm/proxy/services/service.py

@@ -108,6 +108,7 @@ def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIReque
         """Get toxicity scores for a batch of text."""
         pass
 
+    @abstractmethod
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         """Get responses to a critique request."""
         pass