Enable check_untyped_defs for mypy (stanford-crfm#1851)

Author: yifanmai

scripts/data_overlap/test/test_compute_data_overlap_metrics.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import List, DefaultDict, Set
 
 from collections import defaultdict

scripts/fact_completion/create_benchmark.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 """
 This script samples triples to use and constructs sentences from templates.
 

setup.cfg

@@ -174,8 +174,7 @@ ignore = E203,E231,E731,W503,W605
 # Settings for Mypy: static type checker for Python 3
 [mypy]
 ignore_missing_imports = True
-# TODO(#1831): Change this to True
-check_untyped_defs = False
+check_untyped_defs = True
 # TODO(#1831): Change this to True
 disallow_untyped_defs = False
 

src/helm/benchmark/adaptation/adapters/test_generation_adapter.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import List
 
 from helm.benchmark.scenarios.scenario import CORRECT_TAG, create_scenario, Instance, Reference, Input, Output

src/helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import List
 
 from helm.common.tokenization_request import TokenizationToken

src/helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from helm.benchmark.scenarios.scenario import Instance, Input, Output, Reference, CORRECT_TAG
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from .adapter_factory import AdapterFactory, ADAPT_MULTIPLE_CHOICE_JOINT

src/helm/benchmark/augmentations/contraction_expansion_perturbation.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import Dict
 import re
 

src/helm/benchmark/augmentations/test_perturbation.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import List
 
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference

src/helm/benchmark/metrics/disinformation_metrics.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 """Diversity metrics for the disinformation scenario."""
 
 import json

src/helm/benchmark/metrics/summac/model_summac.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 ###############################################
 # Source: https://github.com/tingofurro/summac
 ###############################################

src/helm/benchmark/metrics/test_bias_metrics.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from dataclasses import dataclass
 from typing import Callable, List, Optional
 

src/helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from unittest.mock import MagicMock
 
 from transformers import GPT2TokenizerFast

src/helm/benchmark/presentation/create_plots.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import argparse
 from collections import defaultdict
 from dataclasses import dataclass

src/helm/benchmark/presentation/summarize.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 """Reads the output of the benchmark runs and produces:
 - JSON files for the frontend
 - Tables for the paper
@@ -827,7 +828,7 @@ def create_group_table(
                     header_name += " (" + perturbation_field.get_short_display_name() + ")"
                     description += (
                         "\n- Perturbation "
-                        + perturbation_field.display_name
+                        + (perturbation_field.display_name or perturbation_field.name)
                         + ": "
                         + (perturbation_field.description or "???")
                     )

src/helm/benchmark/presentation/test_create_plots.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from helm.common.general import asdict_without_nones
 from helm.benchmark.presentation.table import Table, Cell, HeaderCell
 from helm.benchmark.presentation.create_plots import parse_table

src/helm/benchmark/run_expander.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from abc import ABC, abstractmethod
 from dataclasses import replace
 from typing import Any, List, Dict, Optional, Tuple, Type

src/helm/benchmark/scenarios/code_scenario_helper.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 """Code formatting helper functions for the code scenario.
 
 Sourced from https://github.com/hendrycks/apps

src/helm/benchmark/scenarios/legal_summarization_scenario.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from pathlib import Path
 
 from typing import List, Optional, Any

src/helm/benchmark/scenarios/lsat_qa_scenario.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import json
 from typing import List

src/helm/benchmark/scenarios/msmarco_scenario.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import csv
 import os
 import random

src/helm/benchmark/scenarios/numeracy_scenario.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 # flake8: noqa
 from collections import defaultdict
 from dataclasses import dataclass, field

src/helm/benchmark/scenarios/the_pile_scenario.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import json
 import csv

src/helm/benchmark/server.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 """
 Starts a local HTTP server to display benchmarking assets.
 """

src/helm/benchmark/test_data_preprocessor.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from typing import List
 
 from helm.benchmark.augmentations.data_augmenter import DataAugmenterSpec

src/helm/benchmark/window_services/test_cohere_window_service.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import shutil
 import tempfile

src/helm/common/cache.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from abc import abstractmethod
 import contextlib
 from dataclasses import dataclass

src/helm/proxy/clients/openai_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 from dataclasses import replace, asdict
 from typing import Any, Dict, List, Optional, cast
 

src/helm/proxy/clients/palmyra_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import json
 import requests
 from typing import Any, Dict, List

src/helm/proxy/clients/perspective_api_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import threading
 from dataclasses import asdict
 from typing import List, Dict, Optional

src/helm/proxy/clients/test_anthropic_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import tempfile
 from typing import List

src/helm/proxy/clients/test_huggingface_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import pytest
 import tempfile

src/helm/proxy/clients/test_ice_tokenizer_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import tempfile
 from typing import List

src/helm/proxy/clients/test_yalm_tokenizer_client.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import tempfile
 from typing import List

src/helm/proxy/clients/yalm_tokenizer/yalm_tokenizer.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import importlib_resources as resources
 import torch
 import sentencepiece as spm

src/helm/proxy/server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+# mypy: check_untyped_defs = False
 
 """
 Starts a REST server for the frontend to interact with.

src/helm/proxy/services/test_remote_service.py

@@ -1,3 +1,4 @@
+# mypy: check_untyped_defs = False
 import os
 import random
 import shutil