Fix pre-commit breakages (stanford-crfm#1902)

Author: yifanmai

requirements.txt

@@ -170,6 +170,7 @@ transformers==4.28.1
 trio==0.22.0
 trio-websocket==0.9.2
 typer==0.4.2
+types-Pillow==9.3.0.4
 types-pytz==2022.4.0.0
 types-redis==4.3.21.1
 types-requests==2.28.11.2

src/helm/common/images_utils.py

@@ -9,11 +9,11 @@
 from .general import is_url
 
 
-def open_image(image_location: str) -> Image:
+def open_image(image_location: str) -> Image.Image:
     """
     Opens image with the Python Imaging Library.
     """
-    image: Image
+    image: Image.Image
     if is_url(image_location):
         image = Image.open(requests.get(image_location, stream=True).raw)
     else:
@@ -24,7 +24,7 @@ def open_image(image_location: str) -> Image:
 def encode_base64(image_location: str, format="JPEG") -> str:
     """Returns the base64 representation of an image file."""
     image_file = io.BytesIO()
-    image: Image = open_image(image_location)
+    image: Image.Image = open_image(image_location)
     image.save(image_file, format=format)
     return base64.b64encode(image_file.getvalue()).decode("ascii")
 
@@ -36,7 +36,8 @@ def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optiona
     """
     if (width is not None and height is not None) or is_url(src):
         image = open_image(src)
-        resized_image = image.resize((width, height), Image.ANTIALIAS)
-        resized_image.save(dest)
+        if width is not None and height is not None:
+            image = image.resize((width, height), Image.ANTIALIAS)
+        image.save(dest)
     else:
         shutil.copy(src, dest)

src/helm/proxy/clients/huggingface_client.py

@@ -5,8 +5,6 @@
 from transformers.generation.stopping_criteria import (
     StoppingCriteria,
     StoppingCriteriaList,
-    STOPPING_CRITERIA_INPUTS_DOCSTRING,
-    add_start_docstrings,
 )
 from typing import Any, Dict, List, Optional
 
@@ -42,18 +40,17 @@ def resolve_alias(model_name: str) -> str:
 
 
 class StopAtSpecificTokenCriteria(StoppingCriteria):
-    def __init__(self, stop_sequence: List[int] = None):
+    def __init__(self, stop_sequence: List[int]):
         super().__init__()
         self.stop_sequence = stop_sequence
 
-    # @add_start_docstrings(STOPPING_CRITERIA_INPUTS_DOCSTRING)
     def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
         # Create a tensor from the stop_sequence
         stop_sequence_tensor = torch.tensor(self.stop_sequence, device=input_ids.device, dtype=input_ids.dtype)
 
         # Check if the current sequence ends with the stop_sequence
         current_sequence = input_ids[:, -len(self.stop_sequence) :]
-        return torch.all(current_sequence == stop_sequence_tensor).item()
+        return bool(torch.all(current_sequence == stop_sequence_tensor).item())
 
 
 class HuggingFaceServer:
@@ -88,14 +85,17 @@ def serve_request(self, raw_request: Dict[str, Any]):
         raw_request["output_scores"] = True
         top_k_per_token: int = raw_request["top_k_per_token"]
         del raw_request["top_k_per_token"]
+        stopping_criteria: Optional[StoppingCriteriaList] = None
         if len(raw_request["stop_sequences"]) > 0:
             stop_sequence_ids = self.tokenizer(
                 raw_request["stop_sequences"], return_token_type_ids=False, add_special_tokens=False
             )
-            assert len(stop_sequence_ids.input_ids) == 1, "Total number of stop words should be 1."
-            # assert len(stop_sequence_ids.input_ids[0]) == 1, "Total number of tokens in each stop word should be 1."
+            assert len(stop_sequence_ids.input_ids) == 1, "Total number of stop sequences should be 1."
             if len(stop_sequence_ids.input_ids[0]) == 1:
                 raw_request["eos_token_id"] = stop_sequence_ids.input_ids[0][0]
+            else:
+                stopping_criteria = StoppingCriteriaList()
+                stopping_criteria.append(StopAtSpecificTokenCriteria(stop_sequence=stop_sequence_ids.input_ids[0]))
             del raw_request["stop_sequences"]
 
         # Strip out irrelevant parameters
@@ -105,15 +105,11 @@ def serve_request(self, raw_request: Dict[str, Any]):
             if key not in ["engine", "prompt", "echo_prompt", "stop_sequences"]
         }
 
-        stopping_criteria = StoppingCriteriaList()
-        if stop_sequence_ids != None:
-            stopping_criteria.append(StopAtSpecificTokenCriteria(stop_sequence=stop_sequence_ids.input_ids[0]))
-
         # Use HuggingFace's `generate` method.
         output = self.model.generate(
             **encoded_input,
             **relevant_raw_request,
-            stopping_criteria=stopping_criteria if len(stop_sequence_ids.input_ids[0]) > 1 else None,
+            stopping_criteria=stopping_criteria,
         )
         sequences = output.sequences
         scores = output.scores

src/helm/proxy/clients/vision_language/idefics_client.py

@@ -10,6 +10,7 @@
 from helm.common.images_utils import open_image
 from helm.common.gpu_utils import get_torch_device_name
 from helm.common.hierarchical_logger import hlog
+from helm.common.media_object import TEXT_TYPE
 from helm.common.request import Request, RequestResult, Sequence, Token
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -93,12 +94,17 @@ def make_request(self, request: Request) -> RequestResult:
             exit_condition = processor.tokenizer(self.END_OF_UTTERANCE_TOKEN, add_special_tokens=False).input_ids
             generation_args["eos_token_id"] = exit_condition
 
-        multimodal_prompt: List[Union[str, Image]] = [
-            open_image(media_object.location)
-            if media_object.is_type("image") and media_object.location
-            else media_object.text
-            for media_object in request.multimodal_prompt.media_objects
-        ]
+        multimodal_prompt: List[Union[str, Image.Image]] = []
+        for media_object in request.multimodal_prompt.media_objects:
+
+            if media_object.is_type("image") and media_object.location:
+                multimodal_prompt.append(open_image(media_object.location))
+            elif media_object.is_type(TEXT_TYPE):
+                if media_object.text is None:
+                    raise ValueError("MediaObject of text type has missing text field value")
+                multimodal_prompt.append(media_object.text)
+            else:
+                raise ValueError(f"Unrecognized MediaObject type {media_object.type}")
         prompt_text: str = request.multimodal_prompt.text.replace(self.END_OF_UTTERANCE_TOKEN, " ")
 
         try: