⚡️ Speed up _run_llm_or_chain() by 35% in libs/langchain/langchain/smith/evaluation/runner_utils.py

[codeflash-ai]

📄 _run_llm_or_chain() in libs/langchain/langchain/smith/evaluation/runner_utils.py

📈 Performance went up by 35% (0.35x faster)

⏱️ Runtime went down from 105.60μs to 78.10μs

Explanation and details

(click to show)

Optimizing this code won't significantly affect its run-time performance considering the type of operations involved few tips to improve would include reducing the usage of isinstance-checks and handling the use of defaults for Optional parameters efficiently. We can also take advantage of Exception chaining and use a unified RunnableConfig to avoid code repetitions.

This new implementation explicitly handles defaults for Optional parameters and also reduces some of the redundant code in the _run_llm and _run_chain functions.

Correctness verification

The new optimized code was tested for correctness. The results are listed below.

✅ 0 Passed − ⚙️ Existing Unit Tests

✅ 0 Passed − 🎨 Inspired Regression Tests

✅ 2 Passed − 🌀 Generated Regression Tests

(click to show generated tests)

# imports
import pytest
from typing import Any, Callable, Dict, List, Optional, Union
from unittest.mock import Mock, patch

# Assuming the existence of certain classes and functions based on the provided code snippet
class BaseLanguageModel:
    def invoke(self, prompt_or_messages, config):
        pass

class Chain:
    def invoke(self, value, config):
        pass

class Runnable:
    def invoke(self, inputs, config):
        pass

class BaseMessage:
    pass

class RunnableConfig(dict):
    pass

class Example:
    def __init__(self, inputs, id=None):
        self.inputs = inputs
        self.id = id

class Callbacks:
    pass

class MCF:
    pass

class EvalError(Exception):
    def __init__(self, Error):
        self.Error = Error

class InputFormatError(Exception):
    pass

# function to test (provided in the original code snippet)
# ... (omitting the provided _run_llm, _run_chain, and _run_llm_or_chain functions for brevity)

# unit tests

# Test valid execution with a language model
def test_run_llm_or_chain_with_valid_language_model():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    llm = BaseLanguageModel()
    llm.invoke = Mock(return_value="Processed text")
    output = _run_llm_or_chain(example, config, llm_or_chain_factory=lambda: llm)
    assert output == "Processed text", "Should return the result from the language model"

# Test valid execution with a chain
def test_run_llm_or_chain_with_valid_chain():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    chain = Chain()
    chain.invoke = Mock(return_value={"processed": "text"})
    output = _run_llm_or_chain(example, config, llm_or_chain_factory=lambda: chain)
    assert output == {"processed": "text"}, "Should return the result from the chain"

# Test handling of exceptions during language model invocation
def test_run_llm_or_chain_with_language_model_exception():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    llm = BaseLanguageModel()
    llm.invoke = Mock(side_effect=Exception("Test exception"))
    with patch('logger.warning') as mock_logger:
        output = _run_llm_or_chain(example, config, llm_or_chain_factory=lambda: llm)
        assert isinstance(output, EvalError), "Should return an EvalError on exception"
        mock_logger.assert_called_once()

# Test handling of exceptions during chain invocation
def test_run_llm_or_chain_with_chain_exception():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    chain = Chain()
    chain.invoke = Mock(side_effect=Exception("Test exception"))
    with patch('logger.warning') as mock_logger:
        output = _run_llm_or_chain(example, config, llm_or_chain_factory=lambda: chain)
        assert isinstance(output, EvalError), "Should return an EvalError on exception"
        mock_logger.assert_called_once()

# Test with invalid input_mapper return type
def test_run_llm_or_chain_with_invalid_input_mapper():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    input_mapper = Mock(return_value=42)  # Invalid return type
    llm = BaseLanguageModel()
    with pytest.raises(InputFormatError):
        _run_llm_or_chain(example, config, llm_or_chain_factory=lambda: llm, input_mapper=input_mapper)

# Test with unexpected llm_or_chain_factory type
def test_run_llm_or_chain_with_unexpected_factory_type():
    example = Example(inputs={"text": "Hello, world!"})
    config = RunnableConfig(callbacks=Callbacks(), tags=["test"], metadata={"key": "value"})
    llm_or_chain_factory = Mock()  # Not a BaseLanguageModel or Chain instance
    with pytest.raises(AttributeError):
        _run_llm_or_chain(example, config, llm_or_chain_factory=llm_or_chain_factory)

# Add more tests here to cover other scenarios and edge cases

[aphexcx]

Another possible code replacer bug, this is also an underscore function but not a class method