[Bug] NotImplementedError: Images are not yet supported in JSON mode.

[wangweiguo-123]

What happened?

i try to run the multi-model of llama:32b to classify the breed of dog in picture with dspy, however, when i input an image into the module(dspy.chainofthought) , the "NotImplementedError: Images are not yet supported in JSON mode" occur.
i can not figure out the reason why....

Steps to reproduce

import dspy
from dspy.adapters.types import image
import subprocess
import platform
import time

#start ollama serve on local
def start_ollama():
try:
if platform.system() != 'Windows':
return False
process = subprocess.Popen(
'ollama start',
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
time.sleep(3)
return True
except Exception as e:
return False

load the llama:34b mulit-model

def configure_local_model():
try:
lm = dspy.LM(
model="ollama_chat/llava:34b",
base_url="http://localhost:11434"
)
dspy.settings.configure(lm=lm)
return True
except Exception as e:
print(f"模型配置失败: {str(e)}")
return False

class DogPictureSignature(dspy.Signature):
"""Output the dog breed of the dog in the image."""
image_1: dspy.Image = dspy.InputField(desc="An image of a dog")
answer: str = dspy.OutputField(desc="The dog breed of the dog in the image")

def main():
start_ollama()
configure_local_model()
lm = dspy.settings.lm
image_url = "https://picsum.photos/id/237/200/300.jpg"
classify = dspy.ChainOfThought(DogPictureSignature)
image =dspy.Image.from_url(image_url)
classify(image_1 = image)
print(classify.answer)

if name == "main":
main()

DSPy version

2.6.11