[Python] the ouput result is <UNK> models--llava-hf--llava-v1.6-mistral-7b-hf

I have models--llava-hf--llava-v1.6-mistral-7b-hf to extract info and text from image, but all the output result get the inforamtion shown in the image?" [/INST]

import torch
from PIL import Image
import requests
import numpy as np

# Load the model and processor

# device=torch.device("cuda" if torch.cuda.is_available() else "cpu")

if torch.cuda.is_available():
device = torch.device("cuda:0")
print("running on the GPU")
else:
device = torch.device("cpu")
print("running on the CPU")

processor = LlavaNextProcessor.from_pretrained(r"D:\DL_ML_Projects\huggingface\hub\models--llava-hf--llava-v1.6-mistral-7b-hf\snapshots\ed45ac3a134a109ea7148e5ba3639d8972cad49c", )
model = LlavaNextForConditionalGeneration.from_pretrained(r"D:\DL_ML_Projects\huggingface\hub\models--llava-hf--llava-v1.6-mistral-7b-hf\snapshots\ed45ac3a134a109ea7148e5ba3639d8972cad49c", torch_dtype=torch.float16, low_cpu_mem_usage=True)
model= model.to("cuda:0")


image = Image.open(r"page_354.jpg").convert("RGB")

conversation = [ { "role": "user", "content": [ {"type": "text", "text": "get the inforamtion shown in the image?"}, {"type": "image"}, ], }, ]

prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)

# Prepare the input
inputs = processor(text=prompt,images=image, return_tensors="pt").to("cuda:0")
# print(inputs)
# print(f"Input IDs: {inputs['input_ids'][:10]}")
# Generate the caption
output = model.generate(**inputs, pad_token_id=model.config.eos_token_id,max_new_tokens=512)
# Print the caption
print(processor.decode(output[0], skip_special_tokens=False))```

Continue reading...