[Python] OpenAi api | How can I answer a question with an "uploaded" PDF

I’m working on a Python project where I use OpenAI's GPT-4 API to analyze data from screenshots and PDFs.

My goal is to send the request with the screenshot and the PDF (Both in a single request) to the gpt-4 api.

The part with the screenshot sending works and I get no errors when I also attach the pdf. But it does not take into account anything written in that PDF.

Here are the relevant parts of the code:

from PyPDF2 import PdfReader


def encode_pdf_to_base64(pdf_path):
"""
Encodes PDF to base64
"""
try:
with open(pdf_path, "rb") as pdf_file:
return base64.b64encode(pdf_file.read()).decode('utf-8')
except FileNotFoundError:
print(f"Error: The file '{pdf_path}' was not found.")
return None


def analyze_with_pdf_and_screenshot(screenshot_path, pdf_path=None):
"""
Analyzes the Screenshot with the help of the PDF
"""
base64_image = encode_image(screenshot_path)
base64_pdf = encode_pdf_to_base64(pdf_path) if pdf_path else None


payload = {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Analyze the given Screenshot with the help of the attached PDF and answer with its help"
},
# Also tried this one and it didnt work
#{
# "type": "file",
# "file": {
# "name": os.path.basename(pdf_path),
# "type": "application/pdf",
# "content": base64_pdf
# }
#},
{
"type": "image_url",
"image_url": {
"url": f"data:image/jpeg;base64,{base64_image}"
}
},
],
"files": [
{
"name": os.path.basename(pdf_path),
"type": "application/pdf",
"content": base64_pdf
}
]
}
],
"max_tokens": 2500
}

try:
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {api_key}"
}
response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
response.raise_for_status()
result = response.json()
print("API-Answer:", json.dumps(result, indent=2))
return ""

except requests.RequestException as e:
print(f"Error analyzing file: {e}")
return ""

Continue reading...