[Python] Passing a prompt via python to llama.cpp

I have written the following code:

import subprocess
import os
import mysql.connector
from dotenv import load_dotenv


def run_llama_cpp(prompt):
# Specify the absolute path of the executable and current directory
executable_path = 'D:\\Code\\llama_cpp\\w64devkit\\w64devkit.exe'
current_directory = 'D:\\Code\\llama_cpp\\llama.cpp'
model_path = 'D:\\Code\\llama_cpp\\llama.cpp\\models\\llama-2-7b.Q8_0.gguf'

# Debug: Print the current working directory
print("Current Working Directory:", os.getcwd())

# Change directory
os.chdir(current_directory)
print("Changed to Directory:", os.getcwd())

# Define the command
command = [
executable_path, # Use the absolute path of your executable
'./main',
'-ins',
'--color',
'-c', '1024',
'--temp', '0.7',
'--repeat_penalty', '1.1',
'-s', '42',
'-n', '-1',
'-m', model_path,
'-p', "'" , prompt , "'"
]

try:
print("Command: ", ' '.join(command))

process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate()

if stderr:
print("Standard Error Output:", stderr)

if process.returncode != 0:
print("Error running llama_cpp:", stderr)
return None
else:
title_start = stdout.find('[TITLE_START]') + len('[TITLE_START]')
title_end = stdout.find('[TITLE_END]')
description_start = stdout.find('[DESCRIPTION_START]') + len('[DESCRIPTION_START]')
description_end = stdout.find('[DESCRIPTION_END]')

title = stdout[title_start:title_end].strip()
description = stdout[description_start:description_end].strip()

return title, description
except FileNotFoundError as e:
print(f"File not found: {e}")
return None
except Exception as e:
print(f"An error occurred: {e}")
return None

load_dotenv()
db_connection = os.getenv('DB_CONNECTION')
db_host = os.getenv('DB_HOST')
db_port = int(os.getenv('DB_PORT'))
db_database = os.getenv('DB_DATABASE')
db_username = os.getenv('DB_USERNAME')
db_password = os.getenv('DB_PASSWORD')

connection = mysql.connector.connect(host=db_host, user=db_username, password=db_password, database=db_database, port=db_port)
cursor = connection.cursor(dictionary=True)
cursor.execute("SELECT * FROM posts")
prompts = cursor.fetchall()
cursor.close()
connection.close()

for prompt_entity in prompts:
str = prompt_entity['content']
content = "Generate headline for this content: '" + str + "'. Put the headline between [TITLE_START] and [TITLE_END]"

title, description = run_llama_cpp(content)
if title and description:
print("Content:", content)
print("Title:", title)
print("Description:", description)
print("####################################")



I am trying to pass my prompt directly via w64devkit.exe, however when I open the command on a regular terminal w64devkit opens its own terminal.



How to pass my prompt directly to llama.cpp and receive the answer?

Continue reading...