[Python] Running TinyLlama with llama.cpp via Python subprocess hangs

I’m working on an offline RAG-based chatbot for a telemedicine project. The idea is:

Store Q&A pairs in a local SQLite database.

Use FAISS for semantic search to fetch context.

Pass the context + user question into a TinyLlama model (1.1B, GGUF) via llama.cpp.

Run everything fully offline. My setup : OS: Windows 11

Python: 3.12

llama.cpp: compiled on Windows, CLI works fine in terminal

Model: tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf(using this because i want this to run inside an app)

Running directly from terminal: works fine (response comes in seconds).

Running via Python subprocess: it hangs for minutes and sometimes throws decoding errors. I am working with LLMs and vector database python for the first time for thisproject that's why i heavily depend on AI tools to code this ,not understanding where is the error .

`

`your text`


import faiss
import sqlite3
import numpy as np
import os
import subprocess
import sys
from sentence_transformers import SentenceTransformer

# --- 1. Define file paths ---
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
faiss_index_path = os.path.join(base_dir, 'python-microservices', 'embeddings', 'kb.index')
db_file_path = os.path.join(base_dir, 'backend', 'data', 'kb.sqlite')

# --- 2. Load the Local KB and Embedding Model ---
try:
print("Loading FAISS index and local KB for offline chat...")
index = faiss.read_index(faiss_index_path)
conn = sqlite3.connect(db_file_path)
cursor = conn.cursor()
model = SentenceTransformer('all-MiniLM-L6-v2')
print("KB and model loaded successfully! Ready for offline chat.")
except Exception as e:
print(f"Error loading local KB files: {e}")
print("Please ensure you have run 'data_loader.py' and 'update_faiss_index.py' first.")
sys.exit(1)

def get_context_from_index(query: str, k=3):
"""
Takes a user query, searches the FAISS index, and retrieves
the top k most relevant text chunks from the local SQLite DB.
"""
# Convert the user query into an embedding
query_embedding = model.encode([query])
query_embedding = np.array(query_embedding).astype('float32')

# Search the FAISS index for the most similar embeddings
distances, indices = index.search(query_embedding, k)

# Retrieve the original text from the SQLite database using the indices
retrieved_texts = []
for doc_id in indices[0]:
# FAISS index is 0-based, SQLite IDs start from 1.
cursor.execute("SELECT question, answer FROM knowledge_base WHERE id = ?", (int(doc_id) + 1,))
result = cursor.fetchone()
if result:
retrieved_texts.append(f"Question: {result[0]}\nAnswer: {result[1]}")

return "\n---\n".join(retrieved_texts)

def get_llama_response_offline(prompt: str):
"""
This function calls the llama.cpp model with the RAG prompt.
"""
current_script_path = os.path.abspath(__file__)
telemedicine_rag_dir = os.path.dirname(os.path.dirname(current_script_path))
parent_dir = os.path.dirname(telemedicine_rag_dir)
llama_base_dir = os.path.join(parent_dir, 'LLMTools')

llama_executable_path = os.path.join(llama_base_dir, 'llama.cpp', 'build', 'bin', 'Release', 'llama-cli.exe')
llama_model_path = os.path.join(llama_base_dir, 'llama.cpp', 'build', 'bin', 'Release', 'tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf')

try:
command = [
llama_executable_path,
'-m', llama_model_path,
'-p', prompt,
'-n', '256',
'--temp', '0.1',
'--no-warmup'
]

result = subprocess.run(
command,
capture_output=True,
text=True,
check=True,
encoding="utf-8",
errors="replace"
)
return result.stdout.strip()
except FileNotFoundError:
return "Error: Llama.cpp executable or TinyLlama model not found. Please check paths."
except subprocess.CalledProcessError as e:
return f"Error from llama.cpp: {e.stderr}"

def run_chat_session():
"""
Simulates a full chat session with the user.
"""
print("Offline Chatbot is ready. Type your health query (type 'exit' to quit).")
while True:
user_query = input("\nYou: ")
if user_query.lower() == 'exit':
break

# 1. Retrieve the context
context = get_context_from_index(user_query)

# 2. Build the RAG prompt
rag_prompt = f"""You are a medical assistant for Nabha Civil Hospital. Answer the user's question only based on the provided context. If the answer is not in the context, say "I cannot provide an answer based on my current knowledge."

Context:
{context}

User Question: {user_query}

Answer:
"""
# 3. Get the LLM response
response = get_llama_response_offline(rag_prompt)
print(f"\nBot: {response}")

if __name__ == "__main__":
run_chat_session()
conn.close()

Continue reading...