[Python] Chatbot does not read web crawler data

I'm building a chatbot that is trained on data from three sources: PDF, Excel, and a website. I have a crawl.py file that crawls the website to get the contents of the page. The chatbot answers every question related to the PDF and Excel. But only answers some questions related to the website, not all.

The crawler adds the contents to text files inside a folder called /crawl. The data exists in the text files, but the chatbot seems to not receive all the data from the /crawl folder.

data.py: (Loads the crawl data and performs RAG)

CRAWL_OUTPUT_DIR = os.getenv("CRAWL_OUTPUT_DIR", "crawl")

def load_crawled_txt_files(output_dir="crawl"):
"""Load crawled text files with case-insensitive extension checking"""
docs = []
if not os.path.exists(output_dir):
print(f"Directory '{output_dir}' does not exist.")
return docs

for fname in os.listdir(output_dir):
# Case-insensitive check for .txt extension
if fname.lower().endswith(".txt"):
file_path = os.path.join(output_dir, fname)
try:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read().strip()
if content:
docs.append(Document(
page_content=content,
metadata={"source": fname}
))
except (UnicodeDecodeError, IOError) as e:
print(f"Error reading file '{fname}': {e}")

print(f"Loaded {len(docs)} crawled text documents.")
return docs

def load_crawled_txts(output_dir="crawl"):"""Alias for load_crawled_txt_files for backward compatibility"""return load_crawled_txt_files(output_dir)

def create_and_save_vectorstore_with_crawl(base_url, pdf_files=None):
"""Create vectorstore with better error handling"""
if pdf_files is Nonedf_files = get_pdf_files()

print(" Starting vector store creation...")

try:
# Crawl website for text data
print(f" Crawling website: {base_url}")
run_crawler(base_url, depth=2, output_dir=CRAWL_OUTPUT_DIR)
crawled_docs = load_crawled_txt_files(output_dir=CRAWL_OUTPUT_DIR)

# Load PDFs if available
pdf_docs = []
if pdf_files:
pdf_docs = load_pdfs(pdf_files)
else:
print("⚠️ No PDF files found, proceeding with web content only")

all_docs = pdf_docs + crawled_docs

if not all_docs:
raise ValueError("No documents loaded from either PDFs or web crawling")

chunks = split_text_chunks(all_docs)
store = create_vectorstore_from_chunks(chunks)
save_vectorstore(store)
return store

except Exception as e:
print(f"❌ Failed to create vector store: {e}")
raise

def remove_unwanted_elements(soup):
"""Remove navigation, footer, and header elements"""
for tag in soup(['nav', 'footer', 'header']):
tag.decompose()


Every data source works as expected, except for the crawling process. The issue probably lies in reading the text, since the files exist.

I could ask the chatbot about something in the website and it would answer, whereas when I would ask about data from a text file produced by the crawl.py, the chatbot wouldn't answer.

Continue reading...