[Python] All Class Names Show as [Unnamed] When Parsing DreamBot JavaDoc API JSON—Why Isn’t...

I’m building a Java tool that loads a JSON representation of the DreamBot JavaDocs, intending to process its entire API. The tool is designed to extract all relevant structural elements, including: • Class/interface/enum names • Fields, methods, and constructors • Annotations • Superclass and interfaces • Inheritance relationships • Class URLs

I’ve written a full web scraper using Python + Playwright + BeautifulSoup to crawl and extract this data from the DreamBot JavaDoc pages. The goal is to produce a full API model in JSON format for offline analysis and auto-generation tasks.

Goal

Parse DreamBot’s JavaDoc documentation into a structured JSON file with complete metadata for each class/interface, including the name. Example base page:

https://dreambot.org/javadocs/org/dreambot/api/methods/skills/Skills.html

Problem

Although the scraper successfully collects 524 API entries, every class ends up being displayed as [Unnamed] during runtime in my Java code:

System.out.println(" Dumping all available class names:");
for (DreamBotAPIModel.APIClass c : APIAnalyzer.getApiClasses()) {
System.out.println(" - " + (c.name != null ? c.name : "[Unnamed]"));


This produces:

Loaded API classes: 524 Dumping all available class names:

• [Unnamed]
• [Unnamed] ... (524 times) Unnamed classes: 524 ❌ Could not find any class containing 'bank'

} In the JSON, each object is missing a "name" field value despite other fields (like "methods", "url", and "superclass") being present and correctly formatted.

name_tag = soup.find("h2")
if name_tag:
name = name_tag.text.strip()
class_data["name"] = name
else:
class_data["name"] = None


The scraper uses soup.find("h2") assuming the contains the class name (e.g., Class Skills, Interface MethodContext, etc.), then strips the prefix and stores the actual name.

What I’ve Tried • Validated the JSON file loads properly in Java. • Manually confirmed all 524 entries are accessible programmatically. • Cleaned Unicode and whitespace from scraped output. • Verified URLs and method lists are intact—just missing the class names. • Exported CSV as a secondary view—name still blank in all rows.

What I’d Like to Know • Is the correct tag to pull from on JavaDoc pages like DreamBot’s? • Could DreamBot’s JavaDoc HTML structure differ from typical JavaDocs in a way that breaks soup.find("h2")? • Is there a more robust tag/strategy to capture the class/interface/enum name reliably? • Any ideas why name is consistently missing even when the scraper works for other fields?

Below is the full scraper logic if that helps. Any help or insights would be greatly appreciated, thanks in advance!

# -*- coding: utf-8 -*-
import asyncio
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
import json
import aiofiles
import os
import sqlite3
import hashlib
import csv
import re
from datetime import datetime
from tqdm.asyncio import tqdm_asyncio
from urllib.parse import urljoin

BASE_URL = "https://dreambot.org/javadocs/"
TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M%S")
OUTPUT_DIR = "data/output"
LOG_FILE = "data/logs/scraper.log"
DB_FILE = "data/db/dreambot_api.db"
HASH_JSON = f"{OUTPUT_DIR}/class_hashes.json"
OUTPUT_JSON = f"{OUTPUT_DIR}/dreambot_api_{TIMESTAMP}.json"
OUTPUT_CSV = f"{OUTPUT_DIR}/dreambot_api_{TIMESTAMP}.csv"

os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs("data/logs", exist_ok=True)
os.makedirs("data/db", exist_ok=True)

hash_map = {}
if os.path.exists(HASH_JSON):
with open(HASH_JSON, "r", encoding="utf-8") as f:
try:
hash_map = json.load(f)
except:
hash_map = {}

def get_hash(data):
raw = json.dumps(data, sort_keys=True).encode()
return hashlib.md5(raw).hexdigest()

def clean_data(entry):
for key in ["methods", "fields", "constructors", "interfaces", "inner", "annotations"]:
entry[key] = list(set(filter(None, entry.get(key, []))))
entry["superclass"] = entry.get("superclass") or None
return entry

def clean_text(text):
text = text.replace("\u00a0", " ").replace("\\n", "\n")
return text.encode("ascii", "ignore").decode(errors="ignore").strip()

def extract_clean_classname(raw: str) -> str:
raw = raw.strip()
return re.sub(r"^(Class|Enum)\s+", "", raw)

def init_db():
conn = sqlite3.connect(DB_FILE)
cursor = conn.cursor()
cursor.execute("DROP TABLE IF EXISTS api_classes") # Optional: wipe old table
cursor.execute("""
CREATE TABLE IF NOT EXISTS api_classes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT,
url TEXT,
superclass TEXT,
interfaces TEXT,
methods TEXT,
fields TEXT,
constructors TEXT,
inner TEXT,
annotations TEXT,
hash TEXT
)
""")
conn.commit()
conn.close()

def insert_db_record(data, hash_val):
conn = sqlite3.connect(DB_FILE)
cursor = conn.cursor()
cursor.execute("""
INSERT INTO api_classes (
name, url, superclass, interfaces, methods, fields,
constructors, inner, annotations, hash
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
data["class"],
data["url"],
data.get("superclass"),
json.dumps(data.get("interfaces")),
json.dumps(data.get("methods")),
json.dumps(data.get("fields")),
json.dumps(data.get("constructors")),
json.dumps(data.get("inner")),
json.dumps(data.get("annotations")),
hash_val
))
conn.commit()
conn.close()

def write_csv(data):
keys = [
"class", "url", "superclass", "interfaces",
"methods", "fields", "constructors", "inner", "annotations"
]
with open(OUTPUT_CSV, "w", newline='', encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=keys)
writer.writeheader()
for d in data:
row = {k: d[k] if isinstance(d[k], str) else json.dumps(d[k]) for k in keys}
writer.writerow(row)

async def fetch_html(page, url):
for attempt in range(3):
try:
await page.goto(url, timeout=30000)
await page.wait_for_load_state('networkidle')
return await page.content()
except Exception as e:
print(f"❌ Attempt {attempt + 1}/3 failed for {url}: {e}")
await asyncio.sleep(3 + attempt)
return None

async def get_package_links(page):
print(" Loading package links...")
html = await fetch_html(page, urljoin(BASE_URL, "overview-summary.html"))
if not html:
return []
soup = BeautifulSoup(html, "lxml")
return [urljoin(BASE_URL, a['href']) for a in soup.select("a[href*='package-
summary.html']")]

async def get_class_links(page, package_url):
html = await fetch_html(page, package_url)
if not html:
return []
soup = BeautifulSoup(html, "lxml")
anchors = soup.select("table[class*='Summary'] a[href]")
return [urljoin(package_url, a['href']) for a in anchors]

async def parse_class_page(page, url):
html = await fetch_html(page, url)
if not html:
return None
soup = BeautifulSoup(html, "lxml")


h2 = soup.find("h2")
raw_class = h2.text.strip() if h2 else "UnknownClass"
clean_class = extract_clean_classname(raw_class)

data = {
"url": url,
"class": clean_class,
"methods": [],
"fields": [],
"constructors": [],
"interfaces": [],
"superclass": None,
"inner": [],
"annotations": []
}

# Superclass and interfaces
class_decl = soup.find("pre")
if class_decl:
decl_text = class_decl.get_text()
superclass_match = re.search(r"extends\s+([a-zA-Z0-9_.]+)", decl_text)
interface_match = re.search(r"implements\s+([a-zA-Z0-9_., ]+)", decl_text)
if superclass_match:
data["superclass"] = superclass_match.group(1)
if interface_match:
data["interfaces"] = [s.strip() for s in interface_match.group(1).split(",")]

# Member tables
tables = soup.select("table.memberSummary")
for table in tables:
summary_type = table.find("caption")
rows = table.select("tr")[1:]
content = []
for row in rows:
cols = row.find_all("td")
if cols:
text = clean_text(" ".join(c.get_text(separator=" ", strip=True) for c in
cols))
if "inherited" in text.lower():
continue
content.append(text)
if not summary_type:
continue
label = summary_type.text.lower()
if "field" in label:
data["fields"].extend(content)
elif "method" in label:
data["methods"].extend(content)
elif "constructor" in label:
data["constructors"].extend(content)
elif "annotation" in label:
data["annotations"].extend(content)
elif "nested" in label or "inner" in label:
data["inner"].extend(content)

return clean_data(data)

async def main():
init_db()
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page()


results = []
package_urls = await get_package_links(page)
print(f" Found {len(package_urls)} package links")

class_urls = []
for pkg_url in tqdm_asyncio(package_urls, desc="Packages"):
links = await get_class_links(page, pkg_url)
class_urls.extend(links)

print(f" Found {len(class_urls)} class/interface/enum links")

for class_url in tqdm_asyncio(class_urls, desc="Classes"):
data = await parse_class_page(page, class_url)
if data:
hash_val = get_hash(data)
hash_map[data["class"]] = hash_val
insert_db_record(data, hash_val)
results.append(data)

await browser.close()

async with aiofiles.open(OUTPUT_JSON, "w", encoding="utf-8") as f:
await f.write(json.dumps(results, indent=2))
async with aiofiles.open(HASH_JSON, "w", encoding="utf-8") as f:
await f.write(json.dumps(hash_map, indent=2))
write_csv(results)

print(f"✅ Scraped {len(results)} classes/interfaces/enums")

asyncio.run(main())

Continue reading...