[Python] Formatting mismatch for PaddleOCR

Currently, I am working on an OCR project where I need to read text from an invoice (see sample image below). I am having issues with the image skew and also the output format I am getting, I need help to fix the image skew so that the text is horizontal and not slanted as well as the output, I want the output format to be the same as the original document. Initially I tried the box helper format to print the text in the box so that the structure is the same, but because there is a slanted file, the format creates new problems when I extract it with fuzzywuzzy. Currently I need some resources or suggestions to solve my problem.

Code:

# Class FieldReader with fuzzy matching and flexible prefix matching
class FieldReader:
def __init__(self, field_name, keywords, prefix=None, default="-", threshold=60):
self.field_name = field_name
self.keywords = keywords
self.prefix = prefix
self.default = default
self.threshold = threshold

def extract(self, text):
words = text.split()
potential_value = None

for keyword in self.keywords:
# Use fuzzy matching to find the best match for the keyword in the text
best_match = process.extractOne(keyword.rstrip(':'), words, scorer=fuzz.partial_ratio)
if best_match and best_match[1] >= self.threshold:
index = words.index(best_match[0])

# Search for the ":" in the following words
for i in range(index + 1, len(words)):
if ":" in words:
# If found, remove anything between keyword and ":" (clean unwanted characters)
if i > index + 1:
words[index + 1:i] = [] # Remove intermediate words between keyword and ":"

# Extract the word immediately following the ":"
if i + 1 < len(words):
potential_value = words[i + 1]
else:
potential_value = None
break

# If a prefix is defined, check if the value starts with that prefix (without specifying length)
if self.prefix and potential_value and potential_value.startswith(self.prefix):
return potential_value # Return the value if it starts with the prefix

elif potential_value:
return potential_value # Return the value if no prefix is specified or matches

return self.default # Return default value if no match is found



class FieldReader_regex:
def __init__(self, field_name, pattern):
self.field_name = field_name
self.pattern = pattern

def extract(self, text):
# Mencari semua kecocokan dengan regex
matches = re.findall(self.pattern, text)
# Mengambil hanya hasil pertama atau mengembalikan '-' jika tidak ada
return matches[0] if matches else "-"



# Function to extract information from text using fuzzy matching
def extract_information(text, provider):
data = {}
if provider == "Pertamina":
patterns = [
FieldReader("Date", ["Date", "Tanggal", "Issue Date"], default="-"),
FieldReader_regex("Aircraft Reg", r"\bPK\w*\b"), # Ambil semua yang berawalan PK untuk Aircraft Reg
FieldReader_regex("Flight Numbers", r"\bQZ\w*\b"), # Ambil semua yang berawalan QZ untuk nomor penerbangan
# FieldReader("Reg", ["Reg", "Reg g", "Aircraft Reg","Aircraft Reg g", "Alrcraft Reg", "AIRCRAFT REG"], prefix="PK", default="-"),
# FieldReader("Flight No", ["Flight Number", "Flight No", "Flight Nunber","Flight N0", "Fllght Number", "FLIGHT NUMBER"], prefix ="QZ", default="-"),
FieldReader("Dep", ["IATA", "Departure", "Dep", "TATA", "lATA"], default="-"),
FieldReader("Uplift_in_Lts", ["Quantity", "QUANTITY", "Fuel Quantity", "Uplift", "Quantity", "Delivered Volume (L)"], default="-"),
FieldReader_regex("Invoice", r"DELIVERY RECEIPT\s*(\S+)"),
FieldReader("Provider", ["Provider", "Supplier"], default="Pertamina")
]
elif provider == "Shell":
patterns = [
FieldReader("Date", ["Date", "Issue Date"], default="-"),
FieldReader("Reg", ["Reg", "Aircraft Reg"], default="-"),
FieldReader("Flight No", ["Flight No"], default="-"),
FieldReader("Dep", ["IATA", "Departure"], default="-"),
FieldReader("Arr", ["Next Dest", "Arrival"], default="-"),
FieldReader("Uplift_in_Lts", ["QUANTITY", "Fuel Quantity"], default="-"),
FieldReader_regex("Aircraft Reg", r"\bPK\w*\b"), # Ambil semua yang berawalan PK untuk Aircraft Reg
FieldReader_regex("Invoice", r"DELIVERY RECEIPT\s*(\S+)"),
FieldReader("Provider", ["Provider", "Supplier"], default="SHELL")
]

for field in patterns:
data[field.field_name] = field.extract(text)

return data


# Function to preprocess the image (grayscale and threshold)
def preprocess_image(image):
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
_, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
return thresh

# Function to draw boxes and extract text based on boxes with left-to-right order
def extract_text_by_boxes(result, image_width, margin=2):
box_data = []
box_counter = 1

for line in result:
for word_info in line:
box = word_info[0] # Coordinates of the box
text, conf = word_info[1] # Text and confidence

# Get coordinates of the word box
x_min, y_min = box[0]
x_max, y_max = box[2]

# Adjust box to cover the entire width of the image (left to right)
extended_box = [[0, y_min - margin], [image_width, y_min - margin], [image_width, y_max + margin], [0, y_max + margin]]
box_data.append((box_counter, extended_box, text))
box_counter += 1

return box_data

# Function to perform OCR based on bounding boxes
def ocr_boxes(processed_image, box_data):
full_text = ""
for box_info in box_data:
box_counter, extended_box, _ = box_info

# Create a mask for the bounding box
mask = np.zeros_like(processed_image)
cv2.fillPoly(mask, [np.array(extended_box).astype(np.int32)], (255,))

# Extract the region of interest (ROI) using the mask
roi = cv2.bitwise_and(processed_image, mask)

# OCR the ROI without rotation
ocr = PaddleOCR(use_angle_cls=True, lang='en')
result = ocr.ocr(roi, cls=True)

# Concatenate the extracted text
for line in result:
if not line: # This checks for None or empty list
continue
for word_info in line:
text = word_info[1][0]
full_text += f" {text}"

# Draw the box for visualization
cv2.polylines(processed_image, [np.array(extended_box).astype(np.int32)], isClosed=True, color=(0, 255, 0), thickness=2)
cv2.putText(processed_image, str(box_counter), (int(extended_box[0][0]), int(extended_box[0][1] - 10)), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 2)

return full_text

# Function to extract text from images with boxes and sort by box order
def getText(file_path):
images = convert_from_path(file_path, dpi=300)
full_text_all = ""

for i, page in enumerate(images):
page_array = np.array(page)
page_array = cv2.cvtColor(page_array, cv2.COLOR_RGB2BGR)

# Apply preprocessing (optional)
processed_image = preprocess_image(page_array)
image_width = processed_image.shape[1]

# OCR the processed image
ocr = PaddleOCR(use_angle_cls=True, lang='en')
result = ocr.ocr(processed_image, cls=True)

# Extract text using custom boxes
box_data = extract_text_by_boxes(result, image_width, margin=0)

# Sort box data by box order
box_data.sort(key=lambda x: x[0])

# Perform OCR on each bounding box
full_text = ocr_boxes(processed_image, box_data)
full_text_all += full_text

# Print the extracted text from the current image (for debugging)
print(f"\nExtracted text from image {i + 1}:\n{full_text}")

# Display the processed image with boxes
cv2_imshow(processed_image)

# Save the processed image (optional)
cv2.imwrite(f"processed_page_{i+1}.png", processed_image)

return full_text_all


Example:

enter image description here

The output was not printed correctly and mismatch.

If anyone can help me figure this problem out, it would be of much help.

Continue reading...