[Python] OpenCV to detect squares when obscured by handwritten text

I'm a newbie to using OpenCV and I am struggling to get my code to detect squares in the image below

The code I am using to try to detect the grey square is provided below:

import os
import cv2
import numpy as np
from matplotlib import pyplot as plt

lower_gray = np.array([0,0,100])
upper_gray = np.array([255, 5, 255])

sharpen_kernel = np.array([
[+0, -1, +0],
[-1, +5, -1],
[+0, -1, +0]
])

img = cv2.imread(png_path)
sharpened = cv2.filter2D(img, -1, sharpen_kernel)
hsv = cv2.cvtColor(sharpened.copy(), cv2.COLOR_BGR2HSV)
mask_grey = cv2.inRange(hsv, lower_gray, upper_gray)

# Build mask of non-black pixels.
nzmask = cv2.inRange(hsv, (0, 0, 5), (255, 255, 255))

# Erode the mask - all pixels around a black pixels should not be masked.
nzmask = cv2.erode(nzmask, np.ones((3,3)))
mask_grey = mask_grey & nzmask

processed_img = img.copy()
processed_img[np.where(mask_grey)] = 255

gray = cv2.cvtColor(processed_img, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (5,5), 0)
threshold = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)

morph_threshold = threshold[1]


# Fix reference marks boundaries
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
closed = cv2.morphologyEx(morph_threshold, cv2.MORPH_CLOSE, kernel, iterations=2)

contours = cv2.findContours(closed.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]

for contour in contours:
perimeter = cv2.arcLength(contour, True)
approx = cv2.approxPolyDP(contour, 0.02 * perimeter, True)
x,y,w,h = cv2.boundingRect(approx)
area = cv2.contourArea(contour)
aspect_ratio = np.round(w / float(h),2)

if len(approx) > 3 and (2200 < area < 4000): #and (0.725 <= aspect_ratio <= 1.275)

cv2.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)

plt.figure(figsize = (11.69*2.5,8.27*2.5))
plt.axis('off')
plt.imshow(img);


However, the dark grey square on the bottom right is not being detected. What I'm I doing wrong!?!

Continue reading...