[Python] Detecting an object in an gradient background going wrong

I am struggling to detect objects in an image where the background and the object have gradients applied, not only that but have transparency in the object as well, see them as holes in the object.

I've tried doing it with Sobel and more, and using GrabCut, with an background generation, and then compare the pixels from the original and the generated background with each other, where if the pixel in the original image deviates from the background pixel then that pixel is part of the object.

Results from the GrabCut system

Results from the Sobel system

The scripts:

GrabCut System:

#THE ONE USING GRABCUT
import cv2
import numpy as np
import sys
from concurrent.futures import ProcessPoolExecutor
import time

# ------------------ 1. GrabCut Segmentation ------------------
def run_grabcut(img, grabcut_iterations=5, border_margin=5):
h, w = img.shape[:2]
gc_mask = np.zeros((h, w), np.uint8)
# Initialize borders as definite background
gc_mask[:border_margin, :] = cv2.GC_BGD
gc_mask[h-border_margin:, :] = cv2.GC_BGD
gc_mask[:, :border_margin] = cv2.GC_BGD
gc_mask[:, w-border_margin:] = cv2.GC_BGD
# Everything else is set as probable foreground.
gc_mask[border_margin:h-border_margin, border_margin:w-border_margin] = cv2.GC_PR_FGD

bgdModel = np.zeros((1, 65), np.float64)
fgdModel = np.zeros((1, 65), np.float64)

try:
cv2.grabCut(img, gc_mask, None, bgdModel, fgdModel, grabcut_iterations, cv2.GC_INIT_WITH_MASK)
except Exception as e:
print("ERROR: GrabCut failed:", e)
return None, None


fg_mask = np.where((gc_mask == cv2.GC_FGD) | (gc_mask == cv2.GC_PR_FGD), 255, 0).astype(np.uint8)
return fg_mask, gc_mask


def generate_background_inpaint(img, fg_mask):

inpainted = cv2.inpaint(img, fg_mask, inpaintRadius=3, flags=cv2.INPAINT_TELEA)
return inpainted


def compute_final_object_mask_strict(img, background, gc_fg_mask, tol=5.0):

# Convert both images to LAB
lab_orig = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)
lab_bg = cv2.cvtColor(background, cv2.COLOR_BGR2LAB)
# Compute absolute difference per channel.
diff = cv2.absdiff(lab_orig, lab_bg).astype(np.float32)
# Compute Euclidean distance per pixel.
diff_norm = np.sqrt(np.sum(diff**2, axis=2))
# Create a mask: if difference exceeds tol, mark as object (255); else background (0).
obj_mask = np.where(diff_norm > tol, 255, 0).astype(np.uint8)
# Enforce GrabCut: where GrabCut says background (gc_fg_mask == 0), force object mask to 0.
obj_mask[gc_fg_mask == 0] = 0
return obj_mask


def process_image_strict(img, grabcut_iterations=5, tol=5.0):

start_time = time.time()
print("--- Processing Image (GrabCut + Inpaint + Strict Pixel Comparison) ---")

# 1. Run GrabCut
print("[Debug] Running GrabCut...")
fg_mask, gc_mask = run_grabcut(img, grabcut_iterations=grabcut_iterations)
if fg_mask is None or gc_mask is None:
return None, None, None
print("[Debug] GrabCut complete.")

# 2. Generate Background via Inpainting.
print("[Debug] Generating background via inpainting...")
background = generate_background_inpaint(img, fg_mask)
print("[Debug] Background generation complete.")

# 3. Pure Pixel-by-Pixel Comparison in LAB with Tolerance.
print(f"[Debug] Performing pixel comparison with tolerance={tol}...")
final_mask = compute_final_object_mask_strict(img, background, fg_mask, tol=tol)
print("[Debug] Pixel comparison complete.")

total_time = time.time() - start_time
print(f"[Debug] Total processing time: {total_time:.4f} seconds.")


grabcut_disp_mask = fg_mask.copy()
return grabcut_disp_mask, background, final_mask


def process_wrapper(args):
img, version, tol = args
print(f"Starting processing for image {version+1}")
result = process_image_strict(img, tol=tol)
print(f"Finished processing for image {version+1}")
return result, version

def main():
# Load images (from command-line or defaults)
path1 = sys.argv[1] if len(sys.argv) > 1 else "test_gradient.png"
path2 = sys.argv[2] if len(sys.argv) > 2 else "test_gradient_1.png"
img1 = cv2.imread(path1)
img2 = cv2.imread(path2)
if img1 is None or img2 is None:
print("Error: Could not load one or both images.")
sys.exit(1)
images = [img1, img2]


tolerance_value = 5.0


with ProcessPoolExecutor(max_workers=2) as executor:
futures = {executor.submit(process_wrapper, (img, idx, tolerance_value)): idx for idx, img in enumerate(images)}
results = [f.result() for f in futures]

# Display results.
for idx, (res, ver) in enumerate(results):
if res is None:
print(f"Skipping display for image {idx+1} due to processing error.")
continue
grabcut_disp_mask, generated_bg, final_mask = res
disp_orig = cv2.resize(images[idx], (480, 480))
disp_grabcut = cv2.resize(grabcut_disp_mask, (480, 480))
disp_bg = cv2.resize(generated_bg, (480, 480))
disp_final = cv2.resize(final_mask, (480, 480))
combined = np.hstack([
disp_orig,
cv2.merge([disp_grabcut, disp_grabcut, disp_grabcut]),
disp_bg,
cv2.merge([disp_final, disp_final, disp_final])
])
window_title = f"Image {idx+1} (Orig | GrabCut FG | Gen Background | Final Mask)"
cv2.imshow(window_title, combined)
print("Displaying results. Press any key to close.")
cv2.waitKey(0)
cv2.destroyAllWindows()

if __name__ == '__main__':
main()



Sobel system:

import cv2
import numpy as np
import sys
from concurrent.futures import ProcessPoolExecutor


def get_background_constraint_mask(image):

gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# Compute Sobel gradients.
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)
mag = np.sqrt(sobelx**2 + sobely**2)
mag = np.uint8(np.clip(mag, 0, 255))
# Hard–set threshold = 0: any nonzero gradient is an edge.
edge_map = np.zeros_like(mag, dtype=np.uint8)
edge_map[mag > 0] = 255
# No morphological processing is done so that maximum sensitivity is preserved.
inv_edge = cv2.bitwise_not(edge_map)
h, w = inv_edge.shape
flood_filled = inv_edge.copy()
ff_mask = np.zeros((h+2, w+2), np.uint8)
for j in range(w):
if flood_filled[0, j] == 255:
cv2.floodFill(flood_filled, ff_mask, (j, 0), 128)
if flood_filled[h-1, j] == 255:
cv2.floodFill(flood_filled, ff_mask, (j, h-1), 128)
for i in range(h):
if flood_filled[i, 0] == 255:
cv2.floodFill(flood_filled, ff_mask, (0, i), 128)
if flood_filled[i, w-1] == 255:
cv2.floodFill(flood_filled, ff_mask, (w-1, i), 128)
background_mask = np.zeros_like(flood_filled, dtype=np.uint8)
background_mask[flood_filled == 128] = 255
return background_mask


def generate_background_from_constraints(image, fixed_mask, max_iters=5000, tol=1e-3):

H, W, C = image.shape
if fixed_mask.shape != (H, W):
raise ValueError("Fixed mask shape does not match image shape.")
fixed = (fixed_mask == 255)
fixed[0, :], fixed[H-1, :], fixed[:, 0], fixed[:, W-1] = True, True, True, True
new_img = image.astype(np.float32).copy()
for it in range(max_iters):
old_img = new_img.copy()
cardinal = (old_img[1:-1, 0:-2] + old_img[1:-1, 2:] +
old_img[0:-2, 1:-1] + old_img[2:, 1:-1])
diagonal = (old_img[0:-2, 0:-2] + old_img[0:-2, 2:] +
old_img[2:, 0:-2] + old_img[2:, 2:])
weighted_avg = (diagonal + 2 * cardinal) / 12.0
free = ~fixed[1:-1, 1:-1]
temp = old_img[1:-1, 1:-1].copy()
temp[free] = weighted_avg[free]
new_img[1:-1, 1:-1] = temp
new_img[fixed] = image.astype(np.float32)[fixed]
diff = np.linalg.norm(new_img - old_img)
if diff < tol:
break
return new_img.astype(np.uint8)

def compute_final_object_mask(image, background):

lab_orig = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)
lab_bg = cv2.cvtColor(background, cv2.COLOR_BGR2LAB)
diff_lab = cv2.absdiff(lab_orig, lab_bg).astype(np.float32)
diff_norm = np.sqrt(np.sum(diff_lab**2, axis=2))
diff_norm_8u = cv2.convertScaleAbs(diff_norm)
auto_thresh = cv2.threshold(diff_norm_8u, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[0]
# Define weak threshold as 90% of auto_thresh:
weak_thresh = 0.9 * auto_thresh
strong_mask = diff_norm >= auto_thresh
weak_mask = diff_norm >= weak_thresh
final_mask = np.zeros_like(diff_norm, dtype=np.uint8)
final_mask[strong_mask] = 255
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
prev_sum = 0
while True:
dilated = cv2.dilate(final_mask, kernel, iterations=1)
new_mask = np.where((weak_mask) & (dilated > 0), 255, final_mask)
current_sum = np.sum(new_mask)
if current_sum == prev_sum:
break
final_mask = new_mask
prev_sum = current_sum
final_mask = cv2.morphologyEx(final_mask, cv2.MORPH_CLOSE, kernel)
return final_mask


def process_image(img):

constraint_mask = get_background_constraint_mask(img)
background = generate_background_from_constraints(img, constraint_mask)
final_mask = compute_final_object_mask(img, background)
return constraint_mask, background, final_mask


def process_wrapper(args):
img, version = args
result = process_image(img)
return result, version

def main():
# Load two images: default file names.
path1 = sys.argv[1] if len(sys.argv) > 1 else "test_gradient.png"
path2 = sys.argv[2] if len(sys.argv) > 2 else "test_gradient_1.png"

img1 = cv2.imread(path1)
img2 = cv2.imread(path2)
if img1 is None or img2 is None:
print("Error: Could not load one or both images.")
sys.exit(1)
images = [img1, img2] # Use images as loaded (blue gradient is original).

with ProcessPoolExecutor(max_workers=2) as executor:
futures = [executor.submit(process_wrapper, (img, idx)) for idx, img in enumerate(images)]
results = [f.result() for f in futures]

for idx, (res, ver) in enumerate(results):
constraint_mask, background, final_mask = res
disp_orig = cv2.resize(images[idx], (480,480))
disp_cons = cv2.resize(constraint_mask, (480,480))
disp_bg = cv2.resize(background, (480,480))
disp_final = cv2.resize(final_mask, (480,480))
combined = np.hstack([
disp_orig,
cv2.merge([disp_cons, disp_cons, disp_cons]),
disp_bg,
cv2.merge([disp_final, disp_final, disp_final])
])
cv2.imshow(f"Output Image {idx+1}", combined)
cv2.waitKey(0)
cv2.destroyAllWindows()

if __name__ == '__main__':
main()


I really appreciate ANY help!

Continue reading...