[Python] Python image/color recognition and scanning/timing

So I am trying to make a code for game automation. There is this minigame "Claw machine". The claw moves around with the same speed all the time and I need to time it's location with a material location then press space to catch it. The code works fine right now but it misses it often by few pixels. There is also this probem where it detects not the right material but for now it's fine. Would be also helpful if you could recommend me better way to detect the material (pumpkin). I would use image recognition but the pumpkin can spawn in any angle so I use color detection.

So the two main problems are:

The timing sometimes is off and the claw misses the pumpkin. The code sometimes detects the wrong material.

The screenshot of the game (2k monitor): Claw machine game

The Python Code:

import cv2
import numpy as np
import pyautogui
import time
import tkinter as tk
from tkinter import ttk
import threading
import keyboard

# Configuration
SCREEN_REGION = (276, 181, 2003, 1044) # Full game area (X, Y, W, H)
CLAW_Y = 396 # Claw Y position (absolute from top of 2K screen)
CLAW_SEARCH_HEIGHT = 8 # Search band for claw detection (Y-range)
CLAW_RGB = (90, 105, 136) # Claw color
CLAW_TOLERANCE = 30 # Tolerance in color

# Pumpkin (material) color
MATERIAL_HSV = [30, 220, 255] # Center color in HSV (pumpkin)

ALIGN_THRESHOLD = 50 # Alignment threshold (pixels)
CHECK_INTERVAL = 0.02 # Checking interval (seconds)
DEFAULT_REACTION_TIME = 0.0 # Reaction time (default 0)

class ClawAutomator:
def __init__(self):
self.running = False
self.thread = None

self.claw_region = (
SCREEN_REGION[0],
CLAW_Y - CLAW_SEARCH_HEIGHT,
SCREEN_REGION[2],
CLAW_SEARCH_HEIGHT * 2
)

def get_claw_x(self):
try:
screenshot = pyautogui.screenshot(region=self.claw_region)
screen = np.array(screenshot)

lower = np.array([c - CLAW_TOLERANCE for c in CLAW_RGB])
upper = np.array([c + CLAW_TOLERANCE for c in CLAW_RGB])

mask = cv2.inRange(screen, lower, upper)
vertical_proj = np.sum(mask, axis=0)
x_positions = np.where(vertical_proj > 1000)[0]

if len(x_positions) > 0:
avg_x = int(np.median(x_positions))
return self.claw_region[0] + avg_x
return None
except:
return None

def get_material_position(self):
try:
screenshot = pyautogui.screenshot(region=SCREEN_REGION)
screen = np.array(screenshot)
hsv = cv2.cvtColor(screen, cv2.COLOR_RGB2HSV)

lower = np.array([MATERIAL_HSV[0] - 5, 200, 200])
upper = np.array([MATERIAL_HSV[0] + 5, 255, 255])
mask = cv2.inRange(hsv, lower, upper)

mask = cv2.erode(mask, None, iterations=2)
mask = cv2.dilate(mask, None, iterations=3)

contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

if contours:
largest = max(contours, key=cv2.contourArea)
M = cv2.moments(largest)
if M["m00"] > 0:
x = int(M["m10"] / M["m00"])
y = int(M["m01"] / M["m00"])
return (SCREEN_REGION[0] + x, SCREEN_REGION[1] + y)
return None
except:
return None

def run(self):
while self.running:
try:
pyautogui.press('space')
time.sleep(0.5)

material_pos = self.get_material_position()

if material_pos:
aligned = False
start_time = time.time()
while not aligned and self.running:
claw_x = self.get_claw_x()
if claw_x:
material_x = material_pos[0]
diff = abs(claw_x - material_x)

if diff <= ALIGN_THRESHOLD:
pyautogui.press('space')
aligned = True

if time.time() - start_time > 10:
pyautogui.press('space')
break

time.sleep(CHECK_INTERVAL)
else:
time.sleep(1.0)
pyautogui.press('space')
time.sleep(4.5)

time.sleep(4.5)
except:
pass

def start(self):
if not self.running:
self.running = True
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()

def stop(self):
self.running = False

class ControlGUI(tk.Tk):
def __init__(self):
super().__init__()
self.title("Claw Machine Controller")
self.geometry("300x150")
self.automator = ClawAutomator()

self.create_widgets()
self.bind_hotkeys()
self.protocol("WM_DELETE_WINDOW", self.on_close)

def create_widgets(self):
control_frame = ttk.Frame(self)
control_frame.pack(pady=20)

self.status_var = tk.StringVar(value="Status: Stopped")

ttk.Label(control_frame, text="Claw Machine Automator", font=('Arial', 16)).pack(pady=5)

self.start_btn = ttk.Button(control_frame, text="Start", command=self.start_bot)
self.start_btn.pack(pady=5)

self.stop_btn = ttk.Button(control_frame, text="Stop", command=self.stop_bot, state=tk.DISABLED)
self.stop_btn.pack(pady=5)

ttk.Label(control_frame, text="Hotkey: F8").pack(pady=5)
ttk.Label(control_frame, textvariable=self.status_var).pack()

def bind_hotkeys(self):
keyboard.add_hotkey('F8', self.toggle_bot)
self.after(100, self.update_status)

def update_status(self):
if self.automator.running:
self.status_var.set("Status: Running")
self.start_btn.config(state=tk.DISABLED)
self.stop_btn.config(state=tk.NORMAL)
else:
self.status_var.set("Status: Stopped")
self.start_btn.config(state=tk.NORMAL)
self.stop_btn.config(state=tk.DISABLED)
self.after(100, self.update_status)

def toggle_bot(self):
if self.automator.running:
self.stop_bot()
else:
self.start_bot()

def start_bot(self):
self.automator.start()

def stop_bot(self):
self.automator.stop()

def on_close(self):
self.automator.stop()
self.destroy()

if __name__ == "__main__":
app = ControlGUI()
app.mainloop()



I tried many things: letting it scan more often, tweaking the threshold of pixels but right now this seems to be the best configuration.

Some debug screenshot blue line is when the claw alignment was detected and green line is when the material is detected. In this screenshot it missed the pumpkin some pixels to the right side and you can see one extra material that shouldn't show (the rainbow one):

With added debug info lines and visualization

How it looked after the catch in game:

in game after catch

Continue reading...