[Python] Detecting emojis in an image [closed]

I have similar images that show several emojis in PNG format: A preprocessed image with the background removed

I have a set of png images with all possible emojis (format: image number(#).png). There is also a csv mapping (full_emoji.csv) with a description and other information for these emojis (key field #): #,emoji,unicode,name.

I want to define emojis on the source image and as a result return a set of values from csv mapping in the emoji column.

In the example image, I expect to get such a set: ️

In my method uses a feature-based KNN approach to recognize emojis:

import os
import cv2
import numpy as np
import pandas as pd
from sklearn.neighbors import KNeighborsClassifier


class RobustEmojiRecognizer:
def __init__(self, emoji_dir='dataset/emojis', csv_path='dataset/full_emoji.csv'):
self.emoji_dir = emoji_dir
self.emoji_data = pd.read_csv(csv_path)
self.emoji_features = self._extract_emoji_features()
self.knn = self._train_knn_classifier()
self.expected_features = 11

def _extract_emoji_features(self):
features = {}
for idx, row in self.emoji_data.iterrows():
img_path = os.path.join(self.emoji_dir, f"{row['#']}.png")
if os.path.exists(img_path):
img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
if img is not None:
contour_feats = self._get_contour_features(img)
color_feats = self._get_color_features(img)

# Создаем полный вектор признаков с заполнением None
full_features = self._create_full_feature_vector(contour_feats, color_feats)

if full_features is not None:
features[row['#']] = {
'features': full_features,
'emoji': row['emoji'],
'name': row['name']
}
return features

def _get_contour_features(self, img):
if len(img.shape) == 3:
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
else:
gray = img

_, thresh = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
if not contours:
return None

cnt = max(contours, key=cv2.contourArea)
moments = cv2.moments(cnt)
hu_moments = cv2.HuMoments(moments).flatten()

area = cv2.contourArea(cnt)
perimeter = cv2.arcLength(cnt, True)
circularity = (4 * np.pi * area) / (perimeter ** 2) if perimeter > 0 else 0

return {
'hu_moments': hu_moments,
'area': area,
'circularity': circularity
}

def _get_color_features(self, img):
if len(img.shape) < 3 or img.shape[2] < 3:
return None

hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

if img.shape[2] == 4:
mask = img[:, :, 3] > 0
hsv = hsv[mask]
else:
hsv = hsv.reshape(-1, 3)

if len(hsv) == 0:
return None

mean_hue = np.mean(hsv[:, 0])
dominant_hue = np.bincount(hsv[:, 0].astype(int)).argmax()

return {
'mean_hue': mean_hue,
'dominant_hue': dominant_hue
}

def _create_full_feature_vector(self, contour_feats, color_feats):
if contour_feats is None:
return None

# Базовые признаки контура (9 признаков)
features = np.concatenate([
contour_feats['hu_moments'], # 7
[contour_feats['area']], # 1
[contour_feats['circularity']] # 1
])

if color_feats is not None:
features = np.append(features, [
color_feats['mean_hue'],
color_feats['dominant_hue']
])
else:
# Заполняем нулями отсутствующие цветовые признаки
features = np.append(features, [0, 0])

return features

def _train_knn_classifier(self):
X = []
y = []

for emoji_id, data in self.emoji_features.items():
X.append(data['features'])
y.append(emoji_id)

if not X:
return None

knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(X, y)
return knn

def _preprocess_input_image(self, img):
if len(img.shape) == 3:
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
else:
gray = img

gray = cv2.equalizeHist(gray)
_, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

kernel = np.ones((3, 3), np.uint8)
thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)

return thresh

def _find_emoji_regions(self, img):
thresh = self._preprocess_input_image(img)
contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

regions = []
for cnt in contours:
if cv2.contourArea(cnt) > 100:
x, y, w, h = cv2.boundingRect(cnt)
region = img[y:y + h, x:x + w]
regions.append({
'image': region,
'contour': cnt,
'position': (x, y, w, h)
})

return regions

def _extract_features(self, img, cnt):
contour_feats = self._get_contour_features(img)
color_feats = self._get_color_features(img)
return self._create_full_feature_vector(contour_feats, color_feats)

def recognize_emojis(self, image_path):
img = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
if img is None:
return []

if img.shape[2] == 3:
alpha = np.ones((img.shape[0], img.shape[1]), dtype=img.dtype) * 255
img = cv2.merge((img, alpha))

regions = self._find_emoji_regions(img)
results = []

for region in regions:
features = self._extract_features(region['image'], region['contour'])
if features is not None and self.knn is not None:
try:
emoji_id = self.knn.predict([features])[0]
confidence = np.max(self.knn.predict_proba([features]))

if confidence > 0.5:
results.append({
'emoji': self.emoji_features[emoji_id]['emoji'],
'name': self.emoji_features[emoji_id]['name'],
'confidence': confidence,
'position': region['position']
})
except Exception as e:
print(f"Ошибка при классификации: {e}")
continue

results.sort(key=lambda x: x['position'][0])
return results

def recognize_emojis_to_string(self, image_path):
results = self.recognize_emojis(image_path)
return ''.join([r['emoji'] for r in results])

Continue reading...