[Python] Trying to use Faster R-CNN pretrained on COCO on Pascal VOC dataset but CPU/GPU...

I would like to hear some advices from you. For context, I am just a biology student with some coding/AI classes, meaning that I code by patchworking exercizes content, content found on forums with supporting help of AI. I really tried my best to search for answers but obviously my understanding/level of coding is very limited and the problems not begginners friendly.

!!!! I AM WORKING ON GOOGLE COOLAB !!!

For a project I decided to adapt an exercize given by the teacher : finetune Faster R-CNN pretrained on COCO on Pascal VOC dataset.

I started by preparing my dataset.

sys.version

import sys
import os

os.environ['CUDA_LAUNCH_BLOCKING'] = "1"

import torch
import torchvision as tv

print(sys.version)
print(torch.__version__)
print(tv.__version__)

import matplotlib.pyplot as plt #to plot images
import matplotlib.patches as patches
import torchvision.transforms.functional as F #converts a Python Imaging Library (PIL) into a tensor, so the image will become a matrix that Pytorch can work on

from torchvision.datasets import VOCDetection
voc_dataset = VOCDetection(
root='./data', # Folder where you want to save it
year='2012', # Pascal VOC 2012 version
image_set='train', # Choose 'train', 'val', 'trainval', or 'test'
download=True # Download it if it's not already there
)

print("Number of images:", len(voc_dataset))

import os
import numpy as np #library ro work with array and numerical operations
import torch # library to build and train deep learning models
import torch.utils.data # import utils from pytorhc to create and manage datasets
from PIL import Image
from torchvision.datasets import VOCDetection #ro import the pascal VOC dataset
import torchvision.transforms.functional as F # to transform images like convert to tensor
import torchvision.transforms as transforms

class VOCDataset(torch.utils.data.Dataset): #creates a new dataset class
VOC_CLASSES = { #creates a dictionnary of the class name and their associated numbers
'aeroplane': 1,
'bicycle': 2,
'bird': 3,
'boat': 4,
'bottle': 5,
'bus': 6,
'car': 7,
'cat': 8,
'chair': 9,
'cow': 10,
'diningtable': 11,
'dog': 12,
'horse': 13,
'motorbike': 14,
'person': 15,
'pottedplant': 16,
'sheep': 17,
'sofa': 18,
'train': 19,
'tvmonitor': 20
}
def __init__(self, root, transforms=None, image_size=(224,224)): # prepare the dataset
self.root = root #saves where is saved the dataset
self.transforms = transforms #saves images transformation
self.dataset = VOCDetection(root=root, year='2012', image_set='train', download=False) #loads dataset images and labels
self.image_size = image_size # store image_size

#compared to PeenFudanDataset, Pascal vOC is already a standard dataset in Pytorch so it finds images, loads and match everything automatically

def __getitem__(self, idx):
image, target = self.dataset[idx] #gets one image and its annotation

objects = target['annotation']['object'] # list all of the annotated images present on the image
if not isinstance(objects, list):
objects = [objects]
#empty lists to collect bounding boxes and labels
boxes = []
labels = [] #numeric labels
label_names = []

for obj in objects: #reads bounding box coordinates from the annotations
bbox = obj['bndbox']
xmin = int(bbox['xmin'])
ymin = int(bbox['ymin'])
xmax = int(bbox['xmax'])
ymax = int(bbox['ymax'])
boxes.append([xmin, ymin, xmax, ymax])

label = obj['name'] #reads object class name, converts to number and stire borh names and number
label_idx = self.VOC_CLASSES[label]
labels.append(label_idx)
label_names.append(label)

boxes = torch.as_tensor(boxes, dtype=torch.float32) #converts the boxes and labo into tensors that Pytorch can work on
labels = torch.as_tensor(labels, dtype=torch.int64)

image_id = torch.tensor([idx]) #id of the image
area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0]) #area of each box (for later loss calculation)
iscrowd = torch.zeros((len(boxes),), dtype=torch.int64) #if object part of a crowd or not

#final dictionnary
target = {}
target["boxes"] = boxes
target["labels"] = labels
target["label_names"] = label_names
target["image_id"] = image_id
target["area"] = area
target["iscrowd"] = iscrowd

#apply any extra data augmentation when needed
if self.transforms is not None:
image, target = self.transforms(image, target) ##################### before there was no target

return image, target

def __len__(self):
return len(self.dataset) #tells pytorch how many samples we have in the dataset



Then tried to use the I COCO trained Faster R-CNN.

But had a problem when running my code, following Gemini hypothesis : "The error AttributeError: 'list' object has no attribute 'to' occurs in the engine.py file, within the train_one_epoch function. This indicates that the variable 'v' (which represents values in the 'targets' dictionary) is a Python list and not a PyTorch tensor. The '.to(device)' method is used to move PyTorch tensors to a specific device (e.g., CPU or GPU), but it cannot be applied to Python lists.

In the VOCDataset.getitem method, the target['label_names'] is a Python list containing the names of the objects in the image. When the train_one_epoch function iterates over the targets dictionary, it attempts to move all values (including the label_names list) to the specified device using v.to(device). This fails because v is a list and not a tensor. "

So i downloaded the engyne.py to include conditional check.

import math
import sys
import time
import torch

import torchvision.models.detection.mask_rcnn

from coco_utils import get_coco_api_from_dataset
from coco_eval import CocoEvaluator
import utils


def train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq):
model.train()
metric_logger = utils.MetricLogger(delimiter=" ")
metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))
header = 'Epoch: [{}]'.format(epoch)

lr_scheduler = None
if epoch == 0:
warmup_factor = 1. / 1000
warmup_iters = min(1000, len(data_loader) - 1)

lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)

for images, targets in metric_logger.log_every(data_loader, print_freq, header):
images = list(image.to(device) for image in images)
targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] ##### MODIFIED by me

loss_dict = model(images, targets)

losses = sum(loss for loss in loss_dict.values())

# reduce losses over all GPUs for logging purposes
loss_dict_reduced = utils.reduce_dict(loss_dict)
losses_reduced = sum(loss for loss in loss_dict_reduced.values())

loss_value = losses_reduced.item()

if not math.isfinite(loss_value):
print("Loss is {}, stopping training".format(loss_value))
print(loss_dict_reduced)
sys.exit(1)

optimizer.zero_grad()
losses.backward()
optimizer.step()

if lr_scheduler is not None:
lr_scheduler.step()

metric_logger.update(loss=losses_reduced, **loss_dict_reduced)
metric_logger.update(lr=optimizer.param_groups[0]["lr"])

return metric_logger


def _get_iou_types(model):
model_without_ddp = model
if isinstance(model, torch.nn.parallel.DistributedDataParallel):
model_without_ddp = model.module
iou_types = ["bbox"]
if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN):
iou_types.append("segm")
if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN):
iou_types.append("keypoints")
return iou_types


@torch.no_grad()
def evaluate(model, data_loader, device):
n_threads = torch.get_num_threads()
# FIXME remove this and make paste_masks_in_image run on the GPU
torch.set_num_threads(1)
#cpu_device = torch.device("cpu")

# Dynamically select device
if device is None and torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu") # Ensures defaulting to CPU if CUDA unavailable

model.eval()
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'

coco = get_coco_api_from_dataset(data_loader.dataset)
iou_types = _get_iou_types(model)
coco_evaluator = CocoEvaluator(coco, iou_types)

for images, targets in metric_logger.log_every(data_loader, 100, header):
images = list(img.to(device) for img in images)

# torch.cuda.synchronize()
model_time = time.time()
outputs = model(images)

outputs = [{k: v.to(cpu_device("cpu")) for k, v in t.items()} for t in outputs] ## ("cpu")
model_time = time.time() - model_time

res = {target["image_id"].item(): output for target, output in zip(targets, outputs)}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
metric_logger.update(model_time=model_time, evaluator_time=evaluator_time)

# gather the stats from all processes
metric_logger.synchronize_between_processes()
print("Averaged stats:", metric_logger)
coco_evaluator.synchronize_between_processes()

# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
torch.set_num_threads(n_threads)
return coco_evaluator



To finally work on my dataset :

# Download TorchVision repo to use some files from
# references/detection
!git clone https://github.com/mitjanikolaus/vision.git

!cp vision/references/detection/utils.py .
!cp vision/references/detection/transforms.py .
!cp vision/references/detection/coco_eval.py .
!cp vision/references/detection/engine.py .
!cp vision/references/detection/coco_utils.py .

%%shell

pip install cython
# Install pycocotools, the version by default in Colab
# has a bug fixed in https://github.com/cocodataset/cocoapi/pull/354
pip install -U 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'

import torch
import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN_ResNet50_FPN_Weights
import utils
import transforms as T
from my_engine import train_one_epoch, evaluate

# Device
#device = torch.device('cpu')
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

# Load Faster R-CNN pretrained on COCO
#model = torchvision.models.detection.fasterrcnn_resnet50_fpn(weights=FasterRCNN_ResNet50_FPN_Weights.COCO_V1)

# Load Faster R-CNN pretrained on COCO with CPU weights
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
weights=FasterRCNN_ResNet50_FPN_Weights.COCO_V1,
#map_location=torch.device('cpu') # ✅ Load weights on CPU
)

# Modify it for Pascal VOC: 20 classes + background
num_classes = 21 # 20 classes + background
in_features = model.roi_heads.box_predictor.cls_score.in_features
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

# Move model to device
model = model.to(device)

# Setup dataset
from torch.utils.data import DataLoader
import torch
import torchvision
import transforms as T # your detection transforms
import utils
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor, FasterRCNN_ResNet50_FPN_Weights

# Define get_transform function
def get_transform(train, device):
transforms_list = []
transforms_list.append(T.ToTensor()) # Convert PIL image to Tensor
if train:
transforms_list.append(T.RandomHorizontalFlip(0.5)) # Only for training
return T.Compose(transforms_list)

# Load your custom Pascal VOC dataset
train_dataset = VOCDataset(root='./data', transforms=get_transform(train=True, device=device))
test_dataset = VOCDataset(root='./data', transforms=get_transform(train=False, device=device))

# split original full dataset
torch.manual_seed(1)
full_indices = torch.randperm(len(train_dataset)).tolist()
small_train_indices = full_indices[:80] # 80 samples for training
small_test_indices = full_indices[80:100] # 20 samples for testing

train_dataset = torch.utils.data.Subset(train_dataset, small_train_indices)
test_dataset = torch.utils.data.Subset(test_dataset, small_test_indices)


# Loaders
train_loader = DataLoader(
train_dataset,
batch_size=1,
shuffle=True,
num_workers=1,
pin_memory=True, #makes GPU data transfer more efficient
collate_fn=utils.collate_fn
)

test_loader = DataLoader(
test_dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True,
collate_fn=utils.collate_fn
)

# Optimizer and scheduler
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)

# Training loop
num_epochs = 1

for epoch in range(num_epochs):
train_one_epoch(model, optimizer, train_loader, device, epoch, print_freq=10)
lr_scheduler.step()
evaluate(model, test_loader, device=device)


At this point the models runs, epoch 0 goes through all images loss decreases life is good, then



I tried to modify this line in my-engine : outputs = [{k: v.to(torch.device("cpu")) for k, v in t.items()} for t in outputs], but to be honest I am very lost at this point.

My goal here is not to have the most efficient and good model but just to learn how to use a pretrained one on a different dataset, i do not know how to adjust gpu/cpu.

I woul really appreciate your help, thanks a lot :°

Louna

Continue reading...