[Python] Repeated calls to python subprocess.run(['cp', src, dst]) causes out of memory error...

I have a relatively simple python script used to synchronize two media stores via a USB HDD. It checks the files present on a remote slave server via SSH, and compares the list to the files present on the local (master) server. The files which are on the master server, but missing from the remote slave server are copied to the USB HDD. I then physically attach the USB HDD to the slave server and copy the missing files (via another script) to the slave server HDD.

The problem is, this script causes out of memory errors when performing the initial copy to the USB HDD. It is running on a Pi 4 with 8GB ram and the system shows 7.4GB free when the script is started. Here is the python script:

#!/usr/bin/env python3

import os
import subprocess
import json
from datetime import datetime
import argparse
import hashlib

class StorageSync:
def __init__(self, master_path, slave_host, slave_path, usb_path):
self.master_path = os.path.normpath(master_path)
self.slave_host = slave_host
self.slave_path = os.path.normpath(slave_path)
self.usb_path = os.path.normpath(usb_path)

# Create a unique directory name based on the paths
path_hash = hashlib.md5(f"{master_path}:{slave_path}".encode()).hexdigest()[:8]
self.sync_dir = os.path.join(self.usb_path, f"sync_{path_hash}")

def get_file_list(self, path, is_remote=False):
"""Get list of files with their sizes"""
if is_remote:
# Use find with null terminators and stat to handle spaces correctly
cmd = f'ssh {self.slave_host} "find {path} -type f -print0 | xargs -0 stat -c \'%s %n\'"'
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
files = {}
for line in result.stdout.splitlines():
if not line.strip():
continue
# Split on first space only
size_str, filepath = line.split(' ', 1)
try:
size = int(size_str)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = size
except ValueError:
print(f"Warning: Could not parse size from line: {line}")
else:
# Get local file list, including hidden files
files = {}
for root, _, filenames in os.walk(path):
for filename in filenames:
filepath = os.path.join(root, filename)
rel_path = os.path.relpath(filepath, path)
files[rel_path] = os.path.getsize(filepath)
return files

def compare_storage(self):
"""Compare master and slave storage"""
print("Getting master file list...")
master_files = self.get_file_list(self.master_path)
print(f"Found {len(master_files)} files on master")

print("Getting slave file list...")
slave_files = self.get_file_list(self.slave_path, is_remote=True)
print(f"Found {len(slave_files)} files on slave")

to_copy = {}
to_delete = []
size_mismatches = 0

# Find files to copy (new or different size)
for file_path, size in master_files.items():
if file_path not in slave_files:
to_copy[file_path] = size
elif slave_files[file_path] != size:
to_copy[file_path] = size
size_mismatches += 1

# Find files to delete (exists in slave but not in master)
for file_path in slave_files:
if file_path not in master_files:
to_delete.append(file_path)

print(f"Found {size_mismatches} files with size mismatches")

return to_copy, to_delete

def copy_to_usb(self, to_copy):
"""Copy files to USB drive"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

total_files = len(to_copy)
for i, (rel_path, size) in enumerate(to_copy.items(), 1):
src = os.path.join(self.master_path, rel_path)
dst = os.path.join(self.sync_dir, rel_path)
os.makedirs(os.path.dirname(dst), exist_ok=True)
print(f"Copying file {i}/{total_files}: {rel_path}")
subprocess.run(['cp', src, dst])
print(f"Finished copying file {i}/{total_files}: {rel_path}")

def generate_delete_script(self, to_delete):
"""Generate a script to delete files on slave"""
# Create sync directory if it doesn't exist
os.makedirs(self.sync_dir, exist_ok=True)

script_path = os.path.join(self.sync_dir, 'delete_files.sh')
with open(script_path, 'w') as f:
f.write('#!/bin/bash\n')
f.write(f'# Delete script for {self.master_path} -> {self.slave_path}\n')
f.write(f'# Generated on {datetime.now().isoformat()}\n\n')
for file_path in to_delete:
f.write(f'rm "{os.path.join(self.slave_path, file_path)}"\n')
os.chmod(script_path, 0o755)

def main():
# Get the directory where the script is located
script_dir = os.path.dirname(os.path.abspath(__file__))

parser = argparse.ArgumentParser(description='Sync storage between master and slave servers')
parser.add_argument('--master-path', required=True, help='Path to master storage')
parser.add_argument('--slave-host', required=True, help='Slave server SSH host')
parser.add_argument('--slave-path', required=True, help='Path to slave storage')
parser.add_argument('--usb-path', default=script_dir, help='Path to USB drive (defaults to script directory)')

args = parser.parse_args()

syncer = StorageSync(args.master_path, args.slave_host, args.slave_path, args.usb_path)

print("Comparing storage...")
to_copy, to_delete = syncer.compare_storage()

# Save comparison results
results = {
'timestamp': datetime.now().isoformat(),
'master_path': args.master_path,
'slave_path': args.slave_path,
'to_copy': to_copy,
'to_delete': to_delete
}

# Create sync directory if it doesn't exist
os.makedirs(syncer.sync_dir, exist_ok=True)

with open(os.path.join(syncer.sync_dir, 'sync_plan.json'), 'w') as f:
json.dump(results, f, indent=2)

print(f"Files to copy: {len(to_copy)}")
print(f"Files to delete: {len(to_delete)}")
print(f"Sync directory: {syncer.sync_dir}")

if to_copy:
print("Copying files to USB...")
syncer.copy_to_usb(to_copy)

if to_delete:
print("Generating delete script...")
syncer.generate_delete_script(to_delete)

print("Done!")

if __name__ == '__main__':
main()


The python script is called 4 times from a bash script to synchronize 4 different directories:

#! /bin/bash
echo "###### Syncing myuser/downloads ######"
./storage_sync.py --master-path /home/myuser/downloads --slave-host mediauser@slave-server --slave-path /home/mediauser/downloads

echo "###### Syncing videos ######"
./storage_sync.py --master-path /home/myuser/videos --slave-host mediauser@slave-server --slave-path /home/mediauser/videos

echo "###### Syncing keepers ######"
./storage_sync.py --master-path /home/myuser/keepers --slave-host mediauser@slave-server --slave-path /home/mediauser/keepers

echo "###### Syncing dlna content ######"
./storage_sync.py --master-path /dlna/videos --slave-host mediauser@slave-server --slave-path /home/minidlna/share/videos



Can anyone tell me why this is running out of memory? I thought that subprocess run was synchronous and reaped spawned processes.

Continue reading...