[Python] I meet a __gnu_cxx::recursive_init_error, when I try to intercept the cuda driver API...

when I run the relative command, I get this

root@node171:/home/tmp/gpushare# LD_PRELOAD=./libculaunchkernel_override5.so python3 ./runblender.py STDOUT: STDERR: terminate called after throwing an instance of '__gnu_cxx::recursive_init_error' what(): std::exception

root@node171:/home/tmp/gpushare# LD_PRELOAD=./libculaunchkernel_override5.so blender -b /home/tmp/monster/main.blend -E CYCLES -f 20 -- --cycles-device CUDA terminate called after throwing an instance of '__gnu_cxx::recursive_init_error' what(): std::exception Aborted (core dumped)

and this is my C++ code. this C++ code will be compile into .so dynamic file.

this is relative command

//compile command: g++ -shared -fPIC -o libculaunchkernel_override5.so culaunchkernel_override5.cpp -lcuda -lnvidia-ml -g -Wno-overflow -I/usr/local/cuda/include/

//run command: LD_PRELOAD=./libculaunchkernel_override5.so blender -b /home/shy/monster/main.blend -E CYCLES -f 20 -- --cycles-device CUDA

// LD_PRELOAD=./libculaunchkernel_override5.so python3 ./runblender.py

#include "./nvml.h"
#include "./cuda.h"
#include <dlfcn.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <unistd.h>
#include <link.h>
#include <stddef.h>
#include <assert.h>
#include <string.h>
#include <mutex>
#include <vector>
#include <stdexcept>
#include <csignal>

#define GB_BYTES 0x40000000UL
#define MB_BYTES 0x100000UL

#define INTERCEPTER_LOGO "gpu_partition"
#define INFO_PRT(fmt,...) do{ \
printf("%s - [%s] info:" fmt, INTERCEPTER_LOGO, __func__, ##__VA_ARGS__); \
} while(0)
#define WARN_PRT(fmt,...) do{ \
printf("%s - [%s] warn:" fmt, INTERCEPTER_LOGO, __func__, ##__VA_ARGS__); \
} while(0)

static void* dl_hdl_cn_;
// size_t global_memory_size_ = 2147483648 ;
size_t global_memory_size_ = 4294967296 ; //4GB
size_t total_mem = 0;

typedef struct {
const char *sym_name;
void *func_ptr;
} Entry_hook_t;

static Entry_hook_t g_hooks_entry[] = {
{.sym_name = "cuLaunchKernel", .func_ptr = (void *)cuLaunchKernel},
{.sym_name = "cuMemAlloc_v2", .func_ptr = (void *)cuMemAlloc_v2},
};

std::vector<nvmlProcessInfo_t> globalInfos;

bool nvml_get_pid_mem() {
nvmlReturn_t result;
nvmlDevice_t device;
unsigned int deviceCount;

globalInfos.clear();

result = nvmlInit();
if (NVML_SUCCESS != result) {
std::cerr << "Failed to initialize NVML: " << nvmlErrorString(result) << std::endl;
return false;
}

result = nvmlDeviceGetCount(&deviceCount);
if (NVML_SUCCESS != result) {
std::cerr << "Failed to get device count: " << nvmlErrorString(result) << std::endl;
nvmlShutdown();
return false;
}

for (unsigned int i = 0; i < deviceCount; ++i) {
result = nvmlDeviceGetHandleByIndex(i, &device);
if (NVML_SUCCESS != result) {
std::cerr << "Failed to get handle for device " << i << ": " << nvmlErrorString(result) << std::endl;
continue;
}

unsigned int infoCount = 0;
result = nvmlDeviceGetComputeRunningProcesses(device, &infoCount, NULL);
if (NVML_SUCCESS != result && NVML_ERROR_INSUFFICIENT_SIZE != result) {
std::cerr << "Failed to get compute running process count: " << nvmlErrorString(result) << std::endl;
continue;
}

std::vector<nvmlProcessInfo_t> localInfos(infoCount);
result = nvmlDeviceGetComputeRunningProcesses(device, &infoCount, localInfos.data());
if (NVML_SUCCESS != result) {
std::cerr << "Failed to get compute running processes: " << nvmlErrorString(result) << std::endl;
continue;
}

globalInfos.insert(globalInfos.end(), localInfos.begin(), localInfos.end());
}

result = nvmlShutdown();
if (NVML_SUCCESS != result) {
std::cerr << "Failed to shutdown NVML: " << nvmlErrorString(result) << std::endl;
return false;
}

return true;
}

extern "C" void *__libc_dlsym(void *__map, const char *__name);
extern "C" void *__libc_dlopen_mode( const char * pathname, int mode);
typedef void* (*DLSYM_FUNC)(void *, const char *);
static DLSYM_FUNC g_dlsym_func = NULL;

// void *real_dlsym(void *handle, const char *symbol) {
// static void* libc_hdl = dlopen("libc.so.6", RTLD_LAZY);
// g_dlsym_func = (DLSYM_FUNC)dlvsym(libc_hdl, "dlsym", "GLIBC_2.2.5");
// if (!g_dlsym_func) {
// static void* dl_hdl = __libc_dlopen_mode("libdl.so.2", RTLD_LAZY);
// g_dlsym_func = (DLSYM_FUNC)__libc_dlsym(dl_hdl, "dlsym");
// // std::cout << symbol << std::endl;
// }
// assert(g_dlsym_func);
// return g_dlsym_func(handle, symbol);
// }

static std:nce_flag init_flag;
void *real_dlsym(void *handle, const char *symbol) {
std::call_once(init_flag, []() {
libc_hdl = dlopen("libc.so.6", RTLD_LAZY);
g_dlsym_func = (DLSYM_FUNC)dlvsym(libc_hdl, "dlsym", "GLIBC_2.2.5");
});
assert(g_dlsym_func);
return g_dlsym_func(handle, symbol);
}


void *dlsym(void *handle, const char *symbol) {
if (0 != strncmp(symbol, "cu", 2)) {
return real_dlsym(handle, symbol);
}
static int g_hook_nums = sizeof(g_hooks_entry) / sizeof(g_hooks_entry[0]);
for (int i = 0; i < g_hook_nums; ++i) {
if (0 == strcmp(g_hooks_entry.sym_name, symbol)) {
return g_hooks_entry.func_ptr;
}
}
return real_dlsym(handle, symbol);
}

CUresult real_cuLaunchKernel(CUfunction f,
unsigned int gridDimX,
unsigned int gridDimY,
unsigned int gridDimZ,
unsigned int blockDimX,
unsigned int blockDimY,
unsigned int blockDimZ,
unsigned int sharedMemBytes,
CUstream hStream,
void **kernelParams,
void **extra){
using FuncPtr = CUresult (CUDAAPI *)(CUfunction ,
unsigned int ,
unsigned int ,
unsigned int ,
unsigned int ,
unsigned int ,
unsigned int ,
unsigned int ,
CUstream ,
void **,
void **);
dl_hdl_cn_ = dlopen("/usr/lib64/libcuda.so", RTLD_LAZY);
assert(dl_hdl_cn_);
static auto func_ptr = reinterpret_cast<FuncPtr>(real_dlsym(dl_hdl_cn_, "cuLaunchKernel"));

assert(func_ptr);
return func_ptr(f,
gridDimX,
gridDimY,
gridDimZ,
blockDimX,
blockDimY,
blockDimZ,
sharedMemBytes,
hStream,
kernelParams,
extra);
}

int calculateTimeToNextSecond() {
auto now = std::chrono::system_clock::now();
std::time_t now_time_t = std::chrono::system_clock::to_time_t(now);
struct std::tm *now_tm = std::localtime(&now_time_t);

++now_tm->tm_sec;
std::time_t next_second = std::mktime(now_tm);

auto next_second_time_point = std::chrono::system_clock::from_time_t(next_second);

auto duration = next_second_time_point - now;
auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();

return static_cast<int>(milliseconds);
}

CUresult cuLaunchKernel(CUfunction f,
unsigned int gridDimX,
unsigned int gridDimY,
unsigned int gridDimZ,
unsigned int blockDimX,
unsigned int blockDimY,
unsigned int blockDimZ,
unsigned int sharedMemBytes,
CUstream hStream,
void **kernelParams,
void **extra) {
// printf("Try to hook cuLaunchKernel.\n");
auto now = std::chrono::system_clock::now();
auto now_sec = std::chrono::time_point_cast<std::chrono::seconds>(now);
auto current_second = now_sec.time_since_epoch().count() % 60;

if (current_second % 2 != 0) {

} else {
std::cout << "Function cuLaunchKernel() is waiting token." << std::endl;
int tem_int_ms_duration=calculateTimeToNextSecond();
std::cout << "sleep "<< tem_int_ms_duration <<" ms" << std::endl;
usleep(tem_int_ms_duration *1000);

auto now3 = std::chrono::system_clock::now();
auto now3_c = std::chrono::system_clock::to_time_t(now3);
auto now3_ms = std::chrono::duration_cast<std::chrono::milliseconds>(now3.time_since_epoch()).count() % 1000;
std::cout << std:ut_time(std::localtime(&now3_c), "%Y-%m-%d %H:%M:%S");
std::cout << '.' << std::setfill('0') << std::setw(3) << now3_ms << std::endl;
}

return real_cuLaunchKernel(f,
gridDimX,
gridDimY,
gridDimZ,
blockDimX,
blockDimY,
blockDimZ,
sharedMemBytes,
hStream,
kernelParams,
extra);
}

CUresult real_cuMemAlloc_v2(CUdeviceptr *dptr, size_t bytesize) {
using FuncPtr = CUresult (CUDAAPI *)(CUdeviceptr *, size_t);
dl_hdl_cn_ = dlopen("/usr/lib64/libcuda.so", RTLD_LAZY);
assert(dl_hdl_cn_);
static auto func_ptr = reinterpret_cast<FuncPtr>(real_dlsym(dl_hdl_cn_, "cuMemAlloc_v2"));

return func_ptr(dptr, bytesize);
}

void processInfos() {
for (const auto& info : globalInfos) {
std::cout << "Process ID: " << info.pid << ", Used GPU Memory: " << info.usedGpuMemory / (1024 * 1024) << " MB" << std::endl;
}
}

CUresult cuMemAlloc_v2(CUdeviceptr *dptr, size_t bytesize) {
std::cout << "hook cuMemAlloc" << std::endl;
nvml_get_pid_mem();
for (const auto& info : globalInfos) {
std::cout << "Process ID: " << info.pid << ", Used GPU Memory: " << info.usedGpuMemory / (1024 * 1024) << " MB" << std::endl;
if (info.usedGpuMemory > global_memory_size_) {
WARN_PRT("Out of memory: %f MB vs %f MB\n",
(double)(info.usedGpuMemory) / MB_BYTES,
(double)global_memory_size_ / MB_BYTES);
return CUDA_ERROR_OUT_OF_MEMORY;
}
}
return real_cuMemAlloc_v2(dptr, bytesize);
}




this is the relative python subprocess script

import subprocess


command = [
"blender", "-b", "/home/tmp/monster/main.blend",
"-E", "CYCLES", "-f", "20", "--", "--cycles-device", "CUDA"
]


result = subprocess.run(command, capture_output=True, text=True)


print("STDOUT:", result.stdout)
print("STDERR:", result.stderr)


I want to use my C++ code to intercept the blender, then I can control the GPU memory usage and the GPU utility

Continue reading...