[Python] why put the inference session to user compute stream make the inference 1000x slower?

I first create the session and bind the output and input tensors like following:

input_names: List[str] = []
x_dict = OrderedDict()
y_dict = OrderedDict()
output_names: List[str] = []
# by default it is already maximum optimization
session = ort.InferenceSession(model_onnx1.SerializeToString(), providers=['CUDAExecutionProvider'])
model_metadata = session.get_modelmeta()
binding = session.io_binding()
print("Model Metadata:")
print(f" Producer: {model_metadata.producer_name}")
print(f" Model Version: {model_metadata.version}")
print(f" Graph Name: {model_metadata.graph_name}")
print("\nInputs:")
for x_ in session.get_inputs():
print(f" Name: {x_.name}")
print(f" Shape: {x_.shape}")
print(f" Type: {x_.type}")
input_names.append(x_.name)
val_ = torch.zeros(
size=x_.shape,
dtype=TypeHelper.ort_type_to_torch_type(x_.type),
device=device
).contiguous()
binding.bind_input(
name=x_.name,
device_type=device.type,
device_id=device.index,
element_type=TypeHelper.ort_type_to_numpy_type(x_.type),
shape=x_.shape,
buffer_ptr=val_.data_ptr()
)
x_dict[x_.name] = val_

# Get output details (name, shape, type)
print("\nOutputs:")
for y_ in session.get_outputs():
print(f" Name: {y_.name}")
print(f" Shape: {y_.shape}")
print(f" Type: {y_.type}")
output_names.append(y_.name)
val_ = torch.zeros(
size=y_.shape,
dtype=TypeHelper.ort_type_to_torch_type(y_.type),
device=device
).contiguous()
binding.bind_output(
name=output_names[0],
device_type=device.type,
device_id=device.index,
element_type=TypeHelper.ort_type_to_numpy_type(y_.type),
shape=y_.shape,
buffer_ptr=val_.data_ptr()
)
y_dict[y_.name] = val_


Then I try to use the session for inference, the following dt is usually 1ms

# all tensor is on cuda:0 already
for name, val in zip(x_names, xs):
x_dict[name].copy_(val)
torch.cuda.synchronize(device=device)
# why sync here is necessary? shouldn't the previous operation be on default stream
# and automatically get synced on that stream for all operations?
# without this sync the session use incomplete data.
start_time = timeit.default_timer()
session.run_with_iobinding(
binding
)
ret = tuple(y_dict[name].clone() for name in y_names)
torch.cuda.synchronize(device=device)
dt = timeit.default_timer() - start_time
print(dt*1000)


The first strange part is why the copy_ require sync to cpu? shouldn't everything already on gpu and work on the default stream?

then I decide to use user compute stream, but following code result dt = 800ms

provider = 'CUDAExecutionProvider'
stream_ = torch.cuda.Stream(device=device)
session.set_providers(
[provider],
[
{"user_compute_stream": str(stream_.cuda_stream)}
]
)
option = session.get_provider_options()[provider]
print(option)
assert int(option["has_user_compute_stream"]) > 0
for name, val in zip(x_names, xs):
print(val.device)
x_dict[name].copy_(val, non_blocking=True)
stream_.wait_stream(torch.cuda.default_stream(device))
start_event.record(self.stream)
session.run_with_iobinding(
binding
)
torch.cuda.default_stream(device).wait_stream(stream_)
ret = tuple(y_dict[name].clone() for name in y_names)
end_event.record(torch.cuda.default_stream(device))
torch.cuda.synchronize()
dt_ms = start_event.elapsed_time(end_event)


why using user stream make it much slower?

Continue reading...