[Python] OSError: Can't synchronously read data (can't open directory) when trying to index...

I generated a hdf5 file with the code below:

# Create forecast training data file
import h5py
import hdf5plugin
#mmap_array = np.memmap('video_prediction_224.dat', dtype='uint8', mode='w+', shape=(n_images, 224, 224, 3))
sampling_interval = 2
n_valid = 307518 # index 307517 is the last valid index
n_trainval = 149680

n_test = 6145
chunk_shape = (1, stack_height+1, *output_img_shape)

batch_size = 8000
#for sampling_interval in sampling_interval_all:
resume_idx = 0
with h5py.File('video_prediction_224_testing.h5', 'w') as f:

#image_log = f.create_dataset('image_log', shape = (n_images, 224, 224, 3), dtype ='uint8', )

if resume_idx and 'image_log' in f:
# Resume mode - datasets already exist
image_log_ds = f['image_log']
pv_log_ds = f['pv_log']
pv_pred_ds = f['pv_pred']
print(f"Resuming from existing datasets. Current size: {image_log_ds.shape[0]}")
else:
# First run - create new datasets

trainval_group = f.create_group('trainval')
test_group = f.create_group('test')
image_log_trainval_ds = trainval_group.create_dataset(
'image_log',
shape=(n_trainval, stack_height+1, *output_img_shape),
chunks=chunk_shape,
compression=hdf5plugin.Blosc2(cname='zstd', clevel=1, filters=hdf5plugin.Blosc2.SHUFFLE),
dtype='uint8'
)
pv_log_trainval_ds = trainval_group.create_dataset(
'pv_log',
shape=(n_trainval, stack_height+1),
dtype='float64'
)
pv_pred_trainval_ds = trainval_group.create_dataset(
'pv_pred',
shape=(n_trainval,),
dtype='float64'
)
image_log_test_ds = test_group.create_dataset(
'image_log',
shape=(n_test, stack_height+1, *output_img_shape),
chunks=chunk_shape,
compression=hdf5plugin.Blosc2(cname='zstd', clevel=1, filters=hdf5plugin.Blosc2.SHUFFLE),
dtype='uint8'
)
pv_log_test_ds = test_group.create_dataset(
'pv_log',
shape=(n_test, stack_height+1),
dtype='float64'
)
pv_pred_test_ds = test_group.create_dataset(
'pv_pred',
shape=(n_test,),
dtype='float64'
)
print("Creating new datasets")


#image_log = np.empty([0,stack_height+1]+output_img_shape,dtype = 'uint8')
#pv_log = np.empty([0, stack_height+1])
#pv_pred = np.empty([0])

last_valid_index = 0
curr_trainval_size = 0
curr_test_size = 0

tic = time.process_time()
for b in range(resume_idx if (resume_idx and 'image_log' in f) else 0, 8000, batch_size):
current_batch_size = min(batch_size, n_images-b)
# Initialize variables to save pv values
#image_log_batch = np.zeros([n_images,stack_height+1]+output_img_shape,dtype = 'uint8')
idx_test_batch = get_subarray_between_values(np.asarray(idx_test),b,b+current_batch_size)

image_log_batch = np.zeros([current_batch_size,stack_height+1]+output_img_shape,dtype = 'uint8')

#all_times_batch = all_times[b*batch_size : b*batch_size + current_batch_size]
pv_log_batch = np.zeros((current_batch_size,stack_height+1))
pv_pred_batch = np.zeros(current_batch_size)
validity_mask = np.ones(current_batch_size,dtype = bool)



sampling_interval_td = datetime.timedelta(minutes = sampling_interval) - datetime.timedelta(seconds=1)
for i in range(current_batch_size):
count = b+i


# See if the specified sampling frequency is met
if all_times[count] - all_times[last_valid_index] > sampling_interval_td:

# Collecting groud truth for predicted value
pred_time = all_times[count]+datetime.timedelta(minutes=forecast_horizon)

pv_pred_idx = find_time_within_nparray(pv_data.index,pred_time)
if pv_pred_idx is None:# if prediction ground truth not found
validity_mask = False
#print(all_times,'has no PV pred')
else:
pv_pred_batch = pv_data.iloc[pv_pred_idx]

# Collecting image log and PV log
for j in range(stack_height+1):
log_time = all_times[count] - datetime.timedelta(minutes = j)
# Collecting a stack of image
log_time_idx = find_time_within_nparray(all_times,log_time)
if log_time_idx is not None:
image_log_batch[i,j] = all_images[log_time_idx]
else:
validity_mask = False
#print(all_times[count],'has no image log')
break

# Collecting a stack of PV value
pv_log_idx = find_time_within_nparray(pv_data.index,log_time)
# Check if PV value present
if pv_log_idx is None:
validity_mask = False
#print(all_times[count],'has no PV log')
break
else:
pv_log_batch[i,j] = pv_data.iloc[pv_log_idx]

else: # if this is in between the sampling points, discard
validity_mask = False

if validity_mask:
last_valid_index = count

# Prompt progress of current work

print('processed {0}/{1} images'.format(b+current_batch_size,len(all_times)))


# Only pick out the valid time points
#all_times_batch = all_times_batch[validity_mask]
test_mask = np.zeros(current_batch_size, dtype=bool)
test_mask[idx_test_batch-b] = True
validity_test_mask = validity_mask & test_mask
trainval_mask = np.ones(current_batch_size, dtype=bool)
trainval_mask[idx_test_batch-b] = False
validity_trainval_mask = validity_mask & trainval_mask

image_log_trainval_batch = image_log_batch[validity_trainval_mask]
pv_log_trainval_batch = pv_log_batch[validity_trainval_mask]
pv_pred_trainval_batch = pv_pred_batch[validity_trainval_mask]

image_log_test_batch = image_log_batch[validity_test_mask]
pv_log_test_batch = pv_log_batch[validity_test_mask]
pv_pred_test_batch = pv_pred_batch[validity_test_mask]
# Store information

print("storing data")
#image_log_ds[curr_size:curr_size+validity_mask.sum()] = image_log_batch
#pv_log_ds[curr_size:curr_size+validity_mask.sum()] = pv_log_batch
#pv_pred_ds[curr_size:curr_size+validity_mask.sum()] = pv_pred_batch
image_log_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = image_log_trainval_batch
#print("image_log_trainval_batch: ",image_log_trainval_batch)
#print("image_log_trainval_ds.shape: ",image_log_trainval_ds.shape)
print("stored trainval image log")
pv_log_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = pv_log_trainval_batch
print("stored trainval pv log")
pv_pred_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = pv_pred_trainval_batch
print("stored trainval pv pred")
image_log_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = image_log_test_batch
print("stored test image log")
pv_log_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = pv_log_test_batch
pv_pred_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = pv_pred_test_batch

curr_trainval_size += validity_trainval_mask.sum()
print("trainval_size: ",validity_trainval_mask.sum())
curr_test_size += validity_test_mask.sum()
print("test_size: ",validity_test_mask.sum())

print('For sampling frequency: ',sampling_interval,' minutes')
#print('Expected finishing time:', datetime.datetime.now()+
# datetime.timedelta(seconds = (time.process_time() - tic)*(len(all_times)/(b+batch_size))))
f.flush()
del image_log_trainval_batch
del pv_log_trainval_batch
del pv_pred_trainval_batch
del image_log_test_batch
del pv_log_test_batch
del pv_pred_test_batch

pred_folder_child = os.path.join(pred_folder,'frequency_'+str(sampling_interval))
#store_trainval_test(all_times,image_log,pv_log,pv_pred,pred_folder_child)


The file is created with no errors, but when I try to access the image datasets:

# generate handler for the hdf5 data
forecast_dataset = h5py.File(data_path, 'r')

# show structure of the hdf5 data
def get_all(name):
if name!=None:
print(forecast_dataset[name])

forecast_dataset.visit(get_all)

img = forecast_dataset["trainval"]["image_log"][0]
print(img)


I get the error,

data_folder: d:\PVOutputPrediction\data
data_path: d:\PVOutputPrediction\video_prediction_224_second.h5
output_folder: d:\PVOutputPrediction\model_output\UNet_sky_image_PV_mapping
<HDF5 group "/test" (3 members)>
<HDF5 dataset "image_log": shape (6145, 16, 224, 224, 3), type "|u1">
<HDF5 dataset "pv_log": shape (6145, 16), type "<f8">
<HDF5 dataset "pv_pred": shape (6145,), type "<f8">
<HDF5 group "/trainval" (3 members)>
<HDF5 dataset "image_log": shape (149680, 16, 224, 224, 3), type "|u1">
<HDF5 dataset "pv_log": shape (149680, 16), type "<f8">
<HDF5 dataset "pv_pred": shape (149680,), type "<f8">
---------------------------------------------------------------------------
OSError Traceback (most recent call last)
Cell In[7], line 3
1 import matplotlib.pyplot as plt
----> 3 img = forecast_dataset["trainval"]["image_log"][0]
4 print(img)

File h5py/_objects.pyx:56, in h5py._objects.with_phil.wrapper()

File h5py/_objects.pyx:57, in h5py._objects.with_phil.wrapper()

File d:\PVOutputPrediction\pytorch_env\Lib\site-packages\h5py\_hl\dataset.py:820, in Dataset.__getitem__(self, args, new_dtype)
818 if self._fast_read_ok and (new_dtype is None):
819 try:
--> 820 return self._fast_reader.read(args)
821 except TypeError:
822 pass # Fall back to Python read pathway below

File h5py/_selector.pyx:376, in h5py._selector.Reader.read()

OSError: Can't synchronously read data (can't open directory)


The PV datasets, which contain scalar values, can be accessed without errors. I've ran chkdsk for both my drives and re-ran the script several times to no avail. I have 300 GB available on my D Drive where I'm storing the file. I also tried running it without the chunking and blosc compression, but it gave the same error

Continue reading...