[Python] ValueError: Dimensions X and y are not compatible in...

I'm trying to create TensorFlow data batches for training using tf.data.Dataset.from_tensor_slices((X, y)). However, I'm encountering the following error:

ValueError: Dimensions 800 and 200 are not compatible


Here's the relevant part of my code:

data = tf.data.Dataset.from_tensor_slices((X_train, y_train))


From debugging, I found that:

len(X_train),len(y_train)

output:
(800,800)

• 
  Why it is not taking values as it mismatches as it is training and testing dataset I have taken 0.8 as the training set?
  
  
• 
  What is the best way to ensure that X and y always stay aligned — especially if they are shuffled or filtered before batching?
  

BATCH_SIZE = 32

def create_data_batches(X,y=None,batch_size=BATCH_SIZE, valid_data=False, test_data = False):
'''
Creats batches of data out of image (X) and label(Y) patrs.
Shuffles the data if it's tarning data but dosen't shuffel if it's validation data.
Also accepts test data as input (no labels).
'''
# If the data is a test data set, we don't have labels
if test_data:
print("Craeting test data batches....")
# fist convert the data (which is in tensors) into datasets from tensor slices
# test data only contain file path no labels
data = tf.data.Dataset.from_tensor_slices(tf.constant(X)) # this says is pass me some tenses and I'll create a dataset out of that.
data_batch = data.map(process_image).batch(BATCH_SIZE) # .batch converts the dataset into number of batches
return data_batch

# If the data is valid dataset, we dont't need to shuffle it
elif valid_data:
print("Creating Validation data batches ...")
# For training and validation
data = tf.data.Dataset.from_tensor_slices(tf.constant(X),
tf.constant(y))

data_batch = data.map(get_image_label).batch(BATCH_SIZE)
return data_batch
else:
print("Creating Traning data batches ...")
# Turining filepath and labels turning into tensors
# For training and validation
data = tf.data.Dataset.from_tensor_slices(tf.constant(X),
tf.constant(y))

# Shuffling pathnames and labels before mapping processor function is faster than shuffling images
data = data.shuffle(buffer_size=len(X)) # all this is saying is,take this data, which is a TensorFlow data set of the data, so the file names and the labels and then shuffle it up and we want to shuffle the number of examples that is here we have taken X

# create (image,label) tuples (this also turns the image into preprocessed image)
data = data.map(get_image_label)

# Turn the training data into batches
data_batch = data.batch(BATCH_SIZE)

return data_batch

# Creating traning and validation data batches
train_data = create_data_batches(X_train,y_train)
val_data = create_data_batches(X_val,y_val,valid_data=True)


Error message:

Creating Traning data batches ...
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[149], line 2
1 # Creating traning and validation data batches
----> 2 train_data = create_data_batches(X_train,y_train)
3 val_data = create_data_batches(X_val,y_val,valid_data=True)

Cell In[148], line 31, in create_data_batches(X, y, batch_size, valid_data, test_data)
28 print("Creating Traning data batches ...")
29 # Turining filepath and labels turning into tensors
30 # For training and validation
---> 31 data = tf.data.Dataset.from_tensor_slices(tf.constant(X),
32 tf.constant(y))
34 # Shuffling pathnames and labels before mapping processor function is faster than shuffling images
35 data = data.shuffle(buffer_size=len(X)) # all this is saying is,take this data, which is a TensorFlow data set of the data, so the file names and the labels and then shuffle it up and we want to shuffle the number of examples that is here we have taken X

File ~\Machine Learning\ML-ZTM Course\dog_vision\env\Lib\site-packages\tensorflow\python\data\ops\dataset_ops.py:827, in DatasetV2.from_tensor_slices(tensors, name)
823 # Loaded lazily due to a circular dependency (dataset_ops ->
824 # from_tensor_slices_op -> dataset_ops).
825 # pylint: disable=g-import-not-at-top,protected-access
826 from tensorflow.python.data.ops import from_tensor_slices_op
--> 827 return from_tensor_slices_op._from_tensor_slices(tensors, name)

File ~\Machine Learning\ML-ZTM Course\dog_vision\env\Lib\site-packages\tensorflow\python\data\ops\from_tensor_slices_op.py:25, in _from_tensor_slices(tensors, name)
24 def _from_tensor_slices(tensors, name=None):
---> 25 return _TensorSliceDataset(tensors, name=name)

File ~\Machine Learning\ML-ZTM Course\dog_vision\env\Lib\site-packages\tensorflow\python\data\ops\from_tensor_slices_op.py:53, in _TensorSliceDataset.__init__(self, element, is_files, name)
44 for t in self._tensors[1:]:
45 batch_dim.assert_is_compatible_with(
46 tensor_shape.Dimension(
47 tensor_shape.dimension_value(t.get_shape()[0])))
49 variant_tensor = gen_dataset_ops.tensor_slice_dataset(
50 self._tensors,
51 output_shapes=structure.get_flat_tensor_shapes(self._structure),
52 is_files=is_files,
---> 53 metadata=self._metadata.SerializeToString())
54 super().__init__(variant_tensor)

File ~\Machine Learning\ML-ZTM Course\dog_vision\env\Lib\site-packages\tensorflow\python\data\ops\dataset_ops.py:671, in DatasetV2._metadata(self)
669 """Helper for generating dataset metadata."""
670 metadata = dataset_metadata_pb2.Metadata()
--> 671 if self._name:
672 metadata.name = _validate_and_encode(self._name)
673 return metadata

File ~\Machine Learning\ML-ZTM Course\dog_vision\env\Lib\site-packages\tensorflow\python\framework\ops.py:323, in _EagerTensorBase.__bool__(self)
321 x = self._numpy()
322 if isinstance(x, np.ndarray):
--> 323 return bool(x.size > 0 and x)
324 else:
325 return bool(x)

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

Continue reading...