[Python] Huggingface Trainer unable to create tensor

I have a DatasetDict object of comments with text and integer label fields:

comments
DatasetDict({
train: Dataset({
features: ['text', 'label'],
num_rows: 2000
})
validation: Dataset({
features: ['text', 'label'],
num_rows: 500
})
test: Dataset({
features: ['text', 'label'],
num_rows: 500
})
})


I also have a tokenizer that uses padding and truncation:

def tokenize(batch):
return tokenizer(batch["text"], padding=True, truncation=True)

comments_encoded = comments.map(tokenize, batched=True, batch_size=None)


However, when I try to run the following, the Trainer function throws an error that I should try to implement padding and truncation. Since I'm doing so in the tokenizer, I'm not sure what's causing this error.

def extract_hidden_states(batch):
inputs = {k: v for k, v in batch.items() if k in tokenizer.model_input_names}

# Ensure all inputs are tensors before sending them to device
inputs = {k: v.clone().detach().to(device) for k, v in inputs.items()}

with torch.no_grad():
outputs = model(**inputs) # Unpacking inputs properly
last_hidden_state = outputs.last_hidden_state

return {"hidden_state": last_hidden_state[:, 0].cpu().numpy()}

# convert input_ids and attention_mask columns to "torch" format
comments_encoded.set_format("torch", columns=["input_ids", "attention_mask", "label"])

# extract hidden states
comments_hidden = comments_encoded.map(extract_hidden_states, batched=True)

import numpy as np

X_train = np.array(comments_hidden["train"]["hidden_state"])
X_valid = np.array(comments_hidden["validation"]["hidden_state"])
y_train = np.array(comments_hidden["train"]["label"])
y_valid = np.array(comments_hidden["validation"]["label"])
X_train.shape, X_valid.shape

# load a pre-trained model
from transformers import AutoModelForSequenceClassification

num_labels = 9
model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))

# define performance metrics for classification
from sklearn.metrics import accuracy_score, f1_score

def compute_metrics(pred):
labels = pred.label_ids
preds = pred.predictions.argmax(-1)
f1 = f1_score(labels, preds, average="weighted")
acc = accuracy_score(labels, preds)
return{"accuracy": acc, "f1":f1}

# login to the Hugging Face Hub
from huggingface_hub import notebook_login

notebook_login() # use personal token to login

# define training parameters
from transformers import Trainer, TrainingArguments

batch_size = 64
logging_steps = len(comments["train"]) # batch size
model_name = f"{model_ckpt}-finetuned-sms"
training_args = TrainingArguments(output_dir=model_name,
num_train_epochs = 4,
learning_rate=2e-5,
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
weight_decay=0.01,
evaluation_strategy="epoch",
disable_tqdm=False,
logging_steps=logging_steps,
push_to_hub=True,
log_level="error"
)

# instantiate and fine-tune the model
from transformers import Trainer

trainer = Trainer(model=model, args=training_args,
compute_metrics=compute_metrics,
train_dataset=comments_encoded["train"],
eval_dataset=comments_encoded["validation"],
tokenizer=tokenizer)

trainer.train()


Full error:

ValueError Traceback (most recent call last)
File ~\Anaconda3\envs\ml\lib\site-packages\transformers\tokenization_utils_base.py:777, in BatchEncoding.convert_to_tensors(self, tensor_type, prepend_batch_axis)
776 if not is_tensor(value):
--> 777 tensor = as_tensor(value)
779 # Removing this for now in favor of controlling the shape with `prepend_batch_axis`
780 # # at-least2d
781 # if tensor.ndim > 2:
782 # tensor = tensor.squeeze(0)
783 # elif tensor.ndim < 2:
784 # tensor = tensor[None, :]

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\tokenization_utils_base.py:739, in BatchEncoding.convert_to_tensors.<locals>.as_tensor(value, dtype)
738 return torch.from_numpy(np.array(value))
--> 739 return torch.tensor(value)

ValueError: too many dimensions 'str'

The above exception was the direct cause of the following exception:

ValueError Traceback (most recent call last)
Cell In[21], line 10
2 from transformers import Trainer
4 trainer = Trainer(model=model, args=training_args,
5 compute_metrics=compute_metrics,
6 train_dataset=comments_encoded["train"],
7 eval_dataset=comments_encoded["validation"],
8 tokenizer=tokenizer)
---> 10 trainer.train()

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\trainer.py:2236, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
2233 try:
2234 # Disable progress bars when uploading models during checkpoints to avoid polluting stdout
2235 hf_hub_utils.disable_progress_bars()
-> 2236 return inner_training_loop(
2237 args=args,
2238 resume_from_checkpoint=resume_from_checkpoint,
2239 trial=trial,
2240 ignore_keys_for_eval=ignore_keys_for_eval,
2241 )
2242 finally:
2243 hf_hub_utils.enable_progress_bars()

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\trainer.py:2508, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
2506 update_step += 1
2507 num_batches = args.gradient_accumulation_steps if update_step != (total_updates - 1) else remainder
-> 2508 batch_samples, num_items_in_batch = self.get_batch_samples(epoch_iterator, num_batches, args.device)
2509 for i, inputs in enumerate(batch_samples):
2510 step += 1

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\trainer.py:5224, in Trainer.get_batch_samples(self, epoch_iterator, num_batches, device)
5222 for _ in range(num_batches):
5223 try:
-> 5224 batch_samples += [next(epoch_iterator)]
5225 except StopIteration:
5226 break

File ~\Anaconda3\envs\ml\lib\site-packages\accelerate\data_loader.py:566, in DataLoaderShard.__iter__(self)
564 # We iterate one batch ahead to check when we are at the end
565 try:
--> 566 current_batch = next(dataloader_iter)
567 except StopIteration:
568 yield

File ~\Anaconda3\envs\ml\lib\site-packages\torch\utils\data\dataloader.py:631, in _BaseDataLoaderIter.__next__(self)
628 if self._sampler_iter is None:
629 # TODO(https://github.com/pytorch/pytorch/issues/76750)
630 self._reset() # type: ignore[call-arg]
--> 631 data = self._next_data()
632 self._num_yielded += 1
633 if self._dataset_kind == _DatasetKind.Iterable and \
634 self._IterableDataset_len_called is not None and \
635 self._num_yielded > self._IterableDataset_len_called:

File ~\Anaconda3\envs\ml\lib\site-packages\torch\utils\data\dataloader.py:675, in _SingleProcessDataLoaderIter._next_data(self)
673 def _next_data(self):
674 index = self._next_index() # may raise StopIteration
--> 675 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
676 if self._pin_memory:
677 data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)

File ~\Anaconda3\envs\ml\lib\site-packages\torch\utils\data\_utils\fetch.py:54, in _MapDatasetFetcher.fetch(self, possibly_batched_index)
52 else:
53 data = self.dataset[possibly_batched_index]
---> 54 return self.collate_fn(data)

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\data\data_collator.py:272, in DataCollatorWithPadding.__call__(self, features)
271 def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
--> 272 batch = pad_without_fast_tokenizer_warning(
273 self.tokenizer,
274 features,
275 padding=self.padding,
276 max_length=self.max_length,
277 pad_to_multiple_of=self.pad_to_multiple_of,
278 return_tensors=self.return_tensors,
279 )
280 if "label" in batch:
281 batch["labels"] = batch["label"]

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\data\data_collator.py:67, in pad_without_fast_tokenizer_warning(tokenizer, *pad_args, **pad_kwargs)
64 tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = True
66 try:
---> 67 padded = tokenizer.pad(*pad_args, **pad_kwargs)
68 finally:
69 # Restore the state of the warning.
70 tokenizer.deprecation_warnings["Asking-to-pad-a-fast-tokenizer"] = warning_state

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\tokenization_utils_base.py:3407, in PreTrainedTokenizerBase.pad(self, encoded_inputs, padding, max_length, pad_to_multiple_of, padding_side, return_attention_mask, return_tensors, verbose)
3404 batch_outputs[key] = []
3405 batch_outputs[key].append(value)
-> 3407 return BatchEncoding(batch_outputs, tensor_type=return_tensors)

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\tokenization_utils_base.py:241, in BatchEncoding.__init__(self, data, encoding, tensor_type, prepend_batch_axis, n_sequences)
237 n_sequences = encoding[0].n_sequences
239 self._n_sequences = n_sequences
--> 241 self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis)

File ~\Anaconda3\envs\ml\lib\site-packages\transformers\tokenization_utils_base.py:793, in BatchEncoding.convert_to_tensors(self, tensor_type, prepend_batch_axis)
788 if key == "overflowing_tokens":
789 raise ValueError(
790 "Unable to create tensor returning overflowing tokens of different lengths. "
791 "Please see if a fast version of this tokenizer is available to have this feature available."
792 ) from e
--> 793 raise ValueError(
794 "Unable to create tensor, you should probably activate truncation and/or padding with"
795 " 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your"
796 f" features (`{key}` in this case) have excessive nesting (inputs type `list` where type `int` is"
797 " expected)."
798 ) from e
800 return self

ValueError: Unable to create tensor, you should probably activate truncation and/or padding with 'padding=True' 'truncation=True' to have batched tensors with the same length. Perhaps your features (`label` in this case) have excessive nesting (inputs type `list` where type `int` is expected).

Continue reading...