I'm trying to compute a measure of semantic similarity between titles of scientific publications using SPECTER2, but the model performs poorly. Here is my code: from transformers import AutoTokenizer from adapters import AutoAdapterModel import torch # load model and tokenizer tokenizer = AutoTokenizer.from_pretrained("allenai/specter2_base") # load base model model = AutoAdapterModel.from_pretrained("allenai/specter2_base") # load the adapter(s) as per the required task, provide an identifier for the adapter in load_as argument and activate it model.load_adapter("allenai/specter2", source="hf", load_as="proximity", set_active=True) def novel2(txt): assert isinstance(txt, list), "Input should be a list of strings" assert all(isinstance(s, str) for s in txt), ( "All elements in the list should be strings" ) # preprocess the input inputs = tokenizer( txt, padding=True, truncation=True, return_tensors="pt", return_token_type_ids=False, max_length=512, ) output = model(**inputs) # take the first token in the batch as the embedding embeddings = output.last_hidden_state[:, 0, :] print(embeddings.shape) n = len(txt) dim = (n * (n - 1)) // 2 dists = torch.zeros(dim) pos = 0 for i in range(n): for j in range(i + 1, n): dists[pos] = torch.nn.functional.cosine_similarity( embeddings[i, :].reshape(1, -1), embeddings[j, :].reshape(1, -1) ) pos += 1 return dists Testing it with: novel2(["BERT", "Attention is all you need", "How the Romans conquered the world"]) Returns tensor([0.8831, 0.8758, 0.8812], grad_fn=<CopySlices>) These would be the scores for the comparisons document 1 - document 2, document 1 - document 3, document 2 - document 3. I would have expected a very low score for the latter 2 comparisons. Instead, the first document is similar to the second as much as the second is similar to the third. And the droppage when comparing the first model with the third is very low. Continue reading...
