[Python] How to Ensure Reproducibility in a Link Prediction GNN Model in PyTorch?

I'm training a Graph Neural Network (GNN) model for link prediction in PyTorch. I’ve taken several steps to ensure reproducibility, but I’m still getting different results on different runs, even though I expect consistent results.

Here’s what I’ve done to ensure reproducibility:

Set all seeds for relevant libraries (PyTorch, NumPy, random, etc.). Ensured that my training data is static and not randomized during the training process. However, I’ve noticed that the model generates identical feature vectors in the first epoch across runs, but the edge features (output of the link prediction layer) are not the same. Based on that, I suspect the issue might be with the link prediction layer itself.

I'm relatively new to GNNs and PyTorch, so I would appreciate any insights or suggestions or help on the following:

1. Why the edge features might not be consistent even if the other model parameters seem reproducible.
2. Any potential code or model-related changes to help ensure the results are the same on each run.
3. How to verify that all randomness sources are controlled in the link prediction layer or any other components of the model.

Here’s my code:

def split_data_balance(edge_index, edge_label, test_size=0.2, random_state=seed_value):
edge_label = edge_label.numpy()

train_idx, test_idx = train_test_split(range(len(edge_label)), test_size=test_size, stratify=edge_label, random_state=random_state)

train_edge_index = edge_index[:, train_idx]
test_edge_index = edge_index[:, test_idx]
train_edge_label = edge_label[train_idx]
test_edge_label = edge_label[test_idx]

train_edge_label = torch.tensor(train_edge_label)
test_edge_label = torch.tensor(test_edge_label)

return train_edge_index, test_edge_index, train_edge_label, test_edge_label

edg_idx_train, edg_idx_test, edg_lable_train, edg_lable_test = split_data_balance(edge_index, edge_weight)
class GCN_linkPrediction(torch.nn.Module):
def __init__(self, input_dim, hidden_dim, out_dim, num_layers,
dropout, return_embeds=False):

super(GCN_linkPrediction, self).__init__()

self.convs = torch.nn.ModuleList([GCNConv(in_channels=input_dim if i == 0 else hidden_dim, out_channels=hidden_dim) for i in range(num_layers-1)])
self.batchNorm = torch.nn.ModuleList([torch.nn.BatchNorm1d(num_features=hidden_dim) for i in range(num_layers)])
self.convs.append(GCNConv(in_channels=hidden_dim, out_channels=out_dim))

self.dropout = dropout

self.return_embeds = return_embeds

self.linear = torch.nn.Linear(out_dim*2, 3)


def reset_parameters(self):
for conv in self.convs:
conv.reset_parameters()
for batchN in self.batchNorm:
batchN.reset_parameters()

def forward(self, x, adj_t):
for i in range(len(self.convs)-1):
x = self.convs(x, adj_t) # x is the node feature tensor and adj_t is the edge_index tensor
x = self.batchNorm(x)
x = torch.nn.functional.relu(x)
x = F.dropout(x, p=self.dropout, training=self.training)

x = self.convs[-1](x, adj_t)

out = x

return out


def link_prediction(self, x, adj_t):
scr, tgt = adj_t
nodef_scr = x[scr]
nodef_tgt = x[tgt]

link_feature = torch.cat([nodef_scr, nodef_tgt], dim=-1)

link = self.linear(link_feature)

return link
def train(model, x, adj_t, lable, optimizer, loss_fn):

model.train()
optimizer.zero_grad()

x = model(x, adj_t)
link_weight = model.link_prediction(x, adj_t)
y_pred = torch.argmax(link_weight, dim=1)

loss = loss_fn(link_weight, lable.long())

loss.backward()
optimizer.step()

return loss.item(), link_weight, y_pred

def test(model, x, adj_t, lable, save_model_results=False):
model.eval()

x = model(x, adj_t)
link_weight = model.link_prediction(x, adj_t)

y_pred = torch.argmax(link_weight, dim=1)
true_pred = (y_pred == lable).sum().item()
accuracy = true_pred / len(adj_t)

return accuracy, link_weight, y_pred
args = {'num_layers': 4,
'hidden_dim': 150,
'out_dim': 60,
'dropout': 0.5,
'lr': 0.01,
'epochs': 1,
}

model = GCN_linkPrediction(node_features.shape[1], args['hidden_dim'],
args['out_dim'], args['num_layers'],
args['dropout'])
seed_value = 42

random.seed(seed_value)
np.random.seed(seed_value)
torch.manual_seed(seed_value)

torch.backends.cudnn.deterministic = True # Force deterministic algorithms
torch.backends.cudnn.benchmark = False # Disable cuDNN's benchmarking feature (can cause non-determinism)
torch.use_deterministic_algorithms(True) # Set torch to use deterministic algo

model.reset_parameters()

optimizer = torch.optim.Adam(model.parameters(), lr=args['lr'])
loss_fn = torch.nn.CrossEntropyLoss() # classification

best_model = None
best_test_acc = 0

train_losses = []

range_epoch = args['epochs']
for epoch in range(1,range_epoch+1):
train_loss, raw_train, train_pre = train(model, node_features, edg_idx_train, edg_lable_train, optimizer, loss_fn)
train_losses.append(train_loss)

with torch.no_grad():
result, raw_test, test_pre = test(model, node_features, edg_idx_test, edg_lable_test)

prediction_accuracy = result

print(f'Epoch: {epoch:02d}, '
f'Loss: {train_loss:.4f}, '
f'Test: {100 * prediction_accuracy:.2f}%, ')

Continue reading...