Notes

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset
from tqdm import tqdm

# Rest of the code remains the same as provided in the previous messages

# # Instantiate your model
# d_model = ... # Define the d_model value
# ffn_hidden = ... # Define the ffn_hidden value
# num_heads = ... # Define the num_heads value
# drop_prob = ... # Define the drop_prob value
# num_layers = ... # Define the num_layers value
d_model = 512
ffn_hidden = 2048
num_heads = 8
drop_prob = 0.1
num_layers = 1
max_sequence_length = 200
ans_vocab_size = len(answer_vocabulary)



model = Transformer(d_model,
ffn_hidden,
num_heads,
drop_prob,
num_layers,
max_sequence_length,
ans_vocab_size,
question_to_index,
answer_to_index,
START_TOKEN,
END_TOKEN,
PADDING_TOKEN)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

PAD_INDEX = question_to_index[PADDING_TOKEN]

# Define loss function and optimizer
criterion = nn.CrossEntropyLoss(ignore_index=answer_to_index[PADDING_TOKEN])
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Training loop
num_epochs = 10 # Set the number of training epochs
for epoch in range(num_epochs):
model.train() # Set the model to training mode
total_loss = 0.0

for batch in tqdm(train_loader, desc=f"Epoch {epoch + 1}"):
input_question = batch['input_question'].to(device)
target_answer = batch['target_answer'].to(device)
# Find the maximum sequence length in the current batch
# max_sequence_length = max(input_questions.size(1), target_answers.size(1))

# # # Pad input questions and target answers to the same length
# # input_questions = nn.functional.pad(input_questions, (0, max_sequence_length - input_questions.size(1)), value=PAD_INDEX)
# # target_answers = nn.functional.pad(target_answers, (0, max_sequence_length - target_answers.size(1)), value=PAD_INDEX)
optimizer.zero_grad()
try :
output = model(input_question, target_answer) # Clear gradients

except Exception as e:
print("error :" , e)
print("problematic question: ", input_question)
print("problematic answer: ", target_answer)

output_dim = output.shape[-1]
output = output.contiguous().view(-1, output_dim)
target_answer = target_answer.view(-1) # Flatten the target

loss = criterion(output, target_answer) # Calculate loss
loss.backward() # Backpropagation
optimizer.step() # Update weights

total_loss += loss.item()

avg_loss = total_loss / len(train_loader)
print(f"Epoch {epoch + 1}, Loss: {avg_loss:.4f}")

# Inference/Prediction
def predict_answer(input_question, model):
model.eval() # Set the model to evaluation mode
with torch.no_grad():
input_indices = sentence_to_indices(input_question, question_to_index)
input_indices = torch.tensor(input_indices).unsqueeze(0).to(device)

# Generate predictions
predicted_indices = model.generate(input_indices)

# Convert predicted indices to words
predicted_answer = indices_to_sentence(predicted_indices[0], index_to_kannada)
return predicted_answer

# input_question = "Your input question goes here."
# predicted_answer = predict_answer(input_question, model)
print(f"Input Question: {input_question}")
print(f"Predicted Answer: {predicted_answer}")