Notes

Certainly! Here's the complete training code with the missing logic filled in:

```python
import numpy as np

# Define the get_device function
def get_device():
if torch.cuda.is_available():
return torch.device('cuda')
else:
return torch.device('cpu')

# Your existing code...
# (Define classes, Tokenizer, TokenEmbedding, PositionalEncoding, etc.)

# Rest of your code...

# Create the model
model = AutoregressiveWrapper(LanguageModel(
embedding_dimension=embedding_dimension,
number_of_tokens=number_of_tokens,
number_of_heads=4,
number_of_layers=3,
dropout_rate=0.1,
max_sequence_length=max_sequence_length
)).to(get_device()) # Move the model to the same device

# Tokenize and pad the training data
tokenized_and_padded_training_data = tokenize_and_pad_training_data(max_sequence_length, tokenizer, final_data)
tokenized_and_padded_training_data = torch.tensor(tokenized_and_padded_training_data, dtype=torch.long, device=get_device())

# Create training sequences
sequences = create_training_sequences(max_sequence_length, tokenized_and_padded_training_data)

# Move sequences to the same device
sequences = [sequence.to(get_device()) for sequence in sequences]

# Move the optimizer to the same device as well
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)

# Training parameters
epochs = 100
batch_size = 8

# Train the model
for epoch in range(epochs):
model.train() # Set the model to training mode
epoch_losses = []

for i in range(0, len(sequences), batch_size):
batch = sequences[i: i + batch_size]

# Create the input tensor and target tensor for the batch
input_batch = torch.stack([sequence[:, :-1] for sequence in batch], dim=0)
target_batch = torch.stack([sequence[:, 1:] for sequence in batch], dim=0)

# Mask for the input batch
mask_batch = (input_batch != tokenizer.character_to_token('<pad>')).float()

# Move input, target, and mask tensors to the same device
input_batch = input_batch.to(get_device())
target_batch = target_batch.to(get_device())
mask_batch = mask_batch.to(get_device())

# Forward pass
output_batch, target_batch = model(input_batch, mask_batch)

# Calculate the loss
loss = torch.nn.functional.cross_entropy(output_batch.view(-1, output_batch.size(-1)), target_batch.view(-1), ignore_index=tokenizer.character_to_token('<pad>'))

# Backpropagation and optimization
optimizer.zero_grad()
loss.backward()
optimizer.step()

epoch_losses.append(loss.item())

average_loss = np.mean(epoch_losses)
print(f'Epoch [{epoch+1}/{epochs}], Loss: {average_loss}')

print('Training complete!')
```

This code completes your training logic by performing forward passes, calculating losses, and applying backpropagation and optimization for each batch. It also prints the average loss for each epoch. Remember to replace the missing logic and customize the training parameters as needed.