Notes

import torch
from torch.utils.data import DataLoader, Dataset
from pandas_ods_reader import read_ods
import pandas as pd
import nltk

max_sequence_length = 300

# Start, padding, and end tokens
START_TOKEN = '[START]'
PADDING_TOKEN = '[PAD]'
END_TOKEN = '[END]'

# Load your dataset into a DataFrame
data = read_ods("/content/sample_data/Fire_security.ods", columns=["question", "answer"])

questions = data['question'].tolist()
answers = data['answer'].tolist()

# Build vocabulary
def build_vocabulary(data):
vocab = set()
for sentence in data:
tokens = nltk.word_tokenize(sentence.lower())
vocab.update(tokens)
vocab = list(vocab)
vocab.insert(0, START_TOKEN)
vocab.append(PADDING_TOKEN)
vocab.append(END_TOKEN)
return vocab


combined_vocabulary = build_vocabulary(questions + answers)
combined_to_index = {word: idx for idx, word in enumerate(combined_vocabulary)}
index_to_combined = {idx: word for idx, word in enumerate(combined_vocabulary)}

# Convert sentences to indices

def sentences_to_indices(sentences, vocabulary, max_length):
return [[vocabulary.get(token, vocabulary[PADDING_TOKEN]) for token in nltk.word_tokenize(sentence.lower())][:max_length] for sentence in sentences]




questions_indices = sentences_to_indices(questions, combined_to_index, max_sequence_length)
answers_indices = sentences_to_indices(answers, combined_to_index, max_sequence_length)

# Define your Dataset class to handle question-answer pairs
class QADataset(Dataset):
def __init__(self, questions_indices, answers_indices):
self.questions_indices = questions_indices
self.answers_indices = answers_indices

def __len__(self):
return len(self.questions_indices)

def __getitem__(self, idx):
return {
'input_question': self.questions_indices[idx],
'target_answer': self.answers_indices[idx]
}

def collate_fn(self, batch):
combined_sequences = [item['input_question'] + [combined_to_index[PADDING_TOKEN]] + item['target_answer'] for item in batch]

max_sequence_length = max(len(seq) for seq in combined_sequences)



combined_sequences = [seq + [combined_to_index[PADDING_TOKEN]] * (max_sequence_length - len(seq)) for seq in combined_sequences]

return {
'combined_sequence': torch.tensor(combined_sequences)
}

# Create your Dataset and DataLoader
train_dataset = QADataset(questions_indices, answers_indices)
batch_size = 30 # Set your desired batch size
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_dataset.collate_fn)