Notes

class JobPositionDataset(Dataset):
def __init__(self, dataframe, tokenizer, max_len):
self.len = len(dataframe)
self.data = dataframe
self.tokenizer = tokenizer
self.max_len = max_len

def __getitem__(self, index):
text = str(self.data.email_body.iloc[index])

# Process labels: Split by space, strip whitespace, and convert to integers
try:
labels = [int(label.strip()) for label in str(self.data.numeric_labels.iloc[index]).split()]
except ValueError:
# Handle rows where conversion to int fails, potentially logging the error or setting a default value
print(f"Conversion error at index {index} with value {self.data.numeric_labels.iloc[index]}")
labels = [0] * self.max_len # Example default handling, adjust as needed

inputs = self.tokenizer.encode_plus(
text,
None,
add_special_tokens=True,
max_length=self.max_len,
padding='max_length',
truncation=True,
return_attention_mask=True,
return_tensors="pt"
)

input_ids = inputs['input_ids'].squeeze()
attention_mask = inputs['attention_mask'].squeeze()

# Ensure labels are padded correctly to match input_ids length
labels = labels[:self.max_len] + [0] * (self.max_len - len(labels))

return {
'input_ids': input_ids,
'attention_mask': attention_mask,
'labels': torch.tensor(labels[:self.max_len], dtype=torch.long)
}

def __len__(self):
return self.len