Notes

class Tokenizer:

def __init__(self):
self.dictionary = {}
self.reverse_dictionary = {}

# Add the padding token
self.__add_to_dict('<pad>')

# Add characters and numbers to the dictionary
for i in range(10):
self.__add_to_dict(str(i))
for i in range(26):
self.__add_to_dict(chr(ord('a') + i))

# Add space and punctuation to the dictionary
self.__add_to_dict('.')
self.__add_to_dict(' ')

def __add_to_dict(self, character):
if character not in self.dictionary:
self.dictionary[character] = len(self.dictionary)
self.reverse_dictionary[self.dictionary[character]] = character

def tokenize(self, text):
return [self.dictionary[c] for c in text]

def character_to_token(self, character):
return self.dictionary[character]

def token_to_character(self, token):
return self.reverse_dictionary[token]

def size(self):
return len(self.dictionary)


training_data = '. '.join([
'cats rule the world',
'dogs are the best',
'elephants have long trunks',
'monkeys like bananas',
'pandas eat bamboo',
'tigers are dangerous',
'zebras have stripes',
'lions are the kings of the savannah',
'giraffes have long necks',
'hippos are big and scary',
'rhinos have horns',
'penguins live in the arctic',
'polar bears are white'
])

# Tokenize the training data
tokenized_training_data = Tokenizer.tokenize(training_data)


---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-12-0ae517e0b841> in <cell line: 2>()
1 # Tokenize the training data
----> 2 tokenized_training_data = Tokenizer.tokenize(training_data)

TypeError: Tokenizer.tokenize() missing 1 required positional argument: 'text'