Tokenizing Text¶

In [1]:

import urllib.request

import urllib.request

In [2]:

url = ("https://raw.githubusercontent.com/rasbt/"
       "LLMs-from-scratch/main/ch02/01_main-chapter-code/"
       "the-verdict.txt")
file_path = "the-verdict.txt"
urllib.request.urlretrieve(url, file_path)

url = ("https://raw.githubusercontent.com/rasbt/" "LLMs-from-scratch/main/ch02/01_main-chapter-code/" "the-verdict.txt") file_path = "the-verdict.txt" urllib.request.urlretrieve(url, file_path)

Out[2]:

('the-verdict.txt', <http.client.HTTPMessage at 0x11220ed50>)

In [3]:

with open("the-verdict.txt", "r", encoding="utf-8") as f:
    raw_text = f.read()
print("Total number of character:", len(raw_text))
print(raw_text[:99])

with open("the-verdict.txt", "r", encoding="utf-8") as f: raw_text = f.read() print("Total number of character:", len(raw_text)) print(raw_text[:99])

Total number of character: 20479
I HAD always thought Jack Gisburn rather a cheap genius--though a good fellow enough--so it was no 

this is example tokenizer scheme, removing whitespace and split punctuation characters

In [4]:

import re

text = "Hello, world. Is this-- a test?"
result = re.split(r'([,.:;?_!"()\']|--|\s)', text)
result = [item.strip() for item in result if item.strip()]
print(result)

import re text = "Hello, world. Is this-- a test?" result = re.split(r'([,.:;?_!"()\']|--|\s)', text) result = [item.strip() for item in result if item.strip()] print(result)

['Hello', ',', 'world', '.', 'Is', 'this', '--', 'a', 'test', '?']

In [5]:

preprocessed = re.split(r'([,.:;?_!"()\']|--|\s)', raw_text)
preprocessed = [item.strip() for item in preprocessed if item.strip()]
print(len(preprocessed))
print(preprocessed[:30])

preprocessed = re.split(r'([,.:;?_!"()\']|--|\s)', raw_text) preprocessed = [item.strip() for item in preprocessed if item.strip()] print(len(preprocessed)) print(preprocessed[:30])

4690
['I', 'HAD', 'always', 'thought', 'Jack', 'Gisburn', 'rather', 'a', 'cheap', 'genius', '--', 'though', 'a', 'good', 'fellow', 'enough', '--', 'so', 'it', 'was', 'no', 'great', 'surprise', 'to', 'me', 'to', 'hear', 'that', ',', 'in']

Converting tokens into token IDs¶

determine vocabulary size

In [6]:

all_words = sorted(set(preprocessed))
vocab_size = len(all_words)
print(vocab_size)

all_words = sorted(set(preprocessed)) vocab_size = len(all_words) print(vocab_size)

1130

assign each token id

In [7]:

vocab = {token:integer for integer,token in enumerate(all_words)}
for i, item in enumerate(vocab.items()):
    print(item)
    if i >= 50:
        break

vocab = {token:integer for integer,token in enumerate(all_words)} for i, item in enumerate(vocab.items()): print(item) if i >= 50: break

('!', 0)
('"', 1)
("'", 2)
('(', 3)
(')', 4)
(',', 5)
('--', 6)
('.', 7)
(':', 8)
(';', 9)
('?', 10)
('A', 11)
('Ah', 12)
('Among', 13)
('And', 14)
('Are', 15)
('Arrt', 16)
('As', 17)
('At', 18)
('Be', 19)
('Begin', 20)
('Burlington', 21)
('But', 22)
('By', 23)
('Carlo', 24)
('Chicago', 25)
('Claude', 26)
('Come', 27)
('Croft', 28)
('Destroyed', 29)
('Devonshire', 30)
('Don', 31)
('Dubarry', 32)
('Emperors', 33)
('Florence', 34)
('For', 35)
('Gallery', 36)
('Gideon', 37)
('Gisburn', 38)
('Gisburns', 39)
('Grafton', 40)
('Greek', 41)
('Grindle', 42)
('Grindles', 43)
('HAD', 44)
('Had', 45)
('Hang', 46)
('Has', 47)
('He', 48)
('Her', 49)
('Hermia', 50)

In [8]:

class SimpleTokenizerV1:
    def __init__(self, vocab):
        self.str_to_int = vocab
        self.int_to_str = {i:s for s,i in vocab.items()}

    def encode(self, text):
        preprocessed = re.split(r'([,.?_!"()\']|--|\s)', text)
        preprocessed = [
            item.strip() for item in preprocessed if item.strip()
        ]
        ids = [self.str_to_int[s] for s in preprocessed]
        return ids

    def decode(self, ids):
        text = " ".join([self.int_to_str[i] for i in ids]) 

        text = re.sub(r'\s+([,.?!"()\'])', r'\1', text)
        return text

class SimpleTokenizerV1: def __init__(self, vocab): self.str_to_int = vocab self.int_to_str = {i:s for s,i in vocab.items()} def encode(self, text): preprocessed = re.split(r'([,.?_!"()\']|--|\s)', text) preprocessed = [ item.strip() for item in preprocessed if item.strip() ] ids = [self.str_to_int[s] for s in preprocessed] return ids def decode(self, ids): text = " ".join([self.int_to_str[i] for i in ids]) text = re.sub(r'\s+([,.?!"()\'])', r'\1', text) return text

In [9]:

tokenizer = SimpleTokenizerV1(vocab)
text = """"It's the last he painted, you know," 
       Mrs. Gisburn said with pardonable pride."""
ids = tokenizer.encode(text)
print(ids)

print(tokenizer.decode(ids))

tokenizer = SimpleTokenizerV1(vocab) text = """"It's the last he painted, you know," Mrs. Gisburn said with pardonable pride.""" ids = tokenizer.encode(text) print(ids) print(tokenizer.decode(ids))

[1, 56, 2, 850, 988, 602, 533, 746, 5, 1126, 596, 5, 1, 67, 7, 38, 851, 1108, 754, 793, 7]
" It' s the last he painted, you know," Mrs. Gisburn said with pardonable pride.

In [10]:

tokenizer = SimpleTokenizerV1(vocab)
text = "Hello, do you like tea?"
print(tokenizer.encode(text))

tokenizer = SimpleTokenizerV1(vocab) text = "Hello, do you like tea?" print(tokenizer.encode(text))

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[10], line 3
      1 tokenizer = SimpleTokenizerV1(vocab)
      2 text = "Hello, do you like tea?"
----> 3 print(tokenizer.encode(text))

Cell In[8], line 11, in SimpleTokenizerV1.encode(self, text)
      7         preprocessed = re.split(r'([,.?_!"()\']|--|\s)', text)
      8         preprocessed = [
      9             item.strip() for item in preprocessed if item.strip()
     10         ]
---> 11         ids = [self.str_to_int[s] for s in preprocessed]
     12         return ids

KeyError: 'Hello'

Adding special context tokens¶

creating special token for several use cases. to identify unknown word from vocab or defining end of text source (giving token between unrelated text)

In [11]:

all_tokens = sorted(list(set(preprocessed)))
all_tokens.extend(["<|endoftext|>", "<|unk|>"])
vocab = {token:integer for integer,token in enumerate(all_tokens)}

print(len(vocab.items()))

all_tokens = sorted(list(set(preprocessed))) all_tokens.extend(["<|endoftext|>", "<|unk|>"]) vocab = {token:integer for integer,token in enumerate(all_tokens)} print(len(vocab.items()))

1132

In [12]:

for i, item in enumerate(list(vocab.items())[-5:]):
    print(item)

for i, item in enumerate(list(vocab.items())[-5:]): print(item)

('younger', 1127)
('your', 1128)
('yourself', 1129)
('<|endoftext|>', 1130)
('<|unk|>', 1131)

update the tokenizer to support this 2 context tokens

In [13]:

class SimpleTokenizerV2:
    def __init__(self, vocab):
        self.str_to_int = vocab
        self.int_to_str = { i:s for s,i in vocab.items()}

    def encode(self, text):
        preprocessed = re.split(r'([,.:;?_!"()\']|--|\s)', text)
        preprocessed = [
            item.strip() for item in preprocessed if item.strip()
        ]
        preprocessed = [item if item in self.str_to_int
                        else "<|unk|>" for item in preprocessed]

        ids = [self.str_to_int[s] for s in preprocessed]
        return ids

    def decode(self, ids):
        text = " ".join([self.int_to_str[i] for i in ids])

        text = re.sub(r'\s+([,.:;?!"()\'])', r'\1', text)
        return text

class SimpleTokenizerV2: def __init__(self, vocab): self.str_to_int = vocab self.int_to_str = { i:s for s,i in vocab.items()} def encode(self, text): preprocessed = re.split(r'([,.:;?_!"()\']|--|\s)', text) preprocessed = [ item.strip() for item in preprocessed if item.strip() ] preprocessed = [item if item in self.str_to_int else "<|unk|>" for item in preprocessed] ids = [self.str_to_int[s] for s in preprocessed] return ids def decode(self, ids): text = " ".join([self.int_to_str[i] for i in ids]) text = re.sub(r'\s+([,.:;?!"()\'])', r'\1', text) return text

In [14]:

text1 = "Hello, do you like tea?"
text2 = "In the sunlit terraces of the palace."
text = " <|endoftext|> ".join((text1, text2))
print(text)

text1 = "Hello, do you like tea?" text2 = "In the sunlit terraces of the palace." text = " <|endoftext|> ".join((text1, text2)) print(text)

Hello, do you like tea? <|endoftext|> In the sunlit terraces of the palace.

In [15]:

tokenizer = SimpleTokenizerV2(vocab)
print(tokenizer.encode(text))

tokenizer = SimpleTokenizerV2(vocab) print(tokenizer.encode(text))

[1131, 5, 355, 1126, 628, 975, 10, 1130, 55, 988, 956, 984, 722, 988, 1131, 7]

In [16]:

print(tokenizer.decode(tokenizer.encode(text)))

print(tokenizer.decode(tokenizer.encode(text)))

<|unk|>, do you like tea? <|endoftext|> In the sunlit terraces of the <|unk|>.

Depending on the LLM, some researchers also consider additional special tokens such as the following:

[BOS] (beginning of sequence) —This token marks the start of a text. It signifies to the LLM where a piece of content begins. [EOS] (end of sequence) —This token is positioned at the end of a text and is especially useful when concatenating multiple unrelated texts, similar to <|endoftext|>. For instance, when combining two different Wikipedia articles or books, the [EOS] token indicates where one ends and the next begins. [PAD] (padding) —When training LLMs with batch sizes larger than one, the batch might contain texts of varying lengths. To ensure all texts have the same length, the shorter texts are extended or “padded” using the [PAD] token, up to the length of the longest text in the batch.

Byte pair encoding¶

In [17]:

from importlib.metadata import version
import tiktoken
print("tiktoken version:", version("tiktoken"))

from importlib.metadata import version import tiktoken print("tiktoken version:", version("tiktoken"))

tiktoken version: 0.13.0

In [18]:

tokenizer = tiktoken.get_encoding("gpt2")

tokenizer = tiktoken.get_encoding("gpt2")

In [19]:

text = (
    "Hello, do you like tea? <|endoftext|> In the sunlit terraces"
     "of someunknownPlace."
)
integers = tokenizer.encode(text, allowed_special={"<|endoftext|>"})
print(integers)

text = ( "Hello, do you like tea? <|endoftext|> In the sunlit terraces" "of someunknownPlace." ) integers = tokenizer.encode(text, allowed_special={"<|endoftext|>"}) print(integers)

[15496, 11, 466, 345, 588, 8887, 30, 220, 50256, 554, 262, 4252, 18250, 8812, 2114, 1659, 617, 34680, 27271, 13]

In [20]:

strings = tokenizer.decode(integers)
print(strings)

strings = tokenizer.decode(integers) print(strings)

Hello, do you like tea? <|endoftext|> In the sunlit terracesof someunknownPlace.

In [21]:

text = (
    "Akwirw ier"
)
integers = tokenizer.encode(text, allowed_special={"<|endoftext|>"})
print(integers)

text = ( "Akwirw ier" ) integers = tokenizer.encode(text, allowed_special={"<|endoftext|>"}) print(integers)

[33901, 86, 343, 86, 220, 959]

In [22]:

strings = tokenizer.decode(integers)
print(strings)

strings = tokenizer.decode(integers) print(strings)

Akwirw ier

Data sampling with a sliding window¶

In [23]:

with open("the-verdict.txt", "r", encoding="utf-8") as f:
    raw_text = f.read()

enc_text = tokenizer.encode(raw_text)
print(len(enc_text))

with open("the-verdict.txt", "r", encoding="utf-8") as f: raw_text = f.read() enc_text = tokenizer.encode(raw_text) print(len(enc_text))

5145

next word prediction

In [24]:

enc_sample = enc_text[50:]

enc_sample = enc_text[50:]

In [25]:

context_size = 4
x = enc_sample[:context_size]
y = enc_sample[1:context_size+1]
print(f"x: {x}")
print(f"y:      {y}")

context_size = 4 x = enc_sample[:context_size] y = enc_sample[1:context_size+1] print(f"x: {x}") print(f"y: {y}")

x: [290, 4920, 2241, 287]
y:      [4920, 2241, 287, 257]

In [26]:

for i in range(1, context_size+1):
    context = enc_sample[:i]
    desired = enc_sample[i]
    print(context, "---->", desired)

for i in range(1, context_size+1): context = enc_sample[:i] desired = enc_sample[i] print(context, "---->", desired)

[290] ----> 4920
[290, 4920] ----> 2241
[290, 4920, 2241] ----> 287
[290, 4920, 2241, 287] ----> 257

In [27]:

for i in range(1, context_size+1):
    context = enc_sample[:i]
    desired = enc_sample[i]
    print(tokenizer.decode(context), "---->", tokenizer.decode([desired]))

for i in range(1, context_size+1): context = enc_sample[:i] desired = enc_sample[i] print(tokenizer.decode(context), "---->", tokenizer.decode([desired]))

 and ---->  established
 and established ---->  himself
 and established himself ---->  in
 and established himself in ---->  a

A data loader is the component that feeds training examples to the model in manageable batches.

inputs: token-ID sequences the LLM reads targets: the same sequences shifted one token to the left—the next tokens the LLM should predict For example, with a context length of 4:

inputs = [40, 367, 2885, 1464] targets = [367, 2885, 1464, 1807]

input: and established himself in target: established himself in a

During training, the data loader repeatedly supplies batches of:

input text/tokens the model can see target tokens that show the correct next token at each position The model tries to predict the targets from the inputs, and training adjusts its weights when predictions are wrong.

tokenize all and convert into input and target

In [28]:

import torch
from torch.utils.data import Dataset, DataLoader
class GPTDatasetV1(Dataset):
    def __init__(self, txt, tokenizer, max_length, stride):
        self.input_ids = []
        self.target_ids = []

        token_ids = tokenizer.encode(txt)

        for i in range(0, len(token_ids) - max_length, stride):
            input_chunk = token_ids[i:i + max_length]
            target_chunk = token_ids[i + 1: i + max_length + 1]
            self.input_ids.append(torch.tensor(input_chunk))
            self.target_ids.append(torch.tensor(target_chunk))

    def __len__(self):
        return len(self.input_ids)

    def __getitem__(self, idx):
        return self.input_ids[idx], self.target_ids[idx]

import torch from torch.utils.data import Dataset, DataLoader class GPTDatasetV1(Dataset): def __init__(self, txt, tokenizer, max_length, stride): self.input_ids = [] self.target_ids = [] token_ids = tokenizer.encode(txt) for i in range(0, len(token_ids) - max_length, stride): input_chunk = token_ids[i:i + max_length] target_chunk = token_ids[i + 1: i + max_length + 1] self.input_ids.append(torch.tensor(input_chunk)) self.target_ids.append(torch.tensor(target_chunk)) def __len__(self): return len(self.input_ids) def __getitem__(self, idx): return self.input_ids[idx], self.target_ids[idx]

In [29]:

import torch
from torch.utils.data import DataLoader
def create_dataloader_v1(txt, batch_size=4, max_length=256,
                         stride=128, shuffle=True, drop_last=True,
                         num_workers=0):
    tokenizer = tiktoken.get_encoding("gpt2")
    dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)
    dataloader = DataLoader(
        dataset,
        batch_size=batch_size,
        shuffle=shuffle,
        drop_last=drop_last,
        num_workers=num_workers
    )

    return dataloader

import torch from torch.utils.data import DataLoader def create_dataloader_v1(txt, batch_size=4, max_length=256, stride=128, shuffle=True, drop_last=True, num_workers=0): tokenizer = tiktoken.get_encoding("gpt2") dataset = GPTDatasetV1(txt, tokenizer, max_length, stride) dataloader = DataLoader( dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=num_workers ) return dataloader

In [45]:

with open("the-verdict.txt", "r", encoding="utf-8") as f:
    raw_text = f.read()

dataloader = create_dataloader_v1(
    raw_text, batch_size=1, max_length=4, stride=1, shuffle=False)
data_iter = iter(dataloader)
first_batch = next(data_iter)
print(first_batch)

with open("the-verdict.txt", "r", encoding="utf-8") as f: raw_text = f.read() dataloader = create_dataloader_v1( raw_text, batch_size=1, max_length=4, stride=1, shuffle=False) data_iter = iter(dataloader) first_batch = next(data_iter) print(first_batch)

[tensor([[  40,  367, 2885, 1464]]), tensor([[ 367, 2885, 1464, 1807]])]

In [46]:

second_batch = next(data_iter)
print(second_batch)

second_batch = next(data_iter) print(second_batch)

[tensor([[ 367, 2885, 1464, 1807]]), tensor([[2885, 1464, 1807, 3619]])]

In [47]:

dataloader = create_dataloader_v1(
    raw_text, batch_size=8, max_length=4, stride=4,
    shuffle=False
)

data_iter = iter(dataloader)
inputs, targets = next(data_iter)
print("Inputs:\n", inputs)
print("\nTargets:\n", targets)

dataloader = create_dataloader_v1( raw_text, batch_size=8, max_length=4, stride=4, shuffle=False ) data_iter = iter(dataloader) inputs, targets = next(data_iter) print("Inputs:\n", inputs) print("\nTargets:\n", targets)

Inputs:
 tensor([[   40,   367,  2885,  1464],
        [ 1807,  3619,   402,   271],
        [10899,  2138,   257,  7026],
        [15632,   438,  2016,   257],
        [  922,  5891,  1576,   438],
        [  568,   340,   373,   645],
        [ 1049,  5975,   284,   502],
        [  284,  3285,   326,    11]])

Targets:
 tensor([[  367,  2885,  1464,  1807],
        [ 3619,   402,   271, 10899],
        [ 2138,   257,  7026, 15632],
        [  438,  2016,   257,   922],
        [ 5891,  1576,   438,   568],
        [  340,   373,   645,  1049],
        [ 5975,   284,   502,   284],
        [ 3285,   326,    11,   287]])

Creating token embeddings¶

In [ ]: