Tokens

import tiktoken

# GPT-4 tokenizer (cl100k_base, ~100k vocabulary)
enc = tiktoken.get_encoding("cl100k_base")

def tokenize(text: str):
    ids     = enc.encode(text)
    tokens  = [enc.decode([i]) for i in ids]
    return ids, tokens

# Example 1: Common English words
ids, tokens = tokenize("The quick brown fox")
print(f"Tokens: {tokens}")   # ['The', ' quick', ' brown', ' fox']
print(f"IDs:    {ids}")      # [791, 4062, 14198, 39935]
print(f"Count:  {len(ids)}")  # 4

# Example 2: Rare/technical words split into subwords
ids, tokens = tokenize("tokenization")
print(f"Tokens: {tokens}")   # ['token', 'ization'] → 2 tokens for 1 word!

# Example 3: Unicode (non-English is more expensive)
ids, tokens = tokenize("नमस्ते")   # Hindi for "hello"
print(f"Tokens: {tokens}")   # many more tokens than the English equivalent

# Example 4: Code
ids, tokens = tokenize("def hello_world():")
print(f"Tokens: {tokens}")   # ['def', ' hello', '_world', '():']

# Rule of thumb: 1 token ≈ 4 chars in English
test = "Hello, how are you doing today?"
print(f"Chars: {len(test)}, Tokens: {len(enc.encode(test))}")
# Chars: 31, Tokens: 8

from collections import Counter

def get_stats(vocab):
    """Count all adjacent symbol pairs across the vocabulary."""
    pairs = Counter()
    for word, freq in vocab.items():
        symbols = word.split()
        for i in range(len(symbols) - 1):
            pairs[(symbols[i], symbols[i+1])] += freq
    return pairs

def merge_vocab(pair, vocab):
    """Merge the most frequent pair in all vocabulary entries."""
    bigram = ' '.join(pair)
    replacement = ''.join(pair)
    return {
        word.replace(bigram, replacement): freq
        for word, freq in vocab.items()
    }

# Initial vocabulary: words split into characters + </w> end marker
vocab = {
    'l o w </w>': 5,
    'l o w e r </w>': 2,
    'n e w e s t </w>': 6,
    'w i d e s t </w>': 3,
}

print("Initial vocab:", vocab)

# Run BPE for 10 merge operations
merges = []
for step in range(10):
    pairs = get_stats(vocab)
    if not pairs:
        break
    best = max(pairs, key=pairs.get)
    vocab = merge_vocab(best, vocab)
    merges.append(best)
    print(f"Step {step+1}: Merge {best} → {''.join(best)}")

# Step 1: Merge ('e', 's') → es
# Step 2: Merge ('es', 't') → est
# Step 3: Merge ('est', '</w>') → est</w>
# Step 4: Merge ('l', 'o') → lo
# ...
# Eventually: 'newest' becomes a single token!

import tiktoken

def count_tokens(text: str, model: str = "gpt-4o") -> int:
    """Count tokens before sending to API to estimate cost."""
    enc = tiktoken.encoding_for_model(model)
    return len(enc.encode(text))

def estimate_cost(input_text: str, output_tokens: int = 500):
    """Estimate API cost in USD (GPT-4o pricing, 2025)."""
    input_tokens  = count_tokens(input_text)
    input_cost    = input_tokens  * 2.50 / 1_000_000   # $2.50/M input tokens
    output_cost   = output_tokens * 10.0 / 1_000_000   # $10.0/M output tokens
    total = input_cost + output_cost

    print(f"Input tokens:  {input_tokens:,}")
    print(f"Output tokens: {output_tokens:,} (estimated)")
    print(f"Input cost:    ${input_cost:.4f}")
    print(f"Output cost:   ${output_cost:.4f}")
    print(f"Total:         ${total:.4f}")
    return total

document = "This is a long research paper..." * 100  # simulated
estimate_cost(document)