ord('牛') #29275
chr(29275) #'牛'
chr(0) #'\x00'
print(chr(0)) #
"this is a test" + chr(0) + "string" #'this is a test\x00string'
print("this is a test" + chr(0) + "string")#this is a teststring

def run_train_bpe(
    input_path: str | os.PathLike,
    vocab_size: int,
    special_tokens: list[str],
    **kwargs,
) -> tuple[dict[int, bytes], list[tuple[bytes, bytes]]]:
    """Given the path to an input corpus, run train a BPE tokenizer and
    output its vocabulary and merges.

    Args:
        input_path (str | os.PathLike): Path to BPE tokenizer training data.
        vocab_size (int): Total number of items in the tokenizer's vocabulary (including special tokens).
        special_tokens (list[str]): A list of string special tokens to be added to the tokenizer vocabulary.
            These strings will never be split into multiple tokens, and will always be
            kept as a single token. If these special tokens occur in the `input_path`,
            they are treated as any other string.

    Returns:
        tuple[dict[int, bytes], list[tuple[bytes, bytes]]]:
            vocab:
                The trained tokenizer vocabulary, a mapping from int (token ID in the vocabulary)
                to bytes (token bytes)
            merges:
                BPE merges. Each list item is a tuple of bytes (<token1>, <token2>),
                representing that <token1> was merged with <token2>.
                Merges are ordered by order of creation.
    """
    # GPT-2 pretokenization pattern
    GPT2_PATTERN = regex.compile(
        r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
    )

    def pretokenize(text: str) -> list[str]:
        """Split text into tokens using GPT-2 regex pattern."""
        return GPT2_PATTERN.findall(text)

    def split_on_special_tokens(text: str, special_tokens: list[str]) -> list[str]:
        """Split text on special token boundaries."""
        # Build regex pattern to match any special token
        pattern = "|".join(map(regex.escape, special_tokens))
        # Split and keep delimiters
        parts = regex.split(f"({pattern})", text)
        return [p for p in parts if p]

    # Step 1: Initialize vocabulary with 256 byte values
    vocab: dict[int, bytes] = {i: bytes([i]) for i in range(256)}

    # Add special tokens to vocab
    special_token_set = set(special_tokens)
    for tok in special_tokens:
        vocab[len(vocab)] = tok.encode("utf-8")

    # Step 2: Read and preprocess the training text
    with open(input_path, "r", encoding="utf-8") as f:
        text = f.read()

    # Split on special tokens and pretokenize each part
    parts = split_on_special_tokens(text, special_tokens)
    tokens_list: list[list[bytes]] = []

    for part in parts:
        if part in special_token_set:
            # Special token: keep as single token
            tokens_list.append([part.encode("utf-8")])
        elif part:  # Non-empty non-special text
            # Regular text: pretokenize and convert to bytes
            for word in pretokenize(part):
                # Convert each character to its byte representation
                tokens_list.append([bytes([b]) for b in word.encode("utf-8")])

    merges: list[tuple[bytes, bytes]] = []

    # Step 3: BPE training loop
    while len(vocab) < vocab_size:
        # Count pair frequencies
        pair_counts: Counter[tuple[bytes, bytes]] = Counter()
        for tokens in tokens_list:
            for i in range(len(tokens) - 1):
                pair = (tokens[i], tokens[i + 1])
                pair_counts[pair] += 1

        if not pair_counts:
            break

        # Select highest frequency pair (with lexicographically larger as tiebreaker)
        best_pair = max(pair_counts.items(), key=lambda x: (x[1], x[0]))[0]

        # Create new token by merging
        new_token = best_pair[0] + best_pair[1]
        vocab[len(vocab)] = new_token
        merges.append(best_pair)

        # Update tokens by merging all occurrences of best_pair
        for tokens in tokens_list:
            i = 0
            while i < len(tokens) - 1:
                if (tokens[i], tokens[i + 1]) == best_pair:
                    # Merge the pair
                    tokens[i : i + 2] = [new_token]
                    # Don't increment i - check if the new token can merge again
                else:
                    i += 1

    return vocab, merges

GPT2_PATTERN = regex.compile(
    r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
)

def split_on_special_tokens(text: str, special_tokens: list[str]) -> list[str]:
    pattern = "|".join(map(regex.escape, special_tokens))
    parts = regex.split(f"({pattern})", text)
    return [p for p in parts if p]

while len(vocab) < vocab_size:
    # Count all adjacent pairs across all tokens
    pair_counts: Counter[tuple[bytes, bytes]] = Counter()
    for tokens in tokens_list:
        for i in range(len(tokens) - 1):
            pair = (tokens[i], tokens[i + 1])
            pair_counts[pair] += 1

    if not pair_counts:
        break  # Nothing left to merge

    # Select the best pair: highest frequency, ties broken lexicographically
    best_pair = max(pair_counts.items(), key=lambda x: (x[1], x[0]))[0]

    # Create new merged token
    new_token = best_pair[0] + best_pair[1]
    vocab[len(vocab)] = new_token
    merges.append(best_pair)

    # Update all token sequences with the new merge
    for tokens in tokens_list:
        i = 0
        while i < len(tokens) - 1:
            if (tokens[i], tokens[i + 1]) == best_pair:
                tokens[i : i + 2] = [new_token]
                # Don't increment i — the new token might merge again
            else:
                i += 1