Using BPE Tokenizer

README.md

@@ -1,3 +1,19 @@
----
-license: mit
----
+# TinyStack Tokenizer
+
+ByteLevel BPE tokenizer trained on fhswf/tiny-stack dataset.
+
+## Usage
+
+```python
+from tokenizers.implementations import ByteLevelBPETokenizer
+from tokenizers.processors import BertProcessing
+
+tokenizer = ByteLevelBPETokenizer("./vocab.json", "./merges.txt")
+tokenizer._tokenizer.post_processor = BertProcessing(
+    ("</s>", tokenizer.token_to_id("</s>")),
+    ("<s>", tokenizer.token_to_id("<s>")),
+)
+tokenizer.enable_truncation(max_length=512)
+```
+
+Vocab size: 52000

config.json

@@ -0,0 +1,17 @@
+{
+  "vocab_size": 52000,
+  "min_frequency": 2,
+  "special_tokens": [
+    "<s>",
+    "<pad>",
+    "</s>",
+    "<unk>",
+    "<mask>",
+    "<code>",
+    "</code>",
+    "<error_message>",
+    "</error_message>",
+    "<description>",
+    "</description>"
+  ]
+}

special_tokens_map.json

@@ -1,6 +0,0 @@
-{
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "<pad>",
-  "unk_token": "<unk>"
-}

tokenizer_config.json

@@ -1,20 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "unk_token": "<unk>",
-  "pad_token": "<pad>",
-  "mask_token": "<mask>",
-  "additional_special_tokens": [
-    "<code>",
-    "</code>",
-    "<error_message>",
-    "</error_message>",
-    "<description>",
-    "</description>"
-  ],
-  "clean_up_tokenization_spaces": true,
-  "model_max_length": 2048,
-  "tokenizer_class": "GPT2Tokenizer"
-}