Upload tokenizer

tokenizer_config.json

@@ -9,6 +9,12 @@
       "special": true
     }
   },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer_utils.CryptGPTTokenizer",
+      null
+    ]
+  },
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "max_length": 1024,

tokenizer_utils.py

@@ -0,0 +1,6 @@
+from transformers import PreTrainedTokenizerFast
+
+class CryptGPTTokenizer(PreTrainedTokenizerFast):
+    @staticmethod
+    def clean_up_tokenization(out_string):
+        return out_string.replace(' ', "")