Update app.py

app.py

@@ -13,6 +13,7 @@ def tokenize(input_text):
     phi3_tokens = len(phi3_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     gemma_tokens = len(gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gemma2_tokens = len(gemma2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     command_r_tokens = len(command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     qwen_tokens = len(qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     codeqwen_tokens = len(codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
@@ -33,6 +34,7 @@ def tokenize(input_text):
         "Phi-3": phi3_tokens,
         "T5": t5_tokens,
         "Gemma": gemma_tokens,
+        "Gemma-2": gemma2_tokens,
         "Command-R": command_r_tokens,
         "Qwen/Qwen1.5": qwen_tokens,
         "CodeQwen": codeqwen_tokens,
@@ -60,6 +62,7 @@ if __name__ == "__main__":
     phi3_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
     t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
     gemma_tokenizer = AutoTokenizer.from_pretrained("alpindale/gemma-2b")
+    gemma2_tokenizer = AutoTokenizer.from_pretrained("UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3")
     command_r_tokenizer = AutoTokenizer.from_pretrained("PJMixers/CohereForAI_c4ai-command-r-plus-tokenizer")
     qwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-7B")
     codeqwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/CodeQwen1.5-7B")
@@ -69,5 +72,5 @@ if __name__ == "__main__":
     internlm_tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-20b", trust_remote_code=True)
     internlm2_tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2-20b", trust_remote_code=True)
 
-    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=19), outputs="text")
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=20), outputs="text")
     iface.launch()