m3b

app.py

@@ -6,6 +6,24 @@ import torch
 
 from conversation import get_default_conv_template
 import gradio as gr
+from pyllamacpp.model import Model
+from urllib import request, response, urlopen, error, parse, robotparser
+
+"""
+
+model = Model(model_path='/path/to/model.bin')
+while True:
+    try:
+        prompt = input("You: ", flush=True)
+        if prompt == '':
+            continue
+        print(f"AI:", end='')
+        for token in model.generate(prompt):
+            print(f"{token}", end='', flush=True)
+        print()
+    except KeyboardInterrupt:
+        break
+"""
 
 talkers = {
     "m3b": {
@@ -15,20 +33,15 @@ talkers = {
     }
 }
 
+request.urlretrieve("https://huggingface.co/GGUF/MiniChat-3B/resolve/main/ggml-model-q8_0.bin", "minichat-3b-q8_0.gguf")
+
+lcpp_model = Model(model_path="minichat-3b-q8_0.gguf")
+
 def m3b_talk(text):
-    m3bconv = talkers["m3b"]["conv"]
-    m3bconv.append_message(m3bconv.roles[0], text)
-    m3bconv.append_message(m3bconv.roles[1], None)
-    input_ids = talkers["m3b"]["tokenizer"]([text]).input_ids
-    response_tokens = talkers["m3b"]["model"](
-        torch.as_tensor(m3bconv.get_prompt()),
-        do_sample=True,
-        temperature=0.2,
-        max_new_tokens=1024,
-    )
-    response_tokens = response_tokens[0][len(input_ids[0]):]
-    response = talkers["m3b"]["tokenizer"].decode(response_tokens, skip_special_tokens=True).strip()
-    return response
+    resp = ""
+    for token in lcpp_model.generate(text):
+        resp += token
+    return resp
 
 def main():
     logging.basicConfig(level=logging.INFO)