Use llama-cpp-python

app.py

@@ -6,7 +6,7 @@ import torch
 
 from conversation import get_default_conv_template
 import gradio as gr
-from pyllamacpp.model import Model
+from llama_cpp import Llama
 import wget
 
 """
@@ -31,11 +31,11 @@ model_path = "minichat-3b.q8_0.gguf"
 
 mdlpath = hf_hub_download(repo_id="afrideva/MiniChat-3B-GGUF", filename=model_path)
 
-lcpp_model = Model(model_path=mdlpath)
+lcpp_model = Llama(model_path=mdlpath)
 
 def m3b_talk(text):
     resp = ""
-    for token in lcpp_model.generate(text):
+    for token in lcpp_model("<s> [|User|]" + text + "</s> [|Assistant|]", stop=["[|User|]", "\n"], echo=True):
         resp += token
     return resp
 

requirements.txt

@@ -8,5 +8,5 @@ git+https://github.com/huggingface/transformers
 torch
 sentencepiece
 accelerate
-pyllamacpp
+llama-cpp-python>=0.2.18
 wget