Upload

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.so filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Minigpt4 Ggml
+emoji: 🌍
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.36.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,107 @@
+import os
+import sys
+import ctypes
+import pathlib
+from typing import Optional, List
+import enum
+from pathlib import Path
+import argparse
+import gradio as gr
+
+import minigpt4_library
+
+from huggingface_hub import hf_hub_download
+
+model_path = hf_hub_download(repo_id='maknee/minigpt4-13b-ggml', filename='minigpt4-13B-f16.bin')
+llm_model_path = hf_hub_download(repo_id='maknee/ggml-vicuna-v0-quantized', filename='ggml-vicuna-13B-v0-q5_k.bin')
+
+title = """<h1 align="center">Demo of MiniGPT-4</h1>"""
+description = """<h3>This is the demo of MiniGPT-4 with ggml (cpu only!). Upload your images and start chatting!</h3>"""
+article = """<div style='display:flex; gap: 0.25rem; '><a href='https://github.com/Vision-CAIR/MiniGPT-4'><img src='https://img.shields.io/badge/Github-Code-blue'></a></div>"""
+
+global minigpt4_chatbot
+minigpt4_chatbot: minigpt4_library.MiniGPT4ChatBot
+
+def user(message, history):
+    history = history or []
+    # Append the user's message to the conversation history
+    history.append([message, ""])
+    return "", history
+
+def chat(history, limit: int = 1024, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, repeat_penalty: float = 1.1):
+    history = history or []
+
+    message = history[-1][0]
+
+    history[-1][1] = ""
+    for output in minigpt4_chatbot.generate(
+        message,
+        limit = int(limit),
+        temp = float(temp),
+        top_k = int(top_k),
+        top_p = float(top_p),
+    ):
+        answer = output
+        history[-1][1] += answer
+        # stream the response
+        yield history, history
+
+def clear_state(history, chat_message, image):
+    history = []
+    minigpt4_chatbot.reset_chat()
+    return history, gr.update(value=None, interactive=True), gr.update(placeholder='Upload image first', interactive=False), gr.update(value="Upload & Start Chat", interactive=True)
+
+def upload_image(image, history):
+    if image is None:
+        return None, None, gr.update(interactive=True), history
+    history = []
+    minigpt4_chatbot.upload_image(image)
+    return gr.update(interactive=False), gr.update(interactive=True, placeholder='Type and press Enter'), gr.update(value="Start Chatting", interactive=False), history
+
+def start():
+    with gr.Blocks() as demo:
+        gr.Markdown(title)
+        gr.Markdown(description)
+        gr.Markdown(article)
+
+        with gr.Row():
+            with gr.Column(scale=0.5):
+                image = gr.Image(type="pil")
+                upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary")
+
+                max_tokens = gr.Slider(1, 1024, label="Max Tokens", step=1, value=128)
+                temperature = gr.Slider(0.0, 1.0, label="Temperature", step=0.05, value=0.8)
+                top_p = gr.Slider(0.0, 1.0, label="Top P", step=0.05, value=0.95)
+                top_k = gr.Slider(0, 100, label="Top K", step=1, value=40)
+                repeat_penalty = gr.Slider(0.0, 2.0, label="Repetition Penalty", step=0.1, value=1.1)
+
+            with gr.Column():
+                chatbot = gr.Chatbot(label='MiniGPT-4')
+                message = gr.Textbox(label='User', placeholder='Upload image first', interactive=False)
+                history = gr.State()
+
+                with gr.Row():
+                    submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)
+                    clear = gr.Button(value="Reset", variant="secondary").style(full_width=False)
+                    # stop = gr.Button(value="Stop", variant="secondary").style(full_width=False)
+
+        clear.click(clear_state, inputs=[history, image, message], outputs=[history, image, message, upload_button], queue=False)
+
+        upload_button.click(upload_image, inputs=[image, history], outputs=[image, message, upload_button, history])
+        
+        submit_click_event = submit.click(
+            fn=user, inputs=[message, history], outputs=[message, history], queue=True
+        ).then(
+            fn=chat, inputs=[history, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, history], queue=True
+        )
+        message_submit_event = message.submit(
+            fn=user, inputs=[message, history], outputs=[message, history], queue=True
+        ).then(
+            fn=chat, inputs=[history, max_tokens, temperature, top_p, top_k, repeat_penalty], outputs=[chatbot, history], queue=True
+        )
+        # stop.click(fn=None, inputs=None, outputs=None, cancels=[submit_click_event, message_submit_event], queue=False)
+
+    demo.launch(enable_queue=True)
+
+minigpt4_chatbot = minigpt4_library.MiniGPT4ChatBot(model_path, llm_model_path, verbosity=minigpt4_library.Verbosity.SILENT)
+start()

libminigpt4.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be54434ed2aa0f41c69bab0531b90e6a9ecb18c805bb8082307bb5e5aa1658d4
+size 1227064

minigpt4_library.py

@@ -0,0 +1,741 @@
+import os
+import sys
+import ctypes
+import pathlib
+from typing import Optional, List
+import enum
+from pathlib import Path
+
+class DataType(enum.IntEnum):
+    def __str__(self):
+        return str(self.name)
+    
+    F16 = 0
+    F32 = 1
+    I32 = 2
+    L64 = 3
+    Q4_0 = 4
+    Q4_1 = 5
+    Q5_0 = 6
+    Q5_1 = 7
+    Q8_0 = 8
+    Q8_1 = 9
+    Q2_K = 10
+    Q3_K = 11
+    Q4_K = 12
+    Q5_K = 13
+    Q6_K = 14
+    Q8_K = 15
+
+class Verbosity(enum.IntEnum):
+    SILENT = 0
+    ERR = 1
+    INFO = 2
+    DEBUG = 3
+
+class ImageFormat(enum.IntEnum):
+    UNKNOWN = 0
+    F32 = 1
+    U8 = 2
+
+I32 = ctypes.c_int32
+U32 = ctypes.c_uint32
+F32 = ctypes.c_float
+SIZE_T = ctypes.c_size_t
+VOID_PTR = ctypes.c_void_p
+CHAR_PTR = ctypes.POINTER(ctypes.c_char)
+FLOAT_PTR = ctypes.POINTER(ctypes.c_float)
+INT_PTR = ctypes.POINTER(ctypes.c_int32)
+CHAR_PTR_PTR = ctypes.POINTER(ctypes.POINTER(ctypes.c_char))
+
+MiniGPT4ContextP = VOID_PTR
+class MiniGPT4Context:
+    def __init__(self, ptr: ctypes.pointer):
+        self.ptr = ptr
+
+class MiniGPT4Image(ctypes.Structure):
+    _fields_ = [
+        ('data', VOID_PTR),
+        ('width', I32),
+        ('height', I32),
+        ('channels', I32),
+        ('format', I32)
+    ]
+
+class MiniGPT4Embedding(ctypes.Structure):
+    _fields_ = [
+        ('data', FLOAT_PTR),
+        ('n_embeddings', SIZE_T),
+    ]
+
+MiniGPT4ImageP = ctypes.POINTER(MiniGPT4Image)
+MiniGPT4EmbeddingP = ctypes.POINTER(MiniGPT4Embedding)
+
+class MiniGPT4SharedLibrary:
+    """
+    Python wrapper around minigpt4.cpp shared library.
+    """
+
+    def __init__(self, shared_library_path: str):
+        """
+        Loads the shared library from specified file.
+        In case of any error, this method will throw an exception.
+
+        Parameters
+        ----------
+        shared_library_path : str
+            Path to minigpt4.cpp shared library. On Windows, it would look like 'minigpt4.dll'. On UNIX, 'minigpt4.so'.
+        """
+
+        self.library = ctypes.cdll.LoadLibrary(shared_library_path)
+
+        self.library.minigpt4_model_load.argtypes = [
+            CHAR_PTR, # const char *path
+            CHAR_PTR, # const char *llm_model
+            I32, # int verbosity
+            I32, # int seed
+            I32, # int n_ctx
+            I32, # int n_batch
+            I32, # int numa
+        ]
+        self.library.minigpt4_model_load.restype = MiniGPT4ContextP
+
+        self.library.minigpt4_image_load_from_file.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            CHAR_PTR, # const char *path
+            MiniGPT4ImageP, # struct MiniGPT4Image *image
+            I32, # int flags
+        ]
+        self.library.minigpt4_image_load_from_file.restype = I32
+
+        self.library.minigpt4_encode_image.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            MiniGPT4ImageP, # const struct MiniGPT4Image *image
+            MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding
+            I32, # size_t n_threads
+        ]
+        self.library.minigpt4_encode_image.restype = I32
+
+        self.library.minigpt4_begin_chat_image.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding
+            CHAR_PTR, # const char *s
+            I32, # size_t n_threads
+        ]
+        self.library.minigpt4_begin_chat_image.restype = I32
+
+        self.library.minigpt4_end_chat_image.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            CHAR_PTR_PTR, # const char **token
+            I32, # size_t n_threads
+            F32, # float temp
+            I32, # int32_t top_k
+            F32, # float top_p
+            F32, # float tfs_z
+            F32, # float typical_p
+            I32, # int32_t repeat_last_n
+            F32, # float repeat_penalty
+            F32, # float alpha_presence
+            F32, # float alpha_frequency
+            I32, # int mirostat
+            F32, # float mirostat_tau
+            F32, # float mirostat_eta
+            I32, # int penalize_nl
+        ]
+        self.library.minigpt4_end_chat_image.restype = I32
+
+        self.library.minigpt4_system_prompt.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            I32, # size_t n_threads
+        ]
+        self.library.minigpt4_system_prompt.restype = I32
+
+        self.library.minigpt4_begin_chat.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            CHAR_PTR, # const char *s
+            I32, # size_t n_threads
+        ]
+        self.library.minigpt4_begin_chat.restype = I32
+
+        self.library.minigpt4_end_chat.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+            CHAR_PTR_PTR, # const char **token
+            I32, # size_t n_threads
+            F32, # float temp
+            I32, # int32_t top_k
+            F32, # float top_p
+            F32, # float tfs_z
+            F32, # float typical_p
+            I32, # int32_t repeat_last_n
+            F32, # float repeat_penalty
+            F32, # float alpha_presence
+            F32, # float alpha_frequency
+            I32, # int mirostat
+            F32, # float mirostat_tau
+            F32, # float mirostat_eta
+            I32, # int penalize_nl
+        ]
+        self.library.minigpt4_end_chat.restype = I32
+
+        self.library.minigpt4_reset_chat.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+        ]
+        self.library.minigpt4_reset_chat.restype = I32
+
+        self.library.minigpt4_contains_eos_token.argtypes = [
+            CHAR_PTR, # const char *s
+        ]
+        self.library.minigpt4_contains_eos_token.restype = I32
+
+        self.library.minigpt4_is_eos.argtypes = [
+            CHAR_PTR, # const char *s
+        ]
+        self.library.minigpt4_is_eos.restype = I32
+
+        self.library.minigpt4_free.argtypes = [
+            MiniGPT4ContextP, # struct MiniGPT4Context *ctx
+        ]
+        self.library.minigpt4_free.restype = I32
+
+        self.library.minigpt4_free_image.argtypes = [
+            MiniGPT4ImageP, # struct MiniGPT4Image *image
+        ]
+        self.library.minigpt4_free_image.restype = I32
+
+        self.library.minigpt4_free_embedding.argtypes = [
+            MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding
+        ]
+        self.library.minigpt4_free_embedding.restype = I32
+
+        self.library.minigpt4_error_code_to_string.argtypes = [
+            I32, # int error_code
+        ]
+        self.library.minigpt4_error_code_to_string.restype = CHAR_PTR
+
+        self.library.minigpt4_quantize_model.argtypes = [
+            CHAR_PTR, # const char *in_path
+            CHAR_PTR, # const char *out_path
+            I32, # int data_type
+        ]
+        self.library.minigpt4_quantize_model.restype = I32
+
+        self.library.minigpt4_set_verbosity.argtypes = [
+            I32, # int verbosity
+        ]
+        self.library.minigpt4_set_verbosity.restype = None
+
+    def panic_if_error(self, error_code: int) -> None:
+        """
+        Raises an exception if the error code is not 0.
+
+        Parameters
+        ----------
+        error_code : int
+            Error code to check.
+        """
+
+        if error_code != 0:
+            raise RuntimeError(self.library.minigpt4_error_code_to_string(I32(error_code)))
+
+    def minigpt4_model_load(self, model_path: str, llm_model_path: str, verbosity: int = 1, seed: int = 1337, n_ctx: int = 2048, n_batch: int = 512, numa: int = 0) -> MiniGPT4Context:
+        """
+        Loads a model from a file.
+
+        Args:
+            model_path (str): Path to model file.
+            llm_model_path (str): Path to LLM model file.
+            verbosity (int): Verbosity level: 0 = silent, 1 = error, 2 = info, 3 = debug. Defaults to 0.
+            n_ctx (int): Size of context for llm model. Defaults to 2048.
+            seed (int): Seed for llm model. Defaults to 1337.
+            numa (int): NUMA node to use (0 = NUMA disabled, 1 = NUMA enabled). Defaults to 0.
+
+        Returns:
+            MiniGPT4Context: Context.
+        """
+
+        ptr = self.library.minigpt4_model_load(
+            model_path.encode('utf-8'),
+            llm_model_path.encode('utf-8'),
+            I32(verbosity),
+            I32(seed),
+            I32(n_ctx),
+            I32(n_batch),
+            I32(numa),
+        )
+
+        assert ptr is not None, 'minigpt4_model_load failed'
+
+        return MiniGPT4Context(ptr)
+
+    def minigpt4_image_load_from_file(self, ctx: MiniGPT4Context, path: str, flags: int) -> MiniGPT4Image:
+        """
+        Loads an image from a file
+
+        Args:
+            ctx (MiniGPT4Context): context
+            path (str): path
+            flags (int): flags
+
+        Returns:
+            MiniGPT4Image: image
+        """
+
+        image = MiniGPT4Image()
+        self.panic_if_error(self.library.minigpt4_image_load_from_file(ctx.ptr, path.encode('utf-8'), ctypes.pointer(image), I32(flags)))
+        return image
+
+    def minigpt4_preprocess_image(self, ctx: MiniGPT4Context, image: MiniGPT4Image, flags: int = 0) -> MiniGPT4Image:
+        """
+        Preprocesses an image
+
+        Args:
+            ctx (MiniGPT4Context): Context
+            image (MiniGPT4Image): Image
+            flags (int): Flags. Defaults to 0.
+
+        Returns:
+            MiniGPT4Image: Preprocessed image
+        """
+
+        preprocessed_image = MiniGPT4Image()
+        self.panic_if_error(self.library.minigpt4_preprocess_image(ctx.ptr, ctypes.pointer(image), ctypes.pointer(preprocessed_image), I32(flags)))
+        return preprocessed_image
+
+    def minigpt4_encode_image(self, ctx: MiniGPT4Context, image: MiniGPT4Image, n_threads: int = 0) -> MiniGPT4Embedding:
+        """
+        Encodes an image into embedding
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            image (MiniGPT4Image): Image.
+            n_threads (int): Number of threads to use, if 0, uses all available. Defaults to 0.
+
+        Returns:
+            embedding (MiniGPT4Embedding): Output embedding.
+        """
+
+        embedding = MiniGPT4Embedding()
+        self.panic_if_error(self.library.minigpt4_encode_image(ctx.ptr, ctypes.pointer(image), ctypes.pointer(embedding), n_threads))
+        return embedding
+
+    def minigpt4_begin_chat_image(self, ctx: MiniGPT4Context, image_embedding: MiniGPT4Embedding, s: str, n_threads: int = 0):
+        """
+        Begins a chat with an image.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            image_embedding (MiniGPT4Embedding): Image embedding.
+            s (str): Question to ask about the image.
+            n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.
+
+        Returns:
+            None
+        """
+
+        self.panic_if_error(self.library.minigpt4_begin_chat_image(ctx.ptr, ctypes.pointer(image_embedding), s.encode('utf-8'), n_threads))
+
+    def minigpt4_end_chat_image(self, ctx: MiniGPT4Context, n_threads: int = 0, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1) -> str:
+        """
+        Ends a chat with an image.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.
+            temp (float, optional): Temperature. Defaults to 0.8.
+            top_k (int, optional): Top K. Defaults to 40.
+            top_p (float, optional): Top P. Defaults to 0.9.
+            tfs_z (float, optional): Tfs Z. Defaults to 1.0.
+            typical_p (float, optional): Typical P. Defaults to 1.0.
+            repeat_last_n (int, optional): Repeat last N. Defaults to 64.
+            repeat_penalty (float, optional): Repeat penality. Defaults to 1.1.
+            alpha_presence (float, optional): Alpha presence. Defaults to 1.0.
+            alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.
+            mirostat (int, optional): Mirostat. Defaults to 0.
+            mirostat_tau (float, optional): Mirostat Tau. Defaults to 5.0.
+            mirostat_eta (float, optional): Mirostat Eta. Defaults to 1.0.
+            penalize_nl (int, optional): Penalize NL. Defaults to 1.
+
+        Returns:
+            str: Token generated.
+        """
+
+        token = CHAR_PTR()
+        self.panic_if_error(self.library.minigpt4_end_chat_image(ctx.ptr, ctypes.pointer(token), n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl))
+        return ctypes.cast(token, ctypes.c_char_p).value.decode('utf-8')
+
+    def minigpt4_system_prompt(self, ctx: MiniGPT4Context, n_threads: int = 0):
+        """
+        Generates a system prompt.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.
+        """
+
+        self.panic_if_error(self.library.minigpt4_system_prompt(ctx.ptr, n_threads))
+
+    def minigpt4_begin_chat(self, ctx: MiniGPT4Context, s: str, n_threads: int = 0):
+        """
+        Begins a chat continuing after minigpt4_begin_chat_image
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            s (str): Question to ask about the image.
+            n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.
+
+        Returns:
+            None
+        """
+        self.panic_if_error(self.library.minigpt4_begin_chat(ctx.ptr, s.encode('utf-8'), n_threads))
+
+    def minigpt4_end_chat(self, ctx: MiniGPT4Context, n_threads: int = 0, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1) -> str:
+        """
+        Ends a chat.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+            n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.
+            temp (float, optional): Temperature. Defaults to 0.8.
+            top_k (int, optional): Top K. Defaults to 40.
+            top_p (float, optional): Top P. Defaults to 0.9.
+            tfs_z (float, optional): Tfs Z. Defaults to 1.0.
+            typical_p (float, optional): Typical P. Defaults to 1.0.
+            repeat_last_n (int, optional): Repeat last N. Defaults to 64.
+            repeat_penalty (float, optional): Repeat penality. Defaults to 1.1.
+            alpha_presence (float, optional): Alpha presence. Defaults to 1.0.
+            alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.
+            mirostat (int, optional): Mirostat. Defaults to 0.
+            mirostat_tau (float, optional): Mirostat Tau. Defaults to 5.0.
+            mirostat_eta (float, optional): Mirostat Eta. Defaults to 1.0.
+            penalize_nl (int, optional): Penalize NL. Defaults to 1.
+
+        Returns:
+            str: Token generated.
+        """
+
+        token = CHAR_PTR()
+        self.panic_if_error(self.library.minigpt4_end_chat(ctx.ptr, ctypes.pointer(token), n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl))
+        return ctypes.cast(token, ctypes.c_char_p).value.decode('utf-8')
+
+    def minigpt4_reset_chat(self, ctx: MiniGPT4Context):
+        """
+        Resets the chat.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+        """
+        self.panic_if_error(self.library.minigpt4_reset_chat(ctx.ptr))
+
+    def minigpt4_contains_eos_token(self, s: str) -> bool:
+
+        """
+        Checks if a string contains an EOS token.
+
+        Args:
+            s (str): String to check.
+        
+        Returns:
+            bool: True if the string contains an EOS token, False otherwise.
+        """
+
+        return self.library.minigpt4_contains_eos_token(s.encode('utf-8'))
+
+    def minigpt4_is_eos(self, s: str) -> bool:
+
+        """
+        Checks if a string is EOS.
+
+        Args:
+            s (str): String to check.
+        
+        Returns:
+            bool: True if the string contains an EOS, False otherwise.
+        """
+
+        return self.library.minigpt4_is_eos(s.encode('utf-8'))
+
+
+    def minigpt4_free(self, ctx: MiniGPT4Context) -> None:
+        """
+        Frees a context.
+
+        Args:
+            ctx (MiniGPT4Context): Context.
+        """
+
+        self.panic_if_error(self.library.minigpt4_free(ctx.ptr))
+
+    def minigpt4_free_image(self, image: MiniGPT4Image) -> None:
+        """
+        Frees an image.
+
+        Args:
+            image (MiniGPT4Image): Image.
+        """
+
+        self.panic_if_error(self.library.minigpt4_free_image(ctypes.pointer(image)))
+
+    def minigpt4_free_embedding(self, embedding: MiniGPT4Embedding) -> None:
+        """
+        Frees an embedding.
+
+        Args:
+            embedding (MiniGPT4Embedding): Embedding.
+        """
+
+        self.panic_if_error(self.library.minigpt4_free_embedding(ctypes.pointer(embedding)))
+
+    def minigpt4_error_code_to_string(self, error_code: int) -> str:
+        """
+        Converts an error code to a string.
+
+        Args:
+            error_code (int): Error code.
+
+        Returns:
+            str: Error string.
+        """
+
+        return self.library.minigpt4_error_code_to_string(error_code).decode('utf-8')
+
+    def minigpt4_quantize_model(self, in_path: str, out_path: str, data_type: DataType):
+        """
+        Quantizes a model file.
+
+        Args:
+            in_path (str): Path to input model file.
+            out_path (str): Path to write output model file.
+            data_type (DataType): Must be one DataType enum values.
+        """
+
+        self.panic_if_error(self.library.minigpt4_quantize_model(in_path.encode('utf-8'), out_path.encode('utf-8'), data_type))
+
+    def minigpt4_set_verbosity(self, verbosity: Verbosity):
+        """
+        Sets verbosity.
+
+        Args:
+            verbosity (int): Verbosity.
+        """
+
+        self.library.minigpt4_set_verbosity(I32(verbosity))
+
+def load_library() -> MiniGPT4SharedLibrary:
+    """
+    Attempts to find minigpt4.cpp shared library and load it.
+    """
+
+    file_name: str
+
+    if 'win32' in sys.platform or 'cygwin' in sys.platform:
+        file_name = 'minigpt4.dll'
+    elif 'darwin' in sys.platform:
+        file_name = 'libminigpt4.dylib'
+    else:
+        file_name = 'libminigpt4.so'
+
+    repo_root_dir: pathlib.Path = pathlib.Path(os.path.abspath(__file__)).parent.parent
+
+    paths = [
+        # If we are in "minigpt4" directory
+        f'../bin/Release/{file_name}',
+        # If we are in repo root directory
+        f'bin/Release/{file_name}',
+        # If we compiled in build directory
+        f'build/bin/Release/{file_name}',
+        # If we compiled in build directory
+        f'build/{file_name}',
+        f'../build/{file_name}',
+        # Search relative to this file
+        str(repo_root_dir / 'bin' / 'Release' / file_name),
+        # Fallback
+        str(repo_root_dir / file_name)
+    ]
+
+    for path in paths:
+        if os.path.isfile(path):
+            return MiniGPT4SharedLibrary(path)
+
+    return MiniGPT4SharedLibrary(paths[-1])
+
+class MiniGPT4ChatBot:
+    def __init__(self, model_path: str, llm_model_path: str, verbosity: Verbosity = Verbosity.SILENT, n_threads: int = 0):
+        """
+        Creates a new MiniGPT4ChatBot instance.
+
+        Args:
+            model_path (str): Path to model file.
+            llm_model_path (str): Path to language model model file.
+            verbosity (Verbosity, optional): Verbosity. Defaults to Verbosity.SILENT.
+            n_threads (int, optional): Number of threads to use. Defaults to 0.
+        """
+            
+        self.library = load_library()
+        self.ctx = self.library.minigpt4_model_load(model_path, llm_model_path, verbosity)
+        self.n_threads = n_threads
+
+        from PIL import Image
+        from torchvision import transforms
+        from torchvision.transforms.functional import InterpolationMode
+        self.image_size = 224
+
+        mean = (0.48145466, 0.4578275, 0.40821073)
+        std = (0.26862954, 0.26130258, 0.27577711)
+        self.transform = transforms.Compose(
+            [
+                transforms.RandomResizedCrop(
+                    self.image_size,
+                    interpolation=InterpolationMode.BICUBIC,
+                ),
+                transforms.ToTensor(),
+                transforms.Normalize(mean, std)
+            ]
+        )
+        self.embedding: Optional[MiniGPT4Embedding] = None
+        self.is_image_chat = False
+        self.chat_history = []
+
+    def free(self):
+        if self.ctx:
+            self.library.minigpt4_free(self.ctx)
+
+    def generate(self, message: str, limit: int = 1024, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1):
+        """
+        Generates a chat response.
+
+        Args:
+            message (str): Message.
+            limit (int, optional): Limit. Defaults to 1024.
+            temp (float, optional): Temperature. Defaults to 0.8.
+            top_k (int, optional): Top K. Defaults to 40.
+            top_p (float, optional): Top P. Defaults to 0.9.
+            tfs_z (float, optional): TFS Z. Defaults to 1.0.
+            typical_p (float, optional): Typical P. Defaults to 1.0.
+            repeat_last_n (int, optional): Repeat last N. Defaults to 64.
+            repeat_penalty (float, optional): Repeat penalty. Defaults to 1.1.
+            alpha_presence (float, optional): Alpha presence. Defaults to 1.0.
+            alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.
+            mirostat (int, optional): Mirostat. Defaults to 0.
+            mirostat_tau (float, optional): Mirostat tau. Defaults to 5.0.
+            mirostat_eta (float, optional): Mirostat eta. Defaults to 1.0.
+            penalize_nl (int, optional): Penalize NL. Defaults to 1.
+        """
+        if self.is_image_chat:
+            self.is_image_chat = False
+            self.library.minigpt4_begin_chat_image(self.ctx, self.embedding, message, self.n_threads)
+            chat = ''
+            for _ in range(limit):
+                token = self.library.minigpt4_end_chat_image(self.ctx, self.n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl)
+                chat += token
+                if self.library.minigpt4_contains_eos_token(token):
+                    continue
+                if self.library.minigpt4_is_eos(chat):
+                    break
+                yield token
+        else:
+            self.library.minigpt4_begin_chat(self.ctx, message, self.n_threads)
+            chat = ''
+            for _ in range(limit):
+                token = self.library.minigpt4_end_chat(self.ctx, self.n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl)
+                chat += token
+                if self.library.minigpt4_contains_eos_token(token):
+                    continue
+                if self.library.minigpt4_is_eos(chat):
+                    break
+                yield token
+
+    def reset_chat(self):
+        """
+        Resets the chat.
+        """
+
+        self.is_image_chat = False
+        if self.embedding:
+            self.library.minigpt4_free_embedding(self.embedding)
+            self.embedding = None
+
+        self.library.minigpt4_reset_chat(self.ctx)
+        self.library.minigpt4_system_prompt(self.ctx, self.n_threads)
+
+    def upload_image(self, image):
+        """
+        Uploads an image.
+        
+        Args:
+            image (Image): Image.
+        """
+
+        self.reset_chat()
+
+        image = self.transform(image)
+        image = image.unsqueeze(0)
+        image = image.numpy()
+        image = image.ctypes.data_as(ctypes.c_void_p)
+        minigpt4_image = MiniGPT4Image(image, self.image_size, self.image_size, 3, ImageFormat.F32)
+        self.embedding = self.library.minigpt4_encode_image(self.ctx, minigpt4_image, self.n_threads)
+        
+        self.is_image_chat = True
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description='Test loading minigpt4')
+    parser.add_argument('model_path', help='Path to model file')
+    parser.add_argument('llm_model_path', help='Path to llm model file')
+    parser.add_argument('-i', '--image_path', help='Image to test', default='images/llama.png')
+    parser.add_argument('-p', '--prompts', help='Text to test', default='what is the text in the picture?,what is the color of it?')
+    args = parser.parse_args()
+
+    model_path = args.model_path
+    llm_model_path = args.llm_model_path
+    image_path = args.image_path
+    prompts = args.prompts
+
+    if not Path(model_path).exists():
+        print(f'Model does not exist: {model_path}')
+        exit(1) 
+
+    if not Path(llm_model_path).exists():
+        print(f'LLM Model does not exist: {llm_model_path}')
+        exit(1)
+
+    prompts = prompts.split(',')
+
+    print('Loading minigpt4 shared library...')
+    library = load_library()
+    print(f'Loaded library {library}')
+    ctx = library.minigpt4_model_load(model_path, llm_model_path, Verbosity.DEBUG)
+    image = library.minigpt4_image_load_from_file(ctx, image_path, 0)
+    preprocessed_image = library.minigpt4_preprocess_image(ctx, image, 0)
+
+    question = prompts[0]
+    n_threads = 0
+    embedding = library.minigpt4_encode_image(ctx, preprocessed_image, n_threads)
+    library.minigpt4_system_prompt(ctx, n_threads)
+    library.minigpt4_begin_chat_image(ctx, embedding, question, n_threads)
+    chat = ''
+    while True:
+        token = library.minigpt4_end_chat_image(ctx, n_threads)
+        chat += token
+        if library.minigpt4_contains_eos_token(token):
+            continue
+        if library.minigpt4_is_eos(chat):
+            break
+        print(token, end='')
+
+    for i in range(1, len(prompts)):
+        prompt = prompts[i]
+        library.minigpt4_begin_chat(ctx, prompt, n_threads)
+        chat  = ''
+        while True:
+            token = library.minigpt4_end_chat(ctx, n_threads)
+            chat += token
+            if library.minigpt4_contains_eos_token(token):
+                continue
+            if library.minigpt4_is_eos(chat):
+                break
+            print(token, end='')
+
+    library.minigpt4_free_image(image)
+    library.minigpt4_free_image(preprocessed_image)
+    library.minigpt4_free(ctx)

requirements.txt

@@ -0,0 +1,21 @@
+timm==0.6.7
+deepspeed==0.9.2
+data
+einops==0.6.1
+ftfy==6.1.1
+iopath==0.1.10
+ipdb==0.13.13
+numpy==1.24.3
+peft==0.3.0
+Pillow==9.5.0
+PyYAML==6.0
+regex==2022.10.31
+torchvision==0.14.1
+torchaudio==0.13.1
+pytorchvideo
+fvcore
+decord==0.6.0
+tqdm==4.64.1
+transformers==4.29.1
+gradio
+huggingface_hub