Duplicate from mikeee/30B-Lazarus-GGML

Co-authored-by: mikeee <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+call-activate.bat
+okteto.yml
+okteto-up.bat
+install-sw.sh
+install-sw1.sh
+start-sshd.sh
+pyproject.toml
+models
+.ruff_cache
+run-nodemon.sh
+app-.py

.ruff.toml

@@ -0,0 +1,19 @@
+# Assume Python 3.10.
+target-version = "py310"
+# Decrease the maximum line length to 79 characters.
+line-length = 300
+
+# pyflakes, pycodestyle, isort
+# flake8 YTT, pydocstyle D, pylint PLC
+select = ["F", "E", "W", "I001", "YTT", "D", "PLC"]
+# select = ["ALL"]
+
+# D100 Missing docstring in public module
+# E501 Line too long
+# D103 Missing docstring in public function
+# D101 Missing docstring in public class
+# `multi-line-summary-first-line` (D212)
+# `one-blank-line-before-class` (D203)
+extend-ignore = ["E501", "D100", "D103", "D101", "D212", "D203"]
+
+exclude = [".venv"]

.stignore

@@ -0,0 +1,102 @@
+models
+.git
+# Byte-compiled / optimized / DLL files
+__pycache__
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build
+develop-eggs
+dist
+downloads
+eggs
+.eggs
+lib
+lib64
+parts
+sdist
+var
+wheels
+pip-wheel-metadata
+share/python-wheels
+*.egg-info
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build
+
+# PyBuilder
+target
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env
+venv
+ENV
+env.bak
+venv.bak
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre

README.md

@@ -0,0 +1,13 @@
+---
+title: 30B-Lazarus-GGML
+emoji: 🚀
+colorFrom: green
+colorTo: green
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+duplicated_from: mikeee/30B-Lazarus-GGML
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,274 @@
+from pathlib import Path
+from urllib.parse import urlparse
+
+import gradio as gr
+import psutil
+from ctransformers import AutoModelForCausalLM
+from huggingface_hub import hf_hub_download
+
+_ = """
+snapshot_download(
+  repo_id="TheBloke/falcon-7b-instruct-GGML",
+  allow_patterns="falcon7b-instruct.ggmlv3.q4_0.bin",
+  revision="ggmlv3",
+  local_dir="models",
+  local_dir_use_symlinks=False,  # default "auto"
+)
+
+hf_hub_download(
+    repo_id=repo_id,
+    filename=model_filename,
+    local_dir=local_path,
+    local_dir_use_symlinks=True,
+)
+# """
+# 4.06G
+
+_ = """
+llm = AutoModelForCausalLM.from_pretrained(
+    "TheBloke/falcon-7b-instruct-GGML",
+    model_file="falcon7b-instruct.ggmlv3.q4_0.bin",
+    model_type="falcon", gpu_layers=32, threads=2,
+)
+# """
+# _ = Path("models", "falcon7b-instruct.ggmlv3.q4_0.bin").absolute().as_posix()
+# assert Path(_).exists(), f"{_} does not exist, perhaps snapshot_download failed?"
+
+URL = "https://huggingface.co/TheBloke/falcon-7b-instruct-GGML/blob/main/falcon-7b-instruct.ggccv1.q4_1.bin"
+URL = "https://huggingface.co/TheBloke/falcon-7b-instruct-GGML/blob/ggmlv3/falcon7b-instruct.ggmlv3.q4_1.bin"
+repo_id = "/".join(urlparse(URL).path.strip("/").split("/")[:2])
+URL = "https://huggingface.co/TheBloke/30B-Lazarus-GGML/blob/main/30b-Lazarus.ggmlv3.q2_K.bin"  # 13.6 G
+repo_id = "/".join(urlparse(URL).path.strip("/").split("/")[:2])
+model_file = Path(URL).name
+
+_ = """
+_ = hf_hub_download(
+    repo_id=repo_id,
+    revision="ggmlv3",
+    filename=model_file,
+    local_dir="models",
+    # local_dir_use_symlinks=True,
+)
+# """  # for falcon
+
+_ = hf_hub_download(
+    repo_id=repo_id,
+    filename=model_file,
+    local_dir="models",
+    # local_dir_use_symlinks=True,
+)
+
+
+llm = AutoModelForCausalLM.from_pretrained(
+    # repo_id,  # "TheBloke/falcon-7b-instruct-GGML",
+    # model_file=model_file,
+    # model_file=_,
+    _,
+    # model_type="falcon",
+    model_type="llama",
+    threads=psutil.cpu_count(logical=False),
+)
+
+TITLE = f"""<h2 align="center">🦅 30B-Lazarus-GGM ({model_file}) 🦅"""
+USER_NAME = "User"
+BOT_NAME = "Assistant"
+DEFAULT_INSTRUCTIONS = """The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Assistant, and a human user, called User. In the following interactions, User and Assistant will converse in natural language, and Assistant will answer User's questions. Assistant will never decline to answer a question, and always attempts to give an answer that User would be satisfied with. It knows a lot, and always tells the truth. The conversation begins.
+"""
+RETRY_COMMAND = "/retry"
+STOP_STR = f"\n{USER_NAME}:"
+STOP_SUSPECT_LIST = [":", "\n", "User"]
+
+
+def chat_accordion():
+    with gr.Accordion("Parameters", open=False):
+        temperature = gr.Slider(
+            minimum=0.1,
+            maximum=2.0,
+            value=0.8,
+            step=0.1,
+            interactive=True,
+            label="Temperature",
+        )
+        top_p = gr.Slider(
+            minimum=0.1,
+            maximum=0.99,
+            value=0.9,
+            step=0.01,
+            interactive=True,
+            label="p (nucleus sampling)",
+        )
+    return temperature, top_p
+
+
+# TODO: fix prompt
+def format_chat_prompt(message: str, chat_history, instructions: str) -> str:
+    instructions = instructions.strip(" ").strip("\n")
+    prompt = instructions
+    for turn in chat_history:
+        user_message, bot_message = turn
+        prompt = f"{prompt}\n{USER_NAME}: {user_message}\n{BOT_NAME}: {bot_message}"
+    prompt = f"{prompt}\n{USER_NAME}: {message}\n{BOT_NAME}:"
+    return prompt
+
+
+def chat():
+    with gr.Column(elem_id="chat_container"):
+        with gr.Row():
+            chatbot = gr.Chatbot(elem_id="chatbot")
+        with gr.Row():
+            inputs = gr.Textbox(
+                placeholder="Ask me anything...",
+                label="Type an input and press Enter",
+                max_lines=3,
+            )
+
+    with gr.Row(elem_id="button_container"):
+        # with gr.Column(min_width=32):
+            # submit_button = gr.Button("🚀 Submit")
+        with gr.Column(min_width=32):
+            retry_button = gr.Button("♻️ Retry last turn")
+        with gr.Column(min_width=32):
+            delete_turn_button = gr.Button("🧽 Delete last turn")
+        with gr.Column(min_width=32):
+            clear_chat_button = gr.Button("✨ Delete all history")
+
+    gr.Examples(
+        [
+            ["Hey! Any recommendations for my holidays in Abu Dhabi?"],
+            ["What's the Everett interpretation of quantum mechanics?"],
+            [
+                "Give me a list of the top 10 dive sites you would recommend around the world."
+            ],
+            ["Can you tell me more about deep-water soloing?"],
+            [
+                "Can you write a short tweet about 30B-Lazarus-GGM?"
+            ],
+        ],
+        inputs=inputs,
+        label="Click on any example and press Enter in the input textbox!",
+    )
+
+    with gr.Row(elem_id="param_container"):
+        with gr.Column():
+            temperature, top_p = chat_accordion()
+        with gr.Column():
+            with gr.Accordion("Instructions", open=False):
+                instructions = gr.Textbox(
+                    placeholder="LLM instructions",
+                    value=DEFAULT_INSTRUCTIONS,
+                    lines=10,
+                    interactive=True,
+                    label="Instructions",
+                    max_lines=16,
+                    show_label=False,
+                )
+
+    def run_chat(
+        message: str, chat_history, instructions: str, temperature: float, top_p: float
+    ):
+        if not message or (message == RETRY_COMMAND and len(chat_history) == 0):
+            yield chat_history
+            return
+
+        if message == RETRY_COMMAND and chat_history:
+            prev_turn = chat_history.pop(-1)
+            user_message, _ = prev_turn
+            message = user_message
+
+        prompt = format_chat_prompt(message, chat_history, instructions)
+        chat_history = chat_history + [[message, ""]]
+        stream = llm(
+            prompt,
+            max_new_tokens=1024,
+            stop=[STOP_STR, "<|endoftext|>"],
+            temperature=temperature,
+            top_p=top_p,
+            stream=True,
+        )
+        acc_text = ""
+        for idx, response in enumerate(stream):
+            text_token = response
+
+            if text_token in STOP_SUSPECT_LIST:
+                acc_text += text_token
+                continue
+
+            if idx == 0 and text_token.startswith(" "):
+                text_token = text_token[1:]
+
+            acc_text += text_token
+            last_turn = list(chat_history.pop(-1))
+            last_turn[-1] += acc_text
+            chat_history = chat_history + [last_turn]
+            yield chat_history
+            acc_text = ""
+
+    def delete_last_turn(chat_history):
+        if chat_history:
+            chat_history.pop(-1)
+        return {chatbot: gr.update(value=chat_history)}
+
+    def run_retry(
+        message: str, chat_history, instructions: str, temperature: float, top_p: float
+    ):
+        yield from run_chat(
+            RETRY_COMMAND, chat_history, instructions, temperature, top_p
+        )
+
+    def clear_chat():
+        return []
+
+    inputs.submit(
+        run_chat,
+        [inputs, chatbot, instructions, temperature, top_p],
+        outputs=[chatbot],
+        show_progress="minimal",
+    )
+    inputs.submit(lambda: "", inputs=None, outputs=inputs)
+    # submit_button.click(
+        # run_chat,
+        # [inputs, chatbot, instructions, temperature, top_p],
+        # outputs=[chatbot],
+        # show_progress="minimal",
+    # )
+    delete_turn_button.click(delete_last_turn, inputs=[chatbot], outputs=[chatbot])
+    retry_button.click(
+        run_retry,
+        [inputs, chatbot, instructions, temperature, top_p],
+        outputs=[chatbot],
+        show_progress="minimal",
+    )
+    clear_chat_button.click(clear_chat, [], chatbot)
+
+
+def get_demo():
+    with gr.Blocks(
+        # css=None
+        # css="""#chat_container {width: 700px; margin-left: auto; margin-right: auto;}
+        #        #button_container {width: 700px; margin-left: auto; margin-right: auto;}
+        #        #param_container {width: 700px; margin-left: auto; margin-right: auto;}"""
+        css="""#chatbot {
+    font-size: 14px;
+    min-height: 300px;
+}"""
+    ) as demo:
+        gr.HTML(TITLE)
+
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(
+                    """
+                    ⚠️ **Limitations**: the model can and will produce factually incorrect information, hallucinating facts and actions. As it has not undergone any advanced tuning/alignment, it can produce problematic outputs, especially if prompted to do so.
+                    """
+                )
+
+        chat()
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo = get_demo()
+    demo.queue(max_size=64, concurrency_count=8)
+    # demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch()

requirements.txt

@@ -0,0 +1,7 @@
+ctransformers
+transformers
+huggingface_hub
+# gradio
+loguru
+about-time
+psutil