Duplicate from matthoffner/ggml-ctransformers-fastapi

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,29 @@
+FROM python:latest
+
+ENV PYTHONUNBUFFERED 1
+
+EXPOSE 8000
+
+WORKDIR /app
+
+RUN wget -qO- "https://cmake.org/files/v3.17/cmake-3.17.0-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local
+
+COPY requirements.txt ./
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+RUN apt-get install git-lfs
+
+RUN git clone https://github.com/ggerganov/ggml && cd ggml && mkdir build && cd build && cmake ..
+RUN git clone https://huggingface.co/bigcode/gpt_bigcode-santacoder
+RUN python ggml/examples/starcoder/convert-hf-to-ggml.py ./gpt_bigcode-santacoder/
+RUN cd ggml/build && make -j4 starcoder starcoder-quantize
+
+RUN ggml/build/bin/starcoder-quantize models/./gpt_bigcode-santacoder/-ggml.bin ggml-model-q4_1.bin 3
+
+COPY . .
+
+RUN ls -al
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,19 @@
+---
+title: ggml-ctransformers-fastapi
+emoji: 🐳🤗⚡️
+sdk: docker
+app_port: 8000
+duplicated_from: matthoffner/ggml-ctransformers-fastapi
+---
+
+# ggml-ctransformers-fastapi
+
+## <a href="https://github.com/ggerganov/ggml" target="_blank">ggml</a>
+## <a href="https://github.com/marella/ctransformers" target="_blank">ctransformers</a>
+## [FastAPI Docs](https://matthoffner-ggml-ctransformers-fastapi.hf.space/docs)
+
+### Updates
+
+* Added /v1/chat/completions
+* [Start using ctransformers](https://github.com/marella/ctransformers)
+* [Added starcoder example](https://github.com/ggerganov/ggml/tree/master/examples/starcoder)

main.py

@@ -0,0 +1,43 @@
+import fastapi
+import json
+import markdown
+import uvicorn
+from fastapi.responses import HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from sse_starlette.sse import EventSourceResponse
+from ctransformers.langchain import CTransformers
+from pydantic import BaseModel
+
+llm = CTransformers(model='ggml-model-q4_1.bin', model_type='starcoder')
+app = fastapi.FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/")
+async def index():
+    with open("README.md", "r", encoding="utf-8") as readme_file:
+        md_template_string = readme_file.read()
+    html_content = markdown.markdown(md_template_string)
+    return HTMLResponse(content=html_content, status_code=200)
+
+class ChatCompletionRequest(BaseModel):
+    prompt: str
+
+@app.post("/v1/chat/completions")
+async def chat(request: ChatCompletionRequest, response_mode=None):
+    completion = llm(request.prompt)
+    async def server_sent_events(chat_chunks):
+        for chat_chunk in chat_chunks:
+            yield dict(data=json.dumps(chat_chunk))
+        yield dict(data="[DONE]")
+
+    return EventSourceResponse(server_sent_events(completion))
+
+if __name__ == "__main__":
+  uvicorn.run(app, host="0.0.0.0", port=8000)
+

requirements.txt

@@ -0,0 +1,11 @@
+uvicorn
+markdown
+fastapi
+loguru
+torch
+numpy
+transformers
+ctransformers
+accelerate
+langchain
+sse_starlette