## KoboldCpp based GGML Backend by Concedo ## For use as a custom backend in KoboldAI United ## Not intended for general use. from __future__ import annotations import time, json import torch import requests import numpy as np from typing import List, Optional, Union import os from . import koboldcpp import utils from logger import logger from modeling.inference_model import ( GenerationResult, GenerationSettings, InferenceModel, ) model_backend_name = "koboldcpp" #specific instead of ggml model_backend_type = "ggml" #This should be a generic name in case multiple model backends are compatible (think Hugging Face Custom and Basic Hugging Face) kcpp_backend_loaded = False class KoboldCppException(Exception): """To be used for errors on cpp side of KoboldCpp.""" class KcppArgsObject: def __init__(self, **kwargs): self.__dict__.update(kwargs) class model_backend(InferenceModel): def __init__(self) -> None: super().__init__() def is_valid(self, model_name, model_path, menu_path): foundfile = False try: files = os.listdir(model_path) foundfile = len([filename for filename in files if (("ggml" in filename.lower() and ".bin" in filename.lower()) or ".gguf" in filename.lower())])>0 except: pass return foundfile def get_requested_parameters(self, model_name, model_path, menu_path, parameters = {}): self.kcpp_threads = 5 self.model_name = "GGML_Model" self.kcpp_ctxsize = 2048 self.kcpp_blasbatchsize = 512 self.kcpp_gpulayers = 0 self.kcpp_smartcontext = False self.kcpp_ropescale = 0.0 self.kcpp_ropebase = 10000.0 self.kcpp_useclblast = None self.kcpp_usecublas = None self.kcpp_noblas = False self.kcpp_noavx2 = False self.kcpp_nommap = False self.kcpp_debugmode = 0 self.kcpp_tensor_split_str = "" self.kcpp_tensor_split = None files = os.listdir(model_path) foundfiles = [filename for filename in files if (("ggml" in filename.lower() and ".bin" in filename.lower()) or ".gguf" in filename.lower())] requested_parameters = [] foldermdls = [] for ff in foundfiles: foldermdls.append({'text': ff, 'value': os.path.join(model_path, ff)}) requested_parameters.append({ "uitype": "dropdown", "unit": "string", "label": "GGML DataFile Name", "id": "kcpp_filename", "default": os.path.join(model_path, foundfiles[0]) if len(foundfiles)>0 else model_name, "check": {"value": "", 'check': "!="}, "tooltip": "Actual GGML DataFile Name", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "", 'children': foldermdls }) requested_parameters.append({ "uitype": "dropdown", "unit": "int", "label": "KoboldCpp Accelerator", "id": "kcpp_accelerator", "default": 0, "check": {"value": "", 'check': "!="}, 'multiple': False, "tooltip": "KoboldCpp Accelerator", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "", 'children': [{'text': 'Use No BLAS', 'value': 0}, {'text': 'Use OpenBLAS', 'value': 1}, {'text': 'Use CuBLAS', 'value': 2}, {'text': 'Use CLBLast GPU #1', 'value': 3},{'text': 'Use CLBLast GPU #2', 'value': 4},{'text': 'Use CLBLast GPU #3', 'value': 5} ,{'text': 'NoAVX2 Mode (Old CPU)', 'value': 6},{'text': 'Failsafe Mode (Old CPU)', 'value': 7}], }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "Threads", "id": "kcpp_threads", "default": self.kcpp_threads, "check": {"value": "", 'check': "!="}, "tooltip": "Thread Count", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "Max Context Size", "id": "kcpp_ctxsize", "default": self.kcpp_ctxsize, "check": {"value": "", 'check': "!="}, "tooltip": "Max Context Size", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "BLAS Batch Size", "id": "kcpp_blasbatchsize", "default": self.kcpp_blasbatchsize, "check": {"value": "", 'check': "!="}, "tooltip": "BLAS Batch Size", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "GPU Layers", "id": "kcpp_gpulayers", "default": self.kcpp_gpulayers, "check": {"value": "", 'check': "!="}, "tooltip": "GPU Layers", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "Rope Scale", "id": "kcpp_ropescale", "default": self.kcpp_ropescale, "check": {"value": "", 'check': "!="}, "tooltip": "Rope Scale", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "text", "unit": "int", "label": "Rope Base", "id": "kcpp_ropebase", "default": self.kcpp_ropebase, "check": {"value": "", 'check': "!="}, "tooltip": "Rope Base", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "dropdown", "unit": "int", "label": "Smart Context", "id": "kcpp_smartcontext", "default": self.kcpp_smartcontext, "check": {"value": "", 'check': "!="}, 'multiple': False, "tooltip": "Smart Context", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "", 'children': [{'text': 'False', 'value': False}, {'text': 'True', 'value': True}], }) requested_parameters.append({ "uitype": "text", "unit": "text", "label": "GPU ID", "id": "kcpp_tensor_split_str", "default": "1", "check": {"value": "", 'check': "!="}, "tooltip": "Which GPU's do we use? For example:1 2", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "" }) requested_parameters.append({ "uitype": "dropdown", "unit": "int", "label": "Debug Mode", "id": "kcpp_debugmode", "default": self.kcpp_debugmode, "check": {"value": "", 'check': "!="}, 'multiple': False, "tooltip": "Debug Mode", "menu_path": "", "refresh_model_inputs": False, "extra_classes": "", 'children': [{'text': 'False', 'value': 0}, {'text': 'True', 'value': 1}], }) return requested_parameters def set_input_parameters(self, parameters): self.kcpp_threads = parameters["kcpp_threads"] self.kcpp_filename = parameters["kcpp_filename"] self.kcpp_ctxsize = parameters["kcpp_ctxsize"] self.kcpp_blasbatchsize = parameters["kcpp_blasbatchsize"] self.kcpp_gpulayers = parameters["kcpp_gpulayers"] self.kcpp_smartcontext = parameters["kcpp_smartcontext"] self.kcpp_ropescale = parameters["kcpp_ropescale"] self.kcpp_ropebase = parameters["kcpp_ropebase"] self.kcpp_debugmode = parameters["kcpp_debugmode"] self.kcpp_tensor_split_str = parameters["kcpp_tensor_split_str"] if self.kcpp_tensor_split_str and self.kcpp_tensor_split_str!="": splits = self.kcpp_tensor_split_str.split() self.kcpp_tensor_split = [] for s in splits: self.kcpp_tensor_split.append(int(s)) print(self.kcpp_tensor_split) accel = parameters["kcpp_accelerator"] if accel==0: self.kcpp_noblas = True elif accel==1: pass elif accel==2: self.kcpp_usecublas = ["normal"] elif accel==3: self.kcpp_useclblast = [0,0] elif accel==4: self.kcpp_useclblast = [1,0] elif accel==5: self.kcpp_useclblast = [0,1] elif accel==6: self.kcpp_noavx2 = True elif accel==7: self.kcpp_noavx2 = True self.kcpp_noblas = True self.kcpp_nommap = True pass def unload(self): print("Attemping to unload library") koboldcpp.unload_libs() global kcpp_backend_loaded kcpp_backend_loaded = False pass def _load(self, save_model: bool, initial_load: bool) -> None: global kcpp_backend_loaded self.tokenizer = self._get_tokenizer("gpt2") if not kcpp_backend_loaded: kcppargs = KcppArgsObject(model=self.kcpp_filename, model_param=self.kcpp_filename, port=5001, port_param=5001, host='', launch=False, lora=None, threads=self.kcpp_threads, blasthreads=self.kcpp_threads, psutil_set_threads=False, highpriority=False, contextsize=self.kcpp_ctxsize, blasbatchsize=self.kcpp_blasbatchsize, ropeconfig=[self.kcpp_ropescale, self.kcpp_ropebase], stream=False, smartcontext=self.kcpp_smartcontext, unbantokens=False, bantokens=None, usemirostat=None, forceversion=0, nommap=self.kcpp_nommap, usemlock=False, noavx2=self.kcpp_noavx2, debugmode=self.kcpp_debugmode, skiplauncher=True, hordeconfig=None, noblas=self.kcpp_noblas, useclblast=self.kcpp_useclblast, usecublas=self.kcpp_usecublas, gpulayers=self.kcpp_gpulayers, tensor_split=self.kcpp_tensor_split, config=None, onready='', multiuser=False, foreground=False) koboldcpp.main(kcppargs,False) #initialize library without enabling Lite http server kcpp_backend_loaded = True pass def _save_settings(self): pass def _raw_generate( self, prompt_tokens: Union[List[int], torch.Tensor], max_new: int, gen_settings: GenerationSettings, single_line: bool = False, batch_count: int = 1, seed: Optional[int] = None, **kwargs, ) -> GenerationResult: decoded_prompt = utils.decodenewlines(self.tokenizer.decode(prompt_tokens)) # Store context in memory to use it for comparison with generated content utils.koboldai_vars.lastctx = decoded_prompt genresult = koboldcpp.generate(decoded_prompt,max_new,utils.koboldai_vars.max_length, gen_settings.temp,int(gen_settings.top_k),gen_settings.top_a,gen_settings.top_p, gen_settings.typical,gen_settings.tfs,gen_settings.rep_pen,gen_settings.rep_pen_range, sampler_order=gen_settings.sampler_order,use_default_badwordsids=utils.koboldai_vars.use_default_badwordsids) outputs = [genresult] return GenerationResult( model=self, out_batches=np.array( [self.tokenizer.encode(x) for x in outputs] ), prompt=prompt_tokens, is_whole_generation=True, single_line=single_line, )