add models

README.md

@@ -9,4 +9,4 @@ license: apache-2.0
 short_description: Qwen2.5-Instruct models on the browser with candle and wasm
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+- Check out the source code [here](https://github.com/ITHwang/llm-serving-wasm)

build/m.d.ts

@@ -0,0 +1,48 @@
+/* tslint:disable */
+/* eslint-disable */
+export class Model {
+  free(): void;
+  constructor(weights: Uint8Array, tokenizer: Uint8Array, config: Uint8Array, quantized: boolean);
+  init_with_prompt(prompt: string, temp: number, top_p: number, repeat_penalty: number, repeat_last_n: number, seed: bigint): string;
+  next_token(): string;
+}
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+  readonly memory: WebAssembly.Memory;
+  readonly __wbg_model_free: (a: number, b: number) => void;
+  readonly model_load: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number, number];
+  readonly model_init_with_prompt: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: bigint) => [number, number, number, number];
+  readonly model_next_token: (a: number) => [number, number, number, number];
+  readonly main: (a: number, b: number) => number;
+  readonly __wbindgen_exn_store: (a: number) => void;
+  readonly __externref_table_alloc: () => number;
+  readonly __wbindgen_export_2: WebAssembly.Table;
+  readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+  readonly __wbindgen_malloc: (a: number, b: number) => number;
+  readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+  readonly __externref_table_dealloc: (a: number) => void;
+  readonly __wbindgen_start: () => void;
+}
+
+export type SyncInitInput = BufferSource | WebAssembly.Module;
+/**
+* Instantiates the given `module`, which can either be bytes or
+* a precompiled `WebAssembly.Module`.
+*
+* @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
+*
+* @returns {InitOutput}
+*/
+export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
+
+/**
+* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+* for everything else, calls `WebAssembly.instantiate` directly.
+*
+* @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
+*
+* @returns {Promise<InitOutput>}
+*/
+export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;

build/m.js

@@ -0,0 +1,467 @@
+let wasm;
+
+function addToExternrefTable0(obj) {
+    const idx = wasm.__externref_table_alloc();
+    wasm.__wbindgen_export_2.set(idx, obj);
+    return idx;
+}
+
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        const idx = addToExternrefTable0(e);
+        wasm.__wbindgen_exn_store(idx);
+    }
+}
+
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
+
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+let cachedUint8ArrayMemory0 = null;
+
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8ArrayMemory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
+}
+
+let WASM_VECTOR_LEN = 0;
+
+const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
+
+const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
+    ? function (arg, view) {
+    return cachedTextEncoder.encodeInto(arg, view);
+}
+    : function (arg, view) {
+    const buf = cachedTextEncoder.encode(arg);
+    view.set(buf);
+    return {
+        read: arg.length,
+        written: buf.length
+    };
+});
+
+function passStringToWasm0(arg, malloc, realloc) {
+
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8ArrayMemory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
+        const ret = encodeString(arg, view);
+
+        offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+
+let cachedDataViewMemory0 = null;
+
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
+    }
+    return cachedDataViewMemory0;
+}
+
+function isLikeNone(x) {
+    return x === undefined || x === null;
+}
+
+function passArray8ToWasm0(arg, malloc) {
+    const ptr = malloc(arg.length * 1, 1) >>> 0;
+    getUint8ArrayMemory0().set(arg, ptr / 1);
+    WASM_VECTOR_LEN = arg.length;
+    return ptr;
+}
+
+function takeFromExternrefTable0(idx) {
+    const value = wasm.__wbindgen_export_2.get(idx);
+    wasm.__externref_table_dealloc(idx);
+    return value;
+}
+
+const ModelFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_model_free(ptr >>> 0, 1));
+
+export class Model {
+
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+        ModelFinalization.unregister(this);
+        return ptr;
+    }
+
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_model_free(ptr, 0);
+    }
+    /**
+     * @param {Uint8Array} weights
+     * @param {Uint8Array} tokenizer
+     * @param {Uint8Array} config
+     * @param {boolean} quantized
+     */
+    constructor(weights, tokenizer, config, quantized) {
+        const ptr0 = passArray8ToWasm0(weights, wasm.__wbindgen_malloc);
+        const len0 = WASM_VECTOR_LEN;
+        const ptr1 = passArray8ToWasm0(tokenizer, wasm.__wbindgen_malloc);
+        const len1 = WASM_VECTOR_LEN;
+        const ptr2 = passArray8ToWasm0(config, wasm.__wbindgen_malloc);
+        const len2 = WASM_VECTOR_LEN;
+        const ret = wasm.model_load(ptr0, len0, ptr1, len1, ptr2, len2, quantized);
+        if (ret[2]) {
+            throw takeFromExternrefTable0(ret[1]);
+        }
+        this.__wbg_ptr = ret[0] >>> 0;
+        ModelFinalization.register(this, this.__wbg_ptr, this);
+        return this;
+    }
+    /**
+     * @param {string} prompt
+     * @param {number} temp
+     * @param {number} top_p
+     * @param {number} repeat_penalty
+     * @param {number} repeat_last_n
+     * @param {bigint} seed
+     * @returns {string}
+     */
+    init_with_prompt(prompt, temp, top_p, repeat_penalty, repeat_last_n, seed) {
+        let deferred3_0;
+        let deferred3_1;
+        try {
+            const ptr0 = passStringToWasm0(prompt, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+            const len0 = WASM_VECTOR_LEN;
+            const ret = wasm.model_init_with_prompt(this.__wbg_ptr, ptr0, len0, temp, top_p, repeat_penalty, repeat_last_n, seed);
+            var ptr2 = ret[0];
+            var len2 = ret[1];
+            if (ret[3]) {
+                ptr2 = 0; len2 = 0;
+                throw takeFromExternrefTable0(ret[2]);
+            }
+            deferred3_0 = ptr2;
+            deferred3_1 = len2;
+            return getStringFromWasm0(ptr2, len2);
+        } finally {
+            wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
+        }
+    }
+    /**
+     * @returns {string}
+     */
+    next_token() {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const ret = wasm.model_next_token(this.__wbg_ptr);
+            var ptr1 = ret[0];
+            var len1 = ret[1];
+            if (ret[3]) {
+                ptr1 = 0; len1 = 0;
+                throw takeFromExternrefTable0(ret[2]);
+            }
+            deferred2_0 = ptr1;
+            deferred2_1 = len1;
+            return getStringFromWasm0(ptr1, len1);
+        } finally {
+            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+        }
+    }
+}
+
+async function __wbg_load(module, imports) {
+    if (typeof Response === 'function' && module instanceof Response) {
+        if (typeof WebAssembly.instantiateStreaming === 'function') {
+            try {
+                return await WebAssembly.instantiateStreaming(module, imports);
+
+            } catch (e) {
+                if (module.headers.get('Content-Type') != 'application/wasm') {
+                    console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
+
+                } else {
+                    throw e;
+                }
+            }
+        }
+
+        const bytes = await module.arrayBuffer();
+        return await WebAssembly.instantiate(bytes, imports);
+
+    } else {
+        const instance = await WebAssembly.instantiate(module, imports);
+
+        if (instance instanceof WebAssembly.Instance) {
+            return { instance, module };
+
+        } else {
+            return instance;
+        }
+    }
+}
+
+function __wbg_get_imports() {
+    const imports = {};
+    imports.wbg = {};
+    imports.wbg.__wbg_buffer_609cc3eee51ed158 = function(arg0) {
+        const ret = arg0.buffer;
+        return ret;
+    };
+    imports.wbg.__wbg_call_672a4d21634d4a24 = function() { return handleError(function (arg0, arg1) {
+        const ret = arg0.call(arg1);
+        return ret;
+    }, arguments) };
+    imports.wbg.__wbg_call_7cccdd69e0791ae2 = function() { return handleError(function (arg0, arg1, arg2) {
+        const ret = arg0.call(arg1, arg2);
+        return ret;
+    }, arguments) };
+    imports.wbg.__wbg_crypto_ed58b8e10a292839 = function(arg0) {
+        const ret = arg0.crypto;
+        return ret;
+    };
+    imports.wbg.__wbg_error_7534b8e9a36f1ab4 = function(arg0, arg1) {
+        let deferred0_0;
+        let deferred0_1;
+        try {
+            deferred0_0 = arg0;
+            deferred0_1 = arg1;
+            console.error(getStringFromWasm0(arg0, arg1));
+        } finally {
+            wasm.__wbindgen_free(deferred0_0, deferred0_1, 1);
+        }
+    };
+    imports.wbg.__wbg_getRandomValues_bcb4912f16000dc4 = function() { return handleError(function (arg0, arg1) {
+        arg0.getRandomValues(arg1);
+    }, arguments) };
+    imports.wbg.__wbg_log_55f8292edd5cb819 = function(arg0, arg1) {
+        console.log(getStringFromWasm0(arg0, arg1));
+    };
+    imports.wbg.__wbg_msCrypto_0a36e2ec3a343d26 = function(arg0) {
+        const ret = arg0.msCrypto;
+        return ret;
+    };
+    imports.wbg.__wbg_new_8a6f238a6ece86ea = function() {
+        const ret = new Error();
+        return ret;
+    };
+    imports.wbg.__wbg_new_a12002a7f91c75be = function(arg0) {
+        const ret = new Uint8Array(arg0);
+        return ret;
+    };
+    imports.wbg.__wbg_newnoargs_105ed471475aaf50 = function(arg0, arg1) {
+        const ret = new Function(getStringFromWasm0(arg0, arg1));
+        return ret;
+    };
+    imports.wbg.__wbg_newwithbyteoffsetandlength_d97e637ebe145a9a = function(arg0, arg1, arg2) {
+        const ret = new Uint8Array(arg0, arg1 >>> 0, arg2 >>> 0);
+        return ret;
+    };
+    imports.wbg.__wbg_newwithlength_a381634e90c276d4 = function(arg0) {
+        const ret = new Uint8Array(arg0 >>> 0);
+        return ret;
+    };
+    imports.wbg.__wbg_node_02999533c4ea02e3 = function(arg0) {
+        const ret = arg0.node;
+        return ret;
+    };
+    imports.wbg.__wbg_now_807e54c39636c349 = function() {
+        const ret = Date.now();
+        return ret;
+    };
+    imports.wbg.__wbg_process_5c1d670bc53614b8 = function(arg0) {
+        const ret = arg0.process;
+        return ret;
+    };
+    imports.wbg.__wbg_randomFillSync_ab2cfe79ebbf2740 = function() { return handleError(function (arg0, arg1) {
+        arg0.randomFillSync(arg1);
+    }, arguments) };
+    imports.wbg.__wbg_require_79b1e9274cde3c87 = function() { return handleError(function () {
+        const ret = module.require;
+        return ret;
+    }, arguments) };
+    imports.wbg.__wbg_set_65595bdd868b3009 = function(arg0, arg1, arg2) {
+        arg0.set(arg1, arg2 >>> 0);
+    };
+    imports.wbg.__wbg_stack_0ed75d68575b0f3c = function(arg0, arg1) {
+        const ret = arg1.stack;
+        const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len1 = WASM_VECTOR_LEN;
+        getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+        getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+    };
+    imports.wbg.__wbg_static_accessor_GLOBAL_88a902d13a557d07 = function() {
+        const ret = typeof global === 'undefined' ? null : global;
+        return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
+    };
+    imports.wbg.__wbg_static_accessor_GLOBAL_THIS_56578be7e9f832b0 = function() {
+        const ret = typeof globalThis === 'undefined' ? null : globalThis;
+        return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
+    };
+    imports.wbg.__wbg_static_accessor_SELF_37c5d418e4bf5819 = function() {
+        const ret = typeof self === 'undefined' ? null : self;
+        return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
+    };
+    imports.wbg.__wbg_static_accessor_WINDOW_5de37043a91a9c40 = function() {
+        const ret = typeof window === 'undefined' ? null : window;
+        return isLikeNone(ret) ? 0 : addToExternrefTable0(ret);
+    };
+    imports.wbg.__wbg_subarray_aa9065fa9dc5df96 = function(arg0, arg1, arg2) {
+        const ret = arg0.subarray(arg1 >>> 0, arg2 >>> 0);
+        return ret;
+    };
+    imports.wbg.__wbg_versions_c71aa1626a93e0a1 = function(arg0) {
+        const ret = arg0.versions;
+        return ret;
+    };
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
+        return ret;
+    };
+    imports.wbg.__wbindgen_init_externref_table = function() {
+        const table = wasm.__wbindgen_export_2;
+        const offset = table.grow(4);
+        table.set(0, undefined);
+        table.set(offset + 0, undefined);
+        table.set(offset + 1, null);
+        table.set(offset + 2, true);
+        table.set(offset + 3, false);
+        ;
+    };
+    imports.wbg.__wbindgen_is_function = function(arg0) {
+        const ret = typeof(arg0) === 'function';
+        return ret;
+    };
+    imports.wbg.__wbindgen_is_object = function(arg0) {
+        const val = arg0;
+        const ret = typeof(val) === 'object' && val !== null;
+        return ret;
+    };
+    imports.wbg.__wbindgen_is_string = function(arg0) {
+        const ret = typeof(arg0) === 'string';
+        return ret;
+    };
+    imports.wbg.__wbindgen_is_undefined = function(arg0) {
+        const ret = arg0 === undefined;
+        return ret;
+    };
+    imports.wbg.__wbindgen_memory = function() {
+        const ret = wasm.memory;
+        return ret;
+    };
+    imports.wbg.__wbindgen_string_new = function(arg0, arg1) {
+        const ret = getStringFromWasm0(arg0, arg1);
+        return ret;
+    };
+    imports.wbg.__wbindgen_throw = function(arg0, arg1) {
+        throw new Error(getStringFromWasm0(arg0, arg1));
+    };
+
+    return imports;
+}
+
+function __wbg_init_memory(imports, memory) {
+
+}
+
+function __wbg_finalize_init(instance, module) {
+    wasm = instance.exports;
+    __wbg_init.__wbindgen_wasm_module = module;
+    cachedDataViewMemory0 = null;
+    cachedUint8ArrayMemory0 = null;
+
+
+    wasm.__wbindgen_start();
+    return wasm;
+}
+
+function initSync(module) {
+    if (wasm !== undefined) return wasm;
+
+
+    if (typeof module !== 'undefined') {
+        if (Object.getPrototypeOf(module) === Object.prototype) {
+            ({module} = module)
+        } else {
+            console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
+        }
+    }
+
+    const imports = __wbg_get_imports();
+
+    __wbg_init_memory(imports);
+
+    if (!(module instanceof WebAssembly.Module)) {
+        module = new WebAssembly.Module(module);
+    }
+
+    const instance = new WebAssembly.Instance(module, imports);
+
+    return __wbg_finalize_init(instance, module);
+}
+
+async function __wbg_init(module_or_path) {
+    if (wasm !== undefined) return wasm;
+
+
+    if (typeof module_or_path !== 'undefined') {
+        if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
+            ({module_or_path} = module_or_path)
+        } else {
+            console.warn('using deprecated parameters for the initialization function; pass a single object instead')
+        }
+    }
+
+    if (typeof module_or_path === 'undefined') {
+        module_or_path = new URL('m_bg.wasm', import.meta.url);
+    }
+    const imports = __wbg_get_imports();
+
+    if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
+        module_or_path = fetch(module_or_path);
+    }
+
+    __wbg_init_memory(imports);
+
+    const { instance, module } = await __wbg_load(await module_or_path, imports);
+
+    return __wbg_finalize_init(instance, module);
+}
+
+export { initSync };
+export default __wbg_init;

build/m_bg.wasm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f824eb6d35931342c341423cd0af0271fabc129b971828feb9d83aaeba132770
+size 4985301

build/m_bg.wasm.d.ts

@@ -0,0 +1,16 @@
+/* tslint:disable */
+/* eslint-disable */
+export const memory: WebAssembly.Memory;
+export const __wbg_model_free: (a: number, b: number) => void;
+export const model_load: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number, number];
+export const model_init_with_prompt: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: bigint) => [number, number, number, number];
+export const model_next_token: (a: number) => [number, number, number, number];
+export const main: (a: number, b: number) => number;
+export const __wbindgen_exn_store: (a: number) => void;
+export const __externref_table_alloc: () => number;
+export const __wbindgen_export_2: WebAssembly.Table;
+export const __wbindgen_free: (a: number, b: number, c: number) => void;
+export const __wbindgen_malloc: (a: number, b: number) => number;
+export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+export const __externref_table_dealloc: (a: number) => void;
+export const __wbindgen_start: () => void;

index.html

@@ -1,19 +1,359 @@
-<!doctype html>
 <html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
+
+<head>
+  <meta content="text/html;charset=utf-8" http-equiv="Content-Type" />
+  <title>Candle Qwen2.5 Instruct Rust/WASM</title>
+</head>
+
+<body></body>
+
 </html>
+
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <link rel="stylesheet"
+    href="https://cdn.jsdelivr.net/gh/highlightjs/[email protected]/build/styles/default.min.css" />
+  <style>
+    @import url("https://fonts.googleapis.com/css2?family=Source+Code+Pro:wght@200;300;400&family=Source+Sans+3:wght@100;200;300;400;500;600;700;800;900&display=swap");
+
+    html,
+    body {
+      font-family: "Source Sans 3", sans-serif;
+    }
+
+    code,
+    output,
+    select,
+    pre {
+      font-family: "Source Code Pro", monospace;
+    }
+  </style>
+  <style type="text/tailwindcss">
+    .link {
+        @apply underline hover:text-blue-500 hover:no-underline;
+      }
+    </style>
+  <script src="https://cdn.tailwindcss.com"></script>
+  <script type="module">
+    import snarkdown from "https://cdn.skypack.dev/snarkdown";
+    import hljs from "https://cdn.skypack.dev/highlight.js";
+    // models base url
+    const MODELS = {
+      qwen_25_05_instruct_q4k: {
+        model_name: "qwen2.5-0.5b-instruct-q4_k_m.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "491 MB",
+      },
+      qwen_25_05_instruct_q80: {
+        model_name: "qwen2.5-0.5b-instruct-q8_0.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q8_0.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "676 MB",
+      },
+      qwen_25_05_instruct_f16: {
+        model_name: "qwen2.5-0.5b-instruct-f16.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-f16.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "1.27 GB",
+      },
+      qwen_25_15_instruct_q4k: {
+        model_name: "qwen2.5-1.5b-instruct-q4_k_m.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q4_k_m.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "1.12 GB",
+      },
+      qwen_25_15_instruct_q80: {
+        model_name: "qwen2.5-1.5b-instruct-q8_0.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q8_0.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "1.89 GB",
+      },
+      qwen_25_15_instruct_f16: {
+        model_name: "qwen2.5-1.5b-instruct-f16.gguf",
+        model_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-f16.gguf",
+        tokenizer_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/tokenizer.json",
+        config_url: "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct/resolve/main/config.json",
+        quantized: true,
+        seq_len: 2048,
+        size: "3.56 GB",
+      },
+    };
+
+    const qwenWorker = new Worker("./qwenWorker.js", {
+      type: "module",
+    });
+    async function generateSequence(controller) {
+      const getValue = (id) => document.querySelector(`#${id}`).value;
+
+      const modelID = getValue("model");
+      const model = MODELS[modelID];
+      const weightsURL = model.model_url
+      const tokenizerURL = model.tokenizer_url;
+      const configURL = model.config_url;
+
+      const prompt = getValue("prompt").trim();
+      const temperature = getValue("temperature");
+      const topP = getValue("top-p");
+      const repeatPenalty = getValue("repeat_penalty");
+      const seed = getValue("seed");
+      const maxSeqLen = getValue("max-seq");
+
+      function updateStatus(data) {
+        const outStatus = document.querySelector("#output-status");
+        const outGen = document.querySelector("#output-generation");
+        const outCounter = document.querySelector("#output-counter");
+
+        switch (data.status) {
+          case "loading":
+            outStatus.hidden = false;
+            outStatus.textContent = data.message;
+            outGen.hidden = true;
+            outCounter.hidden = true;
+            break;
+          case "generating":
+            const { message, prompt, sentence, tokensSec, totalTime } = data;
+            outStatus.hidden = true;
+            outCounter.hidden = false;
+            outGen.hidden = false;
+            outGen.innerHTML = snarkdown(sentence);
+            outCounter.innerHTML = `${(totalTime / 1000).toFixed(
+              2
+            )}s (${tokensSec.toFixed(2)} tok/s)`;
+            hljs.highlightAll();
+            break;
+          case "complete":
+            outStatus.hidden = true;
+            outGen.hidden = false;
+            break;
+        }
+      }
+
+      return new Promise((resolve, reject) => {
+        qwenWorker.postMessage({
+          weightsURL,
+          modelID,
+          tokenizerURL,
+          configURL,
+          quantized: model.quantized,
+          prompt,
+          temp: temperature,
+          top_p: topP,
+          repeatPenalty,
+          seed: seed,
+          maxSeqLen,
+          command: "start",
+        });
+
+        const handleAbort = () => {
+          qwenWorker.postMessage({ command: "abort" });
+        };
+        const handleMessage = (event) => {
+          const { status, error, message, prompt, sentence } = event.data;
+          if (status) updateStatus(event.data);
+          if (error) {
+            qwenWorker.removeEventListener("message", handleMessage);
+            reject(new Error(`ERROR: ${error}. Please check available RAM in your local machine.`));
+          }
+          if (status === "aborted") {
+            qwenWorker.removeEventListener("message", handleMessage);
+            resolve(event.data);
+          }
+          if (status === "complete") {
+            qwenWorker.removeEventListener("message", handleMessage);
+            resolve(event.data);
+          }
+        };
+
+        controller.signal.addEventListener("abort", handleAbort);
+        qwenWorker.addEventListener("message", handleMessage);
+      });
+    }
+
+    const form = document.querySelector("#form");
+    const prompt = document.querySelector("#prompt");
+    const clearBtn = document.querySelector("#clear-btn");
+    const runBtn = document.querySelector("#run");
+    const modelSelect = document.querySelector("#model");
+    let runController = new AbortController();
+    let isRunning = false;
+
+    document.addEventListener("DOMContentLoaded", () => {
+      for (const [id, model] of Object.entries(MODELS)) {
+        const option = document.createElement("option");
+        option.value = id;
+        option.innerText = `${model.model_name} (${model.size})`;
+        modelSelect.appendChild(option);
+      }
+      const query = new URLSearchParams(window.location.search);
+      const modelID = query.get("model");
+      if (modelID) {
+        modelSelect.value = modelID;
+      } else {
+        modelSelect.value = "qwen_25_05_instruct_q4k";
+      }
+    });
+
+    modelSelect.addEventListener("change", (e) => {
+      const query = new URLSearchParams(window.location.search);
+      query.set("model", e.target.value);
+      window.history.replaceState(
+        {},
+        "",
+        `${window.location.pathname}?${query}`
+      );
+      window.parent.postMessage({ queryString: "?" + query }, "*");
+      const model = MODELS[e.target.value];
+      document.querySelector("#max-seq").max = model.seq_len;
+      document.querySelector("#max-seq").nextElementSibling.value = 512;
+    });
+
+    form.addEventListener("submit", async (e) => {
+      e.preventDefault();
+      if (isRunning) {
+        stopRunning();
+      } else {
+        startRunning();
+        await generateSequence(runController);
+        stopRunning();
+      }
+    });
+
+    function startRunning() {
+      isRunning = true;
+      runBtn.textContent = "Stop";
+    }
+
+    function stopRunning() {
+      runController.abort();
+      runController = new AbortController();
+      runBtn.textContent = "Run";
+      isRunning = false;
+    }
+    clearBtn.addEventListener("click", (e) => {
+      e.preventDefault();
+      prompt.value = "";
+      clearBtn.classList.add("invisible");
+      runBtn.disabled = true;
+      stopRunning();
+    });
+    prompt.addEventListener("input", (e) => {
+      runBtn.disabled = false;
+      if (e.target.value.length > 0) {
+        clearBtn.classList.remove("invisible");
+      } else {
+        clearBtn.classList.add("invisible");
+      }
+    });
+  </script>
+</head>
+
+<body class="container max-w-4xl mx-auto p-4 text-gray-800">
+  <main class="grid grid-cols-1 gap-8 relative">
+    <span class="absolute text-5xl -ml-[1em]"> 🕯️ </span>
+    <div>
+      <h1 class="text-5xl font-bold">Candle Qwen2.5 Instruct</h1>
+      <h2 class="text-2xl font-bold">Rust/WASM Demo</h2>
+      <p class="max-w-lg">
+        The
+        <a href="https://huggingface.co/collections/Qwen/qwen25-66e81a666513e518adb90d9e" class="link"
+          target="_blank">Qwen2.5</a>
+        models are state-of-the-art multi-language models ranging from 0.5B to 72B parameters.
+        From the benefits of WebAssembly, you can try some of them without LLM servers.
+        For more details of these models, please refer to the
+        <a href="https://arxiv.org/pdf/2412.15115" class="link" target="_blank">technical report</a>.
+      </p>
+    </div>
+    <div>
+      <p class="text-xs italic max-w-lg">
+        <b>Note:</b>
+        When first run, the app will download and cache the model, which could take a few minutes.
+      </p>
+    </div>
+    <div>
+      <label for="model" class="font-medium">Models Options: </label>
+      <select id="model" class="border-2 border-gray-500 rounded-md font-light"></select>
+    </div>
+    <form id="form" class="flex text-normal px-1 py-1 border border-gray-700 rounded-md items-center">
+      <input type="submit" hidden />
+      <textarea type="text" id="prompt" class="font-light text-lg w-full px-3 py-2 mx-1 resize-none outline-none"
+        oninput="this.style.height = 0;this.style.height = this.scrollHeight + 'px'"
+        placeholder="Add your prompt here..."></textarea>
+      <button id="clear-btn">
+        <svg fill="none" xmlns="http://www.w3.org/2000/svg" width="40" viewBox="0 0 70 40">
+          <path opacity=".5" d="M39 .2v40.2" stroke="#1F2937" />
+          <path d="M1.5 11.5 19 29.1m0-17.6L1.5 29.1" opacity=".5" stroke="#1F2937" stroke-width="2" />
+        </svg>
+      </button>
+      <button id="run"
+        class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-2 w-16 rounded disabled:bg-gray-300 disabled:cursor-not-allowed">
+        Run
+      </button>
+    </form>
+    <details>
+      <summary class="font-medium cursor-pointer">Advanced Options</summary>
+
+      <div class="grid grid-cols-3 max-w-md items-center gap-3 py-3">
+        <label class="text-sm font-medium" for="max-seq">Maximum length
+        </label>
+        <input type="range" id="max-seq" name="max-seq" min="1" max="2048" step="1" value="512"
+          oninput="this.nextElementSibling.value = Number(this.value)" />
+        <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+          512</output>
+        <label class="text-sm font-medium" for="temperature">Temperature</label>
+        <input type="range" id="temperature" name="temperature" min="0" max="2" step="0.01" value="0.00"
+          oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+        <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+          0.00</output>
+        <label class="text-sm font-medium" for="top-p">Top-p</label>
+        <input type="range" id="top-p" name="top-p" min="0" max="1" step="0.01" value="0.90"
+          oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+        <output class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">
+          0.90</output>
+        <label class="text-sm font-medium" for="repeat_penalty">Repeat Penalty</label>
+        <input type="range" id="repeat_penalty" name="repeat_penalty" min="1" max="2" step="0.01" value="1.10"
+          oninput="this.nextElementSibling.value = Number(this.value).toFixed(2)" />
+        <output
+          class="text-xs w-[50px] text-center font-light px-1 py-1 border border-gray-700 rounded-md">1.10</output>
+        <label class="text-sm font-medium" for="seed">Seed</label>
+        <input type="number" id="seed" name="seed" value="42"
+          class="font-light border border-gray-700 text-right rounded-md p-2" />
+        <button id="run"
+          onclick="document.querySelector('#seed').value = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)"
+          class="bg-gray-700 hover:bg-gray-800 text-white font-normal py-1 w-[50px] rounded disabled:bg-gray-300 disabled:cursor-not-allowed text-sm">
+          Rand
+        </button>
+      </div>
+    </details>
+
+    <div>
+      <h3 class="font-medium">Generation:</h3>
+      <div class="min-h-[250px] bg-slate-100 text-gray-500 p-4 rounded-md flex flex-col gap-2">
+        <div id="output-counter" hidden class="ml-auto font-semibold grid-rows-1"></div>
+        <p hidden id="output-generation" class="grid-rows-2 text-lg"></p>
+        <span id="output-status" class="m-auto font-light">No output yet</span>
+      </div>
+    </div>
+  </main>
+</body>
+
+</html>

qwenWorker.js

@@ -0,0 +1,162 @@
+import init, { Model } from "./build/m.js";
+
+async function fetchArrayBuffer(url) {
+  const cacheName = "qwen-instruct-candle-cache";
+  const cache = await caches.open(cacheName);
+  const cachedResponse = await cache.match(url);
+  if (cachedResponse) {
+    console.log("load from cache");
+    const data = await cachedResponse.arrayBuffer();
+    return new Uint8Array(data);
+  }
+
+  console.log("load from huggingface");
+  const res = await fetch(url, { cache: "force-cache" });
+  cache.put(url, res.clone());
+  return new Uint8Array(await res.arrayBuffer());
+}
+async function concatenateArrayBuffers(urls) {
+  const arrayBuffers = await Promise.all(urls.map(url => fetchArrayBuffer(url)));
+
+  let totalLength = arrayBuffers.reduce((acc, arrayBuffer) => acc + arrayBuffer.byteLength, 0);
+  let concatenatedBuffer = new Uint8Array(totalLength);
+
+  let offset = 0;
+  arrayBuffers.forEach(buffer => {
+    concatenatedBuffer.set(new Uint8Array(buffer), offset);
+    offset += buffer.byteLength;
+  });
+  return concatenatedBuffer;
+}
+
+class Qwen {
+  static instance = {};
+
+  static async getInstance(
+    weightsURL,
+    modelID,
+    tokenizerURL,
+    configURL,
+    quantized
+  ) {
+    // load individual modelID only once
+    if (!this.instance[modelID]) {
+      await init();
+
+      self.postMessage({ status: "loading", message: "Loading Model" });
+      const [weightsArrayU8, tokenizerArrayU8, configArrayU8] =
+        await Promise.all([
+          weightsURL instanceof Array ? concatenateArrayBuffers(weightsURL) : fetchArrayBuffer(weightsURL),
+          fetchArrayBuffer(tokenizerURL),
+          fetchArrayBuffer(configURL),
+        ]);
+
+      this.instance[modelID] = new Model(
+        weightsArrayU8,
+        tokenizerArrayU8,
+        configArrayU8,
+        quantized
+      );
+    }
+    return this.instance[modelID];
+  }
+}
+
+let controller = null;
+self.addEventListener("message", (event) => {
+  if (event.data.command === "start") {
+    controller = new AbortController();
+    generate(event.data);
+  } else if (event.data.command === "abort") {
+    controller.abort();
+  }
+});
+
+async function generate(data) {
+  const {
+    weightsURL,
+    modelID,
+    tokenizerURL,
+    configURL,
+    quantized,
+    prompt,
+    temp,
+    top_p,
+    repeatPenalty,
+    seed,
+    maxSeqLen,
+  } = data;
+  try {
+    self.postMessage({ status: "loading", message: "Starting Qwen" });
+    const model = await Qwen.getInstance(
+      weightsURL,
+      modelID,
+      tokenizerURL,
+      configURL,
+      quantized
+    );
+
+    self.postMessage({ status: "loading", message: "Initializing model" });
+    const promptString = `<|im_start|>user\n${prompt}<|im_end|>\n<|im_start|>assistant\n`;
+    const firstToken = model.init_with_prompt(
+      promptString,
+      temp,
+      top_p,
+      repeatPenalty,
+      64,
+      BigInt(seed)
+    );
+
+    let sentence = firstToken;
+    let maxTokens = maxSeqLen;
+    let startTime = performance.now();
+    let tokensCount = 0;
+    while (tokensCount < maxTokens) {
+      await new Promise(async (resolve) => {
+        if (controller && controller.signal.aborted) {
+          self.postMessage({
+            status: "aborted",
+            message: "Aborted",
+            output: sentence,
+          });
+          return;
+        }
+
+        const token = await model.next_token();
+
+        if (token === "<|im_end|>") {
+          self.postMessage({
+            status: "complete",
+            message: "complete",
+            output: sentence,
+          });
+          return;
+        }
+
+        const tokensSec =
+          ((tokensCount + 1) / (performance.now() - startTime)) * 1000;
+
+        sentence += token;
+
+        self.postMessage({
+          status: "generating",
+          message: "Generating token",
+          token: token,
+          sentence: sentence,
+          totalTime: performance.now() - startTime,
+          tokensSec,
+          prompt: prompt,
+        });
+        setTimeout(resolve, 0);
+      });
+      tokensCount++;
+    }
+    self.postMessage({
+      status: "complete",
+      message: "complete",
+      output: prompt + sentence,
+    });
+  } catch (e) {
+    self.postMessage({ error: e });
+  }
+}

style.css

@@ -1,28 +0,0 @@
-body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
-}
-
-h1 {
-	font-size: 16px;
-	margin-top: 0;
-}
-
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
-}
-
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
-}
-
-.card p:last-child {
-	margin-bottom: 0;
-}