Upload model

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-afq4-0.uqff filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-afq6-0.uqff filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-afq8-0.uqff filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-q4k-0.uqff filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-q5k-0.uqff filter=lfs diff=lfs merge=lfs -text
+gemma3-270m-it-q8_0-0.uqff filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,29 @@
+---
+tags:
+  - uqff
+  - mistral.rs
+base_model: google/gemma-3-270m-it
+base_model_relation: quantized
+---
+
+<!-- Autogenerated from user input. -->
+
+# `google/gemma-3-270m-it`, UQFF quantization
+
+
+Run with [mistral.rs](https://github.com/EricLBuehler/mistral.rs). Documentation: [UQFF docs](https://github.com/EricLBuehler/mistral.rs/blob/master/docs/UQFF.md).
+
+1) **Flexible** 🌀: Multiple quantization formats in *one* file format with *one* framework to run them all.
+2) **Reliable** 🔒: Compatibility ensured with *embedded* and *checked* semantic versioning information from day 1.
+3) **Easy** 🤗: Download UQFF models *easily* and *quickly* from Hugging Face, or use a local file.
+3) **Customizable** 🛠️: Make and publish your own UQFF files in minutes.
+
+## Examples
+|Quantization type(s)|Example|
+|--|--|
+|AFQ4|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-afq4-0.uqff`|
+|AFQ6|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-afq6-0.uqff`|
+|AFQ8|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-afq8-0.uqff`|
+|Q4K|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-q4k-0.uqff`|
+|Q5K|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-q5k-0.uqff`|
+|Q8_0|`./mistralrs-server -i plain -m EricB/gemma-3-270m-it-UQFF -f gemma3-270m-it-q8_0-0.uqff`|

chat_template.jinja

@@ -0,0 +1,47 @@
+{{ bos_token }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {%- set first_user_prefix = messages[0]['content'] + '
+
+' -%}
+    {%- else -%}
+        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '
+
+' -%}
+    {%- endif -%}
+    {%- set loop_messages = messages[1:] -%}
+{%- else -%}
+    {%- set first_user_prefix = "" -%}
+    {%- set loop_messages = messages -%}
+{%- endif -%}
+{%- for message in loop_messages -%}
+    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
+        {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
+    {%- endif -%}
+    {%- if (message['role'] == 'assistant') -%}
+        {%- set role = "model" -%}
+    {%- else -%}
+        {%- set role = message['role'] -%}
+    {%- endif -%}
+    {{ '<start_of_turn>' + role + '
+' + (first_user_prefix if loop.first else "") }}
+    {%- if message['content'] is string -%}
+        {{ message['content'] | trim }}
+    {%- elif message['content'] is iterable -%}
+        {%- for item in message['content'] -%}
+            {%- if item['type'] == 'image' -%}
+                {{ '<start_of_image>' }}
+            {%- elif item['type'] == 'text' -%}
+                {{ item['text'] | trim }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{ raise_exception("Invalid content type") }}
+    {%- endif -%}
+    {{ '<end_of_turn>
+' }}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{'<start_of_turn>model
+'}}
+{%- endif -%}

config.json

@@ -0,0 +1,54 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 640,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 512,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.0.dev0",
+  "use_bidirectional_attention": false,
+  "use_cache": true,
+  "vocab_size": 262144
+}

gemma3-270m-it-afq4-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efbfd1857f765ba20eb8e9caea71d70cb3ad3d61af09cdaf530bd5cc967efca5
+size 150791748

gemma3-270m-it-afq6-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80cfb9780dd42a1b70e06503887292b53eca189b5478d69a1d54587fa7aeb99a
+size 217802436

gemma3-270m-it-afq8-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39626729221299d2cd69f39dc82e8e8e20d21e1fc1731b65e64e8f8e2fcf05be
+size 284812996

gemma3-270m-it-q4k-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0320f13b7f8191a4862e61895e7a21a5cba46f0051305978a9f21f357b2216e6
+size 165327286

gemma3-270m-it-q5k-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c07fe08bf3bbef800f478a602285c4e4fb59cb7a331150f0a6e2e154dcb2098f
+size 184291766

gemma3-270m-it-q8_0-0.uqff

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b8bf3633e7aaf9906ee14db9c4c27734d1f908cadd40473aee6e8b31d834459
+size 284807662

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.55.0.dev0"
+}

residual.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8645147b8d7349d9698d9c6692eb86788fa14f1972bae8019003734bc51e590
+size 335668672

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90a4672a51a9f582918ab48639e47c6cf053961c0bb6c6e6e32479f1ff123322
+size 33384604