Add files using upload-large-folder tool

README.md

@@ -0,0 +1,621 @@
+---
+tags:
+- gptq
+- quantization
+- 4bit
+- confidentialmind
+- text-generation
+- apache2.0
+- mistral-small-24b
+---
+# 🔥 Quantized Model: SmolLM-135M_gptq_g32_4bit 🔥
+
+This is a 4-bit quantized version of [HuggingFaceTB/SmolLM-135M](https://huggingface.co/HuggingFaceTB/SmolLM-135M) model, quantized by [ConfidentialMind.com](https://www.confidentialmind.com) 🤖✨  
+It leverages the open-source GPTQModel quantization to achieve 4-bit precision with a group size of 128 resulting in a 
+smaller, 
+faster model with minimal performance degradation.
+
+Ran on a single NVIDIA A100 GPU with 80GB of VRAM.
+
+*Note* `batch_size` is set quite high as the model is small, you may need to adjust this to your GPU VRAM.
+
+## Model Details
+- **Original Model:** [HuggingFaceTB/SmolLM-135M](https://huggingface.co/HuggingFaceTB/SmolLM-135M)
+- **Quantized Model:** SmolLM-135M_gptq_g32_4bit (this repository)
+- **Quantization Method:** GPTQ (4-bit, group size 128)
+- **Quantization Library:** [GPTQModel](https://github.com/ModelCloud/GPTQModel/tree/main)
+- **Calibration Dataset:** wikitext/wikitext-2-raw-v1 (using 512 samples with seq len 4096)
+- **Quantized by:** [ConfidentialMind.com](https://www.confidentialmind.com)
+
+## Usage
+
+```python
+from gptqmodel import GPTQModel
+from transformers import AutoTokenizer
+
+# Use the local directory or JustJaro/SmolLM-135M_gptq_g32_4bit after upload
+quantized_model_id = "/home/jarouljanov/models/quantized/SmolLM-135M_gptq_g32_4bit"  # or "JustJaro/SmolLM-135M_gptq_g32_4bit"
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_id)
+model = GPTQModel.load(quantized_model_id, device="cuda:0")  # or "cpu"
+
+input_text = "This is a test prompt"
+inputs = tokenizer(input_text, return_tensors="pt").to("cuda:0")
+outputs = model.generate(**inputs)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+## Package Versions and Installation Instructions
+
+See pyproject.toml for the exact UV project file. See the [GPTQModel](
+https://github.com/ModelCloud/GPTQModel/tree/main) repo for more details. on how to install the package.
+
+Use the provided pyproject.toml:
+
+```bash
+uv venv
+source venv/bin/activate
+uv sync
+```
+
+### Environment Variables
+
+```bash
+HF_TOKEN=<YOUR_HF_TOKEN>
+TOKENIZERS_PARALLELISM="true"
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+```
+
+## Quantization Script
+Below is the exact quantize.py script used to generate this model (with the exact versions of the dependencies):
+
+```python
+#!/usr/bin/env python3
+"""
+This script loads a source Hugging Face model and a calibration dataset,
+quantizes the model using GPTQModel (with 4-bit precision and group size 128),
+saves the quantized model using the Transformers API with safetensors (safe serialization)
+under ~/models/quantized/, and then creates/updates a Hugging Face repository (with the
+_gptq_g128_4bit suffix) by uploading the model, tokenizer, and an auto-generated README.md.
+
+Usage example:
+    python quantize.py --source-model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
+                       --calibration-dataset wikitext/wikitext-2-raw-v1 \
+                       --seq-len 1024 --nsamples 256 --hf-token <YOUR_HF_TOKEN>
+"""
+
+import os
+import shutil
+import subprocess
+import math
+from enum import Enum
+from pathlib import Path
+from typing import List, Union
+
+import torch
+import typer
+from datasets import load_dataset
+from dotenv import load_dotenv, find_dotenv
+from gptqmodel import GPTQModel, QuantizeConfig
+from huggingface_hub import HfApi
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
+
+load_dotenv(find_dotenv())
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+app = typer.Typer()
+
+
+class GroupSize(str, Enum):
+    accurate: int = 32
+    balanced: int = 64
+    fast: int = 128
+
+
+def get_text_from_example(example: dict) -> str:
+    """
+    Returns text from a dataset example.
+    If the example contains a "text" field, and it is nonempty, that text is used.
+    Otherwise, if it has a "messages" field (a list of dicts with a "content" key),
+    the function returns the concatenation of all non-empty message contents.
+    """
+    if "text" in example and example["text"]:
+        return example["text"]
+    elif "messages" in example:
+        contents = [msg.get("content", "").strip() for msg in example["messages"]]
+        return " ".join([s for s in contents if s])
+    else:
+        return ""
+
+
+def get_calibration_dataset(
+    tokenizer: PreTrainedTokenizerBase,
+    nsamples: int,
+    seqlen: int,
+    calibration_dataset: str
+) -> List[dict]:
+    """
+    Loads a calibration dataset from the Hugging Face Hub (or from a local file).
+    It accepts datasets with a single "text" field (like wikitext)
+    or with a "messages" field (as in the Neural Magic LLM Compression Calibration dataset).
+    Only examples whose extracted text length is at least 'seqlen' are kept.
+    Each chosen example is tokenized (with truncation up to 'seqlen') and returned as a dict.
+    """
+    ds = None
+    try:
+        # Attempt to load from HF Hub.
+        try:
+            if "/" in calibration_dataset:
+                parts = calibration_dataset.split("/", 1)
+                ds = load_dataset(parts[0], parts[1], split="train")
+            else:
+                ds = load_dataset(calibration_dataset, split="train")
+        except Exception as e:
+            print(f"Error loading dataset '{calibration_dataset}' via load_dataset: {e}")
+            ds = load_dataset(calibration_dataset, split="train")
+            print(f"Loaded calibration dataset from full remote path {calibration_dataset}.")
+
+    except Exception as e:
+        print(f"Error loading dataset '{calibration_dataset}' via load_dataset: {e}")
+        # Fallback: if the supplied calibration_dataset is a local path, try to load it as JSON-lines.
+        if os.path.exists(calibration_dataset):
+            try:
+                ds = load_dataset("json", data_files=calibration_dataset, split="train")
+                print(f"Loaded calibration dataset from local file {calibration_dataset}.")
+            except Exception as e2:
+                print(f"Error loading local json dataset from '{calibration_dataset}': {e2}")
+                return []
+        else:
+            return []
+
+    print(f"Dataset features: {ds.features}")
+
+    # Filter examples that have at least 80% 'seqlen' of extracted text (wikitext-2-raw-v1 dataset has short examples).
+    ds = ds.filter(lambda x: len(get_text_from_example(x)) <= int(seqlen * 0.8))
+    sample_range = min(nsamples, len(ds))
+    calibration_data = []
+    for i in range(sample_range):
+        example = ds[i]
+        text = get_text_from_example(example)
+        tokenized = tokenizer(text, truncation=True, max_length=seqlen, return_tensors="pt")
+        tokenized = {k: v.squeeze(0) for k, v in tokenized.items()}
+        calibration_data.append(tokenized)
+    return calibration_data
+
+
+def calculate_perplexity_manual(model, tokenizer, dataset_name="wikitext", dataset_config="wikitext-2-raw-v1", 
+                               split="test", max_samples=100, max_length=512) -> Union[float, str]:
+    """
+    Calculate perplexity manually using a dataset.
+    Based on the research from GPTQModel documentation.
+    """
+    try:
+        # Load test dataset
+        if "/" in dataset_name:
+            dataset = load_dataset(dataset_name, split=split)
+        else:
+            dataset = load_dataset(dataset_name, dataset_config, split=split)
+        
+        # Filter out empty texts
+        texts = [text for text in dataset["text"] if text.strip()]
+        
+        # Limit samples for efficiency
+        texts = texts[:max_samples]
+        
+        typer.echo(f"Calculating perplexity on {len(texts)} samples from {dataset_name}...")
+        
+        model.model.eval()
+        total_loss = 0.0
+        total_tokens = 0
+        
+        with torch.no_grad():
+            for i, text in enumerate(texts):
+                if i % 20 == 0:
+                    typer.echo(f"Processing sample {i+1}/{len(texts)}")
+                
+                # Tokenize the text
+                inputs = tokenizer(
+                    text,
+                    return_tensors="pt",
+                    truncation=True,
+                    max_length=max_length,
+                    padding=False
+                )
+                
+                input_ids = inputs.input_ids.to(model.model.device)
+                
+                # Skip if too short
+                if input_ids.size(1) < 2:
+                    continue
+                
+                # Calculate loss
+                outputs = model.model(input_ids, labels=input_ids)
+                loss = outputs.loss
+                
+                # Accumulate loss and token count
+                total_loss += loss.item() * input_ids.size(1)
+                total_tokens += input_ids.size(1)
+        
+        if total_tokens == 0:
+            return "N/A (No valid tokens processed)"
+        
+        # Calculate perplexity
+        avg_loss = total_loss / total_tokens
+        perplexity = math.exp(avg_loss)
+        
+        return perplexity
+        
+    except Exception as e:
+        typer.echo(f"Error calculating perplexity manually: {e}")
+        return f"N/A (Error: {str(e)})"
+
+
+def calculate_perplexity_lm_eval(model, tokenizer) -> Union[float, str]:
+    """
+    Calculate perplexity using lm-eval framework if available.
+    Based on GPTQModel documentation research.
+    """
+    try:
+        from gptqmodel.utils.eval import EVAL
+        
+        # Try to use GPTQModel's built-in evaluation
+        typer.echo("Attempting to calculate perplexity using GPTQModel.eval...")
+        
+        # Create a temporary directory to save the model for evaluation
+        temp_model_path = "/tmp/temp_gptq_model"
+        os.makedirs(temp_model_path, exist_ok=True)
+        
+        model.save_pretrained(temp_model_path)
+        tokenizer.save_pretrained(temp_model_path)
+        
+        # Use GPTQModel.eval with lm-eval framework
+        results = GPTQModel.eval(
+            temp_model_path,
+            framework=EVAL.LM_EVAL,
+            tasks=["wikitext"],
+            output_file=None
+        )
+        
+        # Clean up temporary directory
+        shutil.rmtree(temp_model_path, ignore_errors=True)
+        
+        # Extract perplexity from results
+        if "wikitext" in results.get("results", {}):
+            wikitext_results = results["results"]["wikitext"]
+            if "perplexity" in wikitext_results:
+                return wikitext_results["perplexity"]
+        
+        return "N/A (Perplexity not found in lm-eval results)"
+        
+    except ImportError:
+        typer.echo("lm-eval framework not available, falling back to manual calculation")
+        return None
+    except Exception as e:
+        typer.echo(f"Error using lm-eval: {e}, falling back to manual calculation")
+        return None
+
+
+def calculate_avg_ppl(model, tokenizer):
+    """
+    Computes the average perplexity using multiple methods.
+    First tries lm-eval framework, then falls back to manual calculation.
+    """
+    typer.echo("Starting perplexity calculation...")
+    
+    # Method 1: Try lm-eval framework
+    ppl_result = calculate_perplexity_lm_eval(model, tokenizer)
+    if ppl_result is not None and not isinstance(ppl_result, str):
+        typer.echo(f"✓ Perplexity calculated using lm-eval: {ppl_result:.4f}")
+        return ppl_result
+    
+    # Method 2: Manual calculation
+    typer.echo("Using manual perplexity calculation...")
+    ppl_result = calculate_perplexity_manual(model, tokenizer)
+    
+    if isinstance(ppl_result, float):
+        typer.echo(f"✓ Perplexity calculated manually: {ppl_result:.4f}")
+        return ppl_result
+    else:
+        typer.echo(f"⚠ Perplexity calculation failed: {ppl_result}")
+        return ppl_result
+
+
+def get_pinned_package_versions():
+    """
+    Retrieves pinned package versions using 'uv pip freeze'.
+    Returns a dictionary mapping lowercased package names to their versions.
+    """
+    try:
+        result = subprocess.run(["uv", "pip", "freeze"], capture_output=True, text=True, check=True)
+        packages_output = result.stdout.strip()
+        versions = {}
+        for line in packages_output.splitlines():
+            if "==" in line:
+                package_name, package_version = line.split("==", 1)
+                versions[package_name.lower()] = package_version
+        return versions
+    except subprocess.CalledProcessError as e:
+        typer.echo(f"Error running 'uv pip freeze': {e}", err=True)
+        return {}
+    except FileNotFoundError:
+        typer.echo("uv command not found. Make sure uv is installed and in your PATH.", err=True)
+        return {}
+
+
+def self_read_script():
+    """
+    Reads the current script file content for inclusion in README.
+    """
+    try:
+        script_path = os.path.abspath(__file__)
+        with open(script_path, "r") as f:
+            script_content = f.read()
+    except Exception as e:
+        script_content = "Error reading script content: " + str(e)
+    return script_content
+
+
+def get_my_user(hf_token):
+    """
+    Gets the Hugging Face username from the provided token.
+    """
+    api = HfApi(token=hf_token)
+    user_info = api.whoami()
+    try:
+        username = user_info.get("name") or user_info.get("username")
+    except Exception as e:
+        typer.echo(f"Error retrieving username from Hugging Face API: {e}. Using default username.")
+        username = api.whoami()
+    if not username:
+        typer.echo("Could not determine your Hugging Face username from the token, defaulting to hard coded username.",
+                   err=True)
+        username = "JustJaro"
+    return username
+
+
+def generate_readme(calibration_dataset, nsamples, quantized_model_dir,
+                    quantized_model_name, script_content, seq_len, source_model, username, avg_ppl):
+    """
+    Generates a comprehensive README.md file for the quantized model.
+    """
+    # Format perplexity value for display
+    if isinstance(avg_ppl, float):
+        ppl_display = f"{avg_ppl:.4f}"
+    else:
+        ppl_display = str(avg_ppl)
+    
+    readme_content = f"""---
+tags:
+- gptq
+- quantization
+- 4bit
+- confidentialmind
+- text-generation
+- apache2.0
+- mistral-small-24b
+---
+# 🔥 Quantized Model: {quantized_model_name} 🔥
+
+This is a 4-bit quantized version of [{source_model}](https://huggingface.co/{source_model}) model, quantized by [ConfidentialMind.com](https://www.confidentialmind.com) 🤖✨  
+It leverages the open-source GPTQModel quantization to achieve 4-bit precision with a group size of 128 resulting in a 
+smaller, 
+faster model with minimal performance degradation.
+
+Ran on a single NVIDIA A100 GPU with 80GB of VRAM.
+
+*Note* `batch_size` is set quite high as the model is small, you may need to adjust this to your GPU VRAM.
+
+## Model Details
+- **Original Model:** [{source_model}](https://huggingface.co/{source_model})
+- **Quantized Model:** {quantized_model_name} (this repository)
+- **Quantization Method:** GPTQ (4-bit, group size 128)
+- **Quantization Library:** [GPTQModel](https://github.com/ModelCloud/GPTQModel/tree/main)
+- **Calibration Dataset:** {calibration_dataset} (using {nsamples} samples with seq len {seq_len})
+- **Quantized by:** [ConfidentialMind.com](https://www.confidentialmind.com)
+
+## Usage
+
+```python
+from gptqmodel import GPTQModel
+from transformers import AutoTokenizer
+
+# Use the local directory or {username}/{quantized_model_name} after upload
+quantized_model_id = "{quantized_model_dir}"  # or "{username}/{quantized_model_name}"
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_id)
+model = GPTQModel.load(quantized_model_id, device="cuda:0")  # or "cpu"
+
+input_text = "This is a test prompt"
+inputs = tokenizer(input_text, return_tensors="pt").to("cuda:0")
+outputs = model.generate(**inputs)
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+## Package Versions and Installation Instructions
+
+See pyproject.toml for the exact UV project file. See the [GPTQModel](
+https://github.com/ModelCloud/GPTQModel/tree/main) repo for more details. on how to install the package.
+
+Use the provided pyproject.toml:
+
+```bash
+uv venv
+source venv/bin/activate
+uv sync
+```
+
+### Environment Variables
+
+```bash
+HF_TOKEN=<YOUR_HF_TOKEN>
+TOKENIZERS_PARALLELISM="true"
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+```
+
+## Quantization Script
+Below is the exact quantize.py script used to generate this model (with the exact versions of the dependencies):
+
+```python
+{script_content}
+```
+
+## Quantization Performance
+
+Average perplexity (PPL) on WikiText-2 test dataset: **{ppl_display}**
+
+*Perplexity calculated using {'GPTQModel evaluation framework' if isinstance(avg_ppl, float) else 'manual calculation method'}*
+
+## Disclaimer
+This model is for research purposes only. It may inherit limitations and biases from the original model and the quantization process. Please use responsibly and refer to the original model card for more details.
+
+## Contact
+For any questions or support, please visit [ConfidentialMind.com](https://www.confidentialmind.com) or contact us directly.
+
+## License
+This model inherits the license from the original model. Please refer to the original model card for more details.
+Original model card: `{source_model}`
+
+## Author
+This model was quantized by [Jaro](https://www.linkedin.com/in/jaroai/)
+
+## Acknowledgements
+Quantization performed using the GPTQModel pipeline.
+
+TODO: Add `gptqmodel.utils.eval` integration and auto-generation of eval table.
+
+---
+*Generated and quantized using GPTQModel.*
+"""
+    readme_path = os.path.join(quantized_model_dir, "README.md")
+    with open(readme_path, "w") as f:
+        f.write(readme_content)
+    typer.echo("README.md created with detailed information.")
+
+
+@app.command()
+def main(
+    seq_len: int = typer.Option(4096, help="Sequence length for tokenization and calibration."),
+    nsamples: int = typer.Option(256, help="Number of samples to use for calibration."),
+    source_model: str = typer.Option("arcee-ai/Virtuoso-Medium-v2",
+                                     help="Source model HF repository identifier."),
+    calibration_dataset: str = typer.Option("wikitext/wikitext-2-raw-v1",
+                                            help="Calibration dataset identifier (in 'dataset/config' format) or local file path."),
+    hf_token: str = typer.Option(HF_TOKEN,
+                                 help="Hugging Face token for creating/updating your repo."),
+    upload_only: bool = typer.Option(False, help="Only upload the quantized model to the Hugging Face Hub."),
+    group_size: GroupSize = typer.Option(GroupSize.accurate, help="Group size for quantization accurate: 32, "
+                                                                  "balanced: 64, fast: 128. Default: accurate."),
+    mse: bool = typer.Option(True, help="Use mse instead of mae for the loss function."),
+):
+    # Prepare destination directory and model names.
+    model_name = source_model.split("/")[-1]
+    quantized_model_name = f"{model_name}_gptq_g{int(group_size.value)}_4bit"
+    quantized_model_dir = os.path.expanduser(os.path.join("~/models/quantized", quantized_model_name))
+    
+    if not os.path.exists(quantized_model_dir) or not upload_only:
+        os.makedirs(quantized_model_dir, exist_ok=True)
+
+        typer.echo("Loading tokenizer from source model...")
+        tokenizer_obj = AutoTokenizer.from_pretrained(source_model, use_fast=True)
+
+        typer.echo("Loading calibration dataset...")
+        typer.echo(f"Calibration dataset: {calibration_dataset}")
+        calibration_data = get_calibration_dataset(tokenizer_obj, nsamples, seq_len, calibration_dataset)
+        if not calibration_data:
+            typer.echo("Calibration dataset is empty. Aborting.", err=True)
+            raise typer.Exit(code=1)
+            
+        if mse:
+            # Fits mistral-small-24b particularly well, as well as the increased damp_percent
+            mse = 0.01
+            quantize_config = QuantizeConfig(bits=4, group_size=int(group_size.value), damp_percent=0.015, mse=mse)
+        else:
+            quantize_config = QuantizeConfig(bits=4, group_size=int(group_size.value), damp_percent=0.01)
+            
+        device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        typer.echo(f"Loading model in {device} mode...")
+        model = GPTQModel.load(source_model, quantize_config)
+
+        typer.echo("Quantizing model...")
+        group_size_factor = int(128 / int(group_size.value))
+        model.quantize(calibration_data, auto_gc=False, batch_size=int((nsamples * 0.1) / group_size_factor))
+        
+        # Retrieve Hugging Face user info for README generation.
+        package_versions = get_pinned_package_versions()
+        username = get_my_user(hf_token)
+        script_content = self_read_script()
+
+        typer.echo(f"Saving quantized model to {quantized_model_dir} using Transformers safe serialization...")
+        try:
+            model.save_pretrained(quantized_model_dir)
+            tokenizer_obj.save_pretrained(quantized_model_dir)
+        except Exception as ex:
+            typer.echo(f"Error during saving with safe_serialization: {ex}. Aborting.")
+            raise
+            
+        typer.echo(f"Model saved to: {quantized_model_dir}")
+    else:
+        tokenizer_obj = AutoTokenizer.from_pretrained(source_model, use_fast=True)
+        package_versions = get_pinned_package_versions()
+        username = get_my_user(hf_token)
+        script_content = self_read_script()
+        device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        
+    # Load the quantized model for perplexity calculation
+    typer.echo("Loading quantized model for evaluation...")
+    model = GPTQModel.load(quantized_model_dir, device=device)
+    
+    # Calculate perplexity with improved method
+    avg_ppl = calculate_avg_ppl(model, tokenizer_obj)
+    typer.echo(f"Final perplexity result: {avg_ppl}")
+    
+    deps = Path("./pyproject.toml")
+    if deps.exists():
+        shutil.copy(deps, quantized_model_dir)
+    
+    generate_readme(calibration_dataset, nsamples, quantized_model_dir,
+                    quantized_model_name, script_content, seq_len, source_model, username, avg_ppl)
+                    
+    typer.echo("Uploading to Hugging Face Hub...")
+    GPTQModel.push_to_hub(quantized_path=quantized_model_dir, private=False, repo_id=quantized_model_name,
+                          token=HF_TOKEN)
+                          
+    typer.echo(f"Model uploaded to Hugging Face repo: {quantized_model_name}")
+    
+    # Run a quick inference demo
+    demo_input = tokenizer_obj("test is", return_tensors="pt").to(device)
+    generated_ids = model.generate(**demo_input)
+    output_text = tokenizer_obj.decode(generated_ids[0])
+    typer.echo(f"Inference demo output: {output_text}")
+    typer.echo(f"Final perplexity on test dataset: {avg_ppl}")
+
+
+if __name__ == "__main__":
+    app()
+
+```
+
+## Quantization Performance
+
+Average perplexity (PPL) on WikiText-2 test dataset: **N/A (Error: Invalid thread config: max_m_blocks = 0, thread_k = -1, thread_n = -1, num_threads = -1 for MKN = [7, 576, 576] and num_bits = 4, group_size = 32, has_act_order = 1, is_k_full = 1, max_shared_mem = 166912)**
+
+*Perplexity calculated using manual calculation method*
+
+## Disclaimer
+This model is for research purposes only. It may inherit limitations and biases from the original model and the quantization process. Please use responsibly and refer to the original model card for more details.
+
+## Contact
+For any questions or support, please visit [ConfidentialMind.com](https://www.confidentialmind.com) or contact us directly.
+
+## License
+This model inherits the license from the original model. Please refer to the original model card for more details.
+Original model card: `HuggingFaceTB/SmolLM-135M`
+
+## Author
+This model was quantized by [Jaro](https://www.linkedin.com/in/jaroai/)
+
+## Acknowledgements
+Quantization performed using the GPTQModel pipeline.
+
+TODO: Add `gptqmodel.utils.eval` integration and auto-generation of eval table.
+
+---
+*Generated and quantized using GPTQModel.*

config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 576,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 9,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 3,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": true,
+    "group_size": 32,
+    "lm_head": false,
+    "meta": {
+      "damp_auto_increment": 0.0025,
+      "damp_percent": 0.015,
+      "mse": 0.01,
+      "quantizer": [
+        "gptqmodel:2.2.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel"
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": true,
+  "vocab_size": 49152
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.53.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b6bc06e57afc22123c73f3ee13ac09257d654a9c1e7cfbcaf7030dfa6252657
+size 118768376

pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "gtpq"
+version = "0.1.0"
+requires-python = ">=3.12"
+dependencies = [
+  "gptqmodel[vllm,bitblas,auto_round] @ git+https://github.com/IntelLabs/gptqmodel.git@main",
+  "flashinfer-python",            # binary wheels; resolve CPU/GPU at install time
+  "hatchling",
+]
+
+[build-system]
+requires = ["setuptools-rust", "Cython", "hatchling"]
+build-backend = "setuptools.build_meta"

quant_log.csv

@@ -0,0 +1,211 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.30323729,0.01500,1.376
+0,self_attn.v_proj,0.00081456,0.01500,1.134
+0,self_attn.q_proj,0.50177538,0.01500,1.143
+0,self_attn.o_proj,0.00410974,0.01500,1.150
+0,mlp.up_proj,0.87140083,0.01500,1.140
+0,mlp.gate_proj,0.88596356,0.01500,1.168
+0,mlp.down_proj,3.59487438,0.01500,2.880
+1,self_attn.k_proj,1.79021871,0.01500,1.075
+1,self_attn.v_proj,0.08866195,0.01500,1.081
+1,self_attn.q_proj,2.48720980,0.01500,1.146
+1,self_attn.o_proj,0.79290819,0.01500,1.133
+1,mlp.up_proj,0.70214444,0.01500,1.135
+1,mlp.gate_proj,0.79838932,0.01500,1.136
+1,mlp.down_proj,2.23402882,0.01500,2.897
+2,self_attn.k_proj,1.86278677,0.01500,1.109
+2,self_attn.v_proj,0.19091517,0.01500,1.116
+2,self_attn.q_proj,1.61090231,0.01500,1.166
+2,self_attn.o_proj,0.63032722,0.01500,1.121
+2,mlp.up_proj,0.84249359,0.01500,1.134
+2,mlp.gate_proj,0.95266348,0.01500,1.129
+2,mlp.down_proj,157.33862305,0.01500,2.853
+3,self_attn.k_proj,1.13371205,0.01500,1.070
+3,self_attn.v_proj,0.19112220,0.01500,1.130
+3,self_attn.q_proj,1.95455039,0.01500,1.110
+3,self_attn.o_proj,0.43849713,0.01500,1.116
+3,mlp.up_proj,0.91409326,0.01500,1.126
+3,mlp.gate_proj,1.00409079,0.01500,1.135
+3,mlp.down_proj,12.42405319,0.01500,2.883
+4,self_attn.k_proj,0.87567437,0.01500,1.108
+4,self_attn.v_proj,0.14826132,0.01500,1.070
+4,self_attn.q_proj,2.46379852,0.01500,1.138
+4,self_attn.o_proj,0.33942133,0.01500,1.126
+4,mlp.up_proj,1.05060482,0.01500,1.138
+4,mlp.gate_proj,1.17461491,0.01500,1.132
+4,mlp.down_proj,1.44320405,0.01500,2.879
+5,self_attn.k_proj,1.32893443,0.01500,1.075
+5,self_attn.v_proj,0.17556122,0.01500,1.062
+5,self_attn.q_proj,2.47826910,0.01500,1.112
+5,self_attn.o_proj,0.59731841,0.01500,1.113
+5,mlp.up_proj,1.13487399,0.01500,1.148
+5,mlp.gate_proj,1.18413568,0.01500,1.135
+5,mlp.down_proj,1.55033576,0.01500,2.881
+6,self_attn.k_proj,1.86866426,0.01500,1.072
+6,self_attn.v_proj,0.21436407,0.01500,1.075
+6,self_attn.q_proj,2.72684455,0.01500,1.111
+6,self_attn.o_proj,0.75209558,0.01500,1.127
+6,mlp.up_proj,1.31616020,0.01500,1.132
+6,mlp.gate_proj,1.32581091,0.01500,1.139
+6,mlp.down_proj,2.13556457,0.01500,2.880
+7,self_attn.k_proj,1.53838611,0.01500,1.087
+7,self_attn.v_proj,0.22950518,0.01500,1.071
+7,self_attn.q_proj,1.99251246,0.01500,1.109
+7,self_attn.o_proj,0.50463128,0.01500,1.118
+7,mlp.up_proj,1.58821356,0.01500,1.125
+7,mlp.gate_proj,1.58579147,0.01500,1.147
+7,mlp.down_proj,13.17560577,0.01500,2.849
+8,self_attn.k_proj,0.73372823,0.01500,1.068
+8,self_attn.v_proj,0.23304689,0.01500,1.088
+8,self_attn.q_proj,2.41669130,0.01500,1.118
+8,self_attn.o_proj,0.46610335,0.01500,1.116
+8,mlp.up_proj,1.65750051,0.01500,1.136
+8,mlp.gate_proj,1.56000757,0.01500,1.156
+8,mlp.down_proj,8.41586971,0.01500,2.874
+9,self_attn.k_proj,1.64468992,0.01500,1.094
+9,self_attn.v_proj,0.18850568,0.01500,1.094
+9,self_attn.q_proj,1.78751516,0.01500,1.112
+9,self_attn.o_proj,1.78545237,0.01500,1.117
+9,mlp.up_proj,1.77682400,0.01500,1.127
+9,mlp.gate_proj,1.57283258,0.01500,1.141
+9,mlp.down_proj,14.36819839,0.01500,2.873
+10,self_attn.k_proj,1.32247448,0.01500,1.061
+10,self_attn.v_proj,0.22332871,0.01500,1.074
+10,self_attn.q_proj,1.70967674,0.01500,1.109
+10,self_attn.o_proj,2.30593014,0.01500,1.119
+10,mlp.up_proj,2.46851778,0.01500,1.129
+10,mlp.gate_proj,2.33083105,0.01500,1.137
+10,mlp.down_proj,17.56273270,0.01500,2.857
+11,self_attn.k_proj,1.05880129,0.01500,1.068
+11,self_attn.v_proj,0.20641716,0.01500,1.074
+11,self_attn.q_proj,2.14066696,0.01500,1.111
+11,self_attn.o_proj,2.36638355,0.01500,1.138
+11,mlp.up_proj,2.62533450,0.01500,1.126
+11,mlp.gate_proj,2.36855841,0.01500,1.117
+11,mlp.down_proj,36624136.00000000,0.01500,2.888
+12,self_attn.k_proj,0.60637605,0.01500,1.081
+12,self_attn.v_proj,0.20556667,0.01500,1.108
+12,self_attn.q_proj,6.41779613,0.01500,1.111
+12,self_attn.o_proj,1.40109611,0.01500,1.119
+12,mlp.up_proj,10.91535950,0.01500,1.132
+12,mlp.gate_proj,4.99505901,0.01500,1.146
+12,mlp.down_proj,9760.42968750,0.01500,2.909
+13,self_attn.k_proj,2.24922800,0.01500,1.082
+13,self_attn.v_proj,0.27756080,0.01500,1.086
+13,self_attn.q_proj,7.30099392,0.01500,1.109
+13,self_attn.o_proj,2.96828890,0.01500,1.118
+13,mlp.up_proj,2.91019154,0.01500,1.123
+13,mlp.gate_proj,6.23261738,0.01500,1.141
+13,mlp.down_proj,6.46631765,0.01500,2.860
+14,self_attn.k_proj,0.69264430,0.01500,1.071
+14,self_attn.v_proj,0.51544046,0.01500,1.091
+14,self_attn.q_proj,3.18851161,0.01500,1.131
+14,self_attn.o_proj,1.25791931,0.01500,1.127
+14,mlp.up_proj,2.58898783,0.01500,1.112
+14,mlp.gate_proj,5.24106836,0.01500,1.133
+14,mlp.down_proj,9.01900768,0.01500,2.851
+15,self_attn.k_proj,0.67096651,0.01500,1.094
+15,self_attn.v_proj,0.36131397,0.01500,1.112
+15,self_attn.q_proj,3.81791139,0.01500,1.097
+15,self_attn.o_proj,2.25381660,0.01500,1.114
+15,mlp.up_proj,2.34244204,0.01500,1.116
+15,mlp.gate_proj,4.29145336,0.01500,1.132
+15,mlp.down_proj,7.26017952,0.01500,2.923
+16,self_attn.k_proj,0.64388847,0.01500,1.060
+16,self_attn.v_proj,0.34459960,0.01500,1.078
+16,self_attn.q_proj,4.61652613,0.01500,1.101
+16,self_attn.o_proj,1.88339543,0.01500,1.119
+16,mlp.up_proj,2.72254467,0.01500,1.121
+16,mlp.gate_proj,3.13850927,0.01500,1.119
+16,mlp.down_proj,6.84692860,0.01500,2.887
+17,self_attn.k_proj,0.55838877,0.01500,1.057
+17,self_attn.v_proj,0.51657593,0.01500,1.071
+17,self_attn.q_proj,4.33557653,0.01500,1.092
+17,self_attn.o_proj,2.56168294,0.01500,1.126
+17,mlp.up_proj,3.06461382,0.01500,1.110
+17,mlp.gate_proj,3.21341205,0.01500,1.114
+17,mlp.down_proj,7.29544258,0.01500,2.854
+18,self_attn.k_proj,0.68149984,0.01500,1.102
+18,self_attn.v_proj,1.32401478,0.01500,1.124
+18,self_attn.q_proj,7.70216036,0.01500,1.098
+18,self_attn.o_proj,2.28013134,0.01500,1.119
+18,mlp.up_proj,3.90698004,0.01500,1.116
+18,mlp.gate_proj,4.11436844,0.01500,1.152
+18,mlp.down_proj,11.87063789,0.01500,2.870
+19,self_attn.k_proj,0.65415150,0.01500,1.066
+19,self_attn.v_proj,0.53765559,0.01500,1.075
+19,self_attn.q_proj,6.70729065,0.01500,1.112
+19,self_attn.o_proj,3.07730746,0.01500,1.132
+19,mlp.up_proj,3.57759047,0.01500,1.119
+19,mlp.gate_proj,5.04411411,0.01500,1.120
+19,mlp.down_proj,13.49418736,0.01500,2.850
+20,self_attn.k_proj,0.45334548,0.01500,1.062
+20,self_attn.v_proj,1.12238181,0.01500,1.083
+20,self_attn.q_proj,5.45634222,0.01500,1.106
+20,self_attn.o_proj,2.11403751,0.01500,1.122
+20,mlp.up_proj,3.80852771,0.01500,1.123
+20,mlp.gate_proj,4.78546810,0.01500,1.129
+20,mlp.down_proj,15.71007252,0.01500,2.860
+21,self_attn.k_proj,0.67353952,0.01500,1.056
+21,self_attn.v_proj,0.74522316,0.01500,1.084
+21,self_attn.q_proj,4.39328623,0.01500,1.095
+21,self_attn.o_proj,2.93690538,0.01500,1.109
+21,mlp.up_proj,4.77232122,0.01500,1.108
+21,mlp.gate_proj,5.83200645,0.01500,1.138
+21,mlp.down_proj,19.96510315,0.01500,2.825
+22,self_attn.k_proj,0.63552809,0.01500,1.060
+22,self_attn.v_proj,1.77812839,0.01500,1.101
+22,self_attn.q_proj,6.14302778,0.01500,1.112
+22,self_attn.o_proj,3.85619378,0.01500,1.108
+22,mlp.up_proj,5.24628830,0.01500,1.106
+22,mlp.gate_proj,8.02893066,0.01500,1.106
+22,mlp.down_proj,24.18618774,0.01500,2.823
+23,self_attn.k_proj,0.82243836,0.01500,1.073
+23,self_attn.v_proj,2.90332031,0.01500,1.065
+23,self_attn.q_proj,9.50726032,0.01500,1.098
+23,self_attn.o_proj,8.13974857,0.01500,1.113
+23,mlp.up_proj,5.42801857,0.01500,1.099
+23,mlp.gate_proj,10.77536011,0.01500,1.128
+23,mlp.down_proj,29.22331047,0.01500,2.812
+24,self_attn.k_proj,0.60857439,0.01500,1.061
+24,self_attn.v_proj,1.17702281,0.01500,1.069
+24,self_attn.q_proj,6.26180887,0.01500,1.110
+24,self_attn.o_proj,4.30931950,0.01500,1.096
+24,mlp.up_proj,5.46759701,0.01500,1.095
+24,mlp.gate_proj,10.43561554,0.01500,1.102
+24,mlp.down_proj,31.56058502,0.01500,2.808
+25,self_attn.k_proj,0.78283930,0.01500,1.041
+25,self_attn.v_proj,1.24567127,0.01500,1.058
+25,self_attn.q_proj,4.97283077,0.01500,1.095
+25,self_attn.o_proj,5.16120386,0.01500,1.105
+25,mlp.up_proj,6.94152546,0.01500,1.102
+25,mlp.gate_proj,9.48820877,0.01500,1.099
+25,mlp.down_proj,37.16706085,0.01500,2.802
+26,self_attn.k_proj,0.95313138,0.01500,1.052
+26,self_attn.v_proj,1.37231970,0.01500,1.054
+26,self_attn.q_proj,6.39235163,0.01500,1.087
+26,self_attn.o_proj,5.90192604,0.01500,1.090
+26,mlp.up_proj,12.41309357,0.01500,1.094
+26,mlp.gate_proj,9.54691982,0.01500,1.093
+26,mlp.down_proj,37.73841095,0.01500,2.798
+27,self_attn.k_proj,0.49448842,0.01500,1.041
+27,self_attn.v_proj,0.81519926,0.01500,1.044
+27,self_attn.q_proj,5.96102524,0.01500,1.087
+27,self_attn.o_proj,5.79659319,0.01500,1.101
+27,mlp.up_proj,14.86005974,0.01500,1.099
+27,mlp.gate_proj,29.81634903,0.01500,1.100
+27,mlp.down_proj,702.13244629,0.01500,2.865
+28,self_attn.k_proj,0.60215271,0.01500,1.045
+28,self_attn.v_proj,1.07407022,0.01500,1.044
+28,self_attn.q_proj,7.19515991,0.01500,1.075
+28,self_attn.o_proj,6.81128788,0.01500,1.090
+28,mlp.up_proj,28.30822754,0.01500,1.100
+28,mlp.gate_proj,35.86479950,0.01500,1.100
+28,mlp.down_proj,33054578.00000000,0.01500,2.789
+29,self_attn.k_proj,0.62672096,0.01500,1.042
+29,self_attn.v_proj,0.87135106,0.01500,1.051
+29,self_attn.q_proj,12.68386078,0.01500,1.077
+29,self_attn.o_proj,41.61279678,0.01500,1.103
+29,mlp.up_proj,224.53115845,0.01500,1.092
+29,mlp.gate_proj,142.67680359,0.01500,1.107
+29,mlp.down_proj,10202.70410156,0.01500,2.805

quantize_config.json

@@ -0,0 +1,21 @@
+{
+  "bits": 4,
+  "group_size": 32,
+  "desc_act": true,
+  "sym": true,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:2.2.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.015,
+    "damp_auto_increment": 0.0025,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.01
+  }
+}

special_tokens_map.json

@@ -0,0 +1,42 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,168 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}