Upload README.md with huggingface_hub
Browse files
README.md
CHANGED
|
@@ -117,9 +117,9 @@ print("content:", content)
|
|
| 117 |
|
| 118 |
Install the required packages:
|
| 119 |
```Shell
|
|
|
|
| 120 |
pip install git+https://github.com/huggingface/transformers@main
|
| 121 |
pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cu126
|
| 122 |
-
pip install torch
|
| 123 |
pip install accelerate
|
| 124 |
```
|
| 125 |
|
|
@@ -137,10 +137,10 @@ model_to_quantize = "google/gemma-3-12b-it"
|
|
| 137 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
| 138 |
quant_config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
| 139 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
| 140 |
-
|
| 141 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
|
| 142 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 143 |
|
|
|
|
| 144 |
# Push to hub
|
| 145 |
USER_ID = "YOUR_USER_ID"
|
| 146 |
MODEL_NAME = model_id.split("/")[-1]
|
|
|
|
| 117 |
|
| 118 |
Install the required packages:
|
| 119 |
```Shell
|
| 120 |
+
pip install torch
|
| 121 |
pip install git+https://github.com/huggingface/transformers@main
|
| 122 |
pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cu126
|
|
|
|
| 123 |
pip install accelerate
|
| 124 |
```
|
| 125 |
|
|
|
|
| 137 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
| 138 |
quant_config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
| 139 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
|
|
|
| 140 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
|
| 141 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
| 142 |
|
| 143 |
+
|
| 144 |
# Push to hub
|
| 145 |
USER_ID = "YOUR_USER_ID"
|
| 146 |
MODEL_NAME = model_id.split("/")[-1]
|