Upload README.md with huggingface_hub
Browse files
README.md
CHANGED
@@ -117,9 +117,9 @@ print("content:", content)
|
|
117 |
|
118 |
Install the required packages:
|
119 |
```Shell
|
|
|
120 |
pip install git+https://github.com/huggingface/transformers@main
|
121 |
pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cu126
|
122 |
-
pip install torch
|
123 |
pip install accelerate
|
124 |
```
|
125 |
|
@@ -137,10 +137,10 @@ model_to_quantize = "google/gemma-3-12b-it"
|
|
137 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
138 |
quant_config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
139 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
140 |
-
|
141 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
|
142 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
143 |
|
|
|
144 |
# Push to hub
|
145 |
USER_ID = "YOUR_USER_ID"
|
146 |
MODEL_NAME = model_id.split("/")[-1]
|
|
|
117 |
|
118 |
Install the required packages:
|
119 |
```Shell
|
120 |
+
pip install torch
|
121 |
pip install git+https://github.com/huggingface/transformers@main
|
122 |
pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cu126
|
|
|
123 |
pip install accelerate
|
124 |
```
|
125 |
|
|
|
137 |
from torchao.quantization import Float8DynamicActivationFloat8WeightConfig, PerRow
|
138 |
quant_config = Float8DynamicActivationFloat8WeightConfig(granularity=PerRow())
|
139 |
quantization_config = TorchAoConfig(quant_type=quant_config)
|
|
|
140 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_to_quantize, device_map="auto", torch_dtype=torch.bfloat16, quantization_config=quantization_config)
|
141 |
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
142 |
|
143 |
+
|
144 |
# Push to hub
|
145 |
USER_ID = "YOUR_USER_ID"
|
146 |
MODEL_NAME = model_id.split("/")[-1]
|