OPEA
/

GGUF
Inference Endpoints
conversational
cicdatopea commited on
Commit
5395fb9
·
verified ·
1 Parent(s): c37e4f8

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +2 -2
README.md CHANGED
@@ -155,9 +155,9 @@ for n, m in block.named_modules():
155
  from auto_round import AutoRound
156
 
157
  autoround = AutoRound(model=model, tokenizer=tokenizer, device_map=device_map,
158
- iters=200,batch_size=8, seqlen=512)
159
  autoround.quantize()
160
- autoround.save_quantized(format="gguf:q4_0", output_dir="tmp_autoround"
161
  ```
162
 
163
  ## Ethical Considerations and Limitations
 
155
  from auto_round import AutoRound
156
 
157
  autoround = AutoRound(model=model, tokenizer=tokenizer, device_map=device_map,
158
+ iters=200,batch_size=8, seqlen=512, enable_torch_compile=False)
159
  autoround.quantize()
160
+ autoround.save_quantized(format="gguf:q4_0", output_dir="tmp_autoround")
161
  ```
162
 
163
  ## Ethical Considerations and Limitations