| { | |
| "batchers": null, | |
| "cachers": null, | |
| "compilers": null, | |
| "distillers": null, | |
| "pruners": null, | |
| "quantizers": "hqq", | |
| "recoverers": null, | |
| "quant_hqq_backend": "torchao_int4", | |
| "quant_hqq_group_size": 64, | |
| "quant_hqq_weight_bits": 4, | |
| "max_batch_size": 1, | |
| "device": "cuda", | |
| "cache_dir": "/tmp/models/tmppxuehdf9", | |
| "task": "", | |
| "save_load_fn": "hqq", | |
| "save_load_fn_args": {}, | |
| "api_key": null | |
| } |