qlora and 4bit check so we are able to merge and unload
Browse files
src/axolotl/utils/models.py
CHANGED
|
@@ -85,7 +85,7 @@ def load_model(
|
|
| 85 |
raise e
|
| 86 |
|
| 87 |
model_kwargs = {}
|
| 88 |
-
if cfg.adapter == "qlora":
|
| 89 |
model_kwargs["quantization_config"] = BitsAndBytesConfig(
|
| 90 |
load_in_4bit=True,
|
| 91 |
llm_int8_threshold=6.0,
|
|
|
|
| 85 |
raise e
|
| 86 |
|
| 87 |
model_kwargs = {}
|
| 88 |
+
if cfg.adapter == "qlora" and cfg.load_in_4bit:
|
| 89 |
model_kwargs["quantization_config"] = BitsAndBytesConfig(
|
| 90 |
load_in_4bit=True,
|
| 91 |
llm_int8_threshold=6.0,
|