wj2003
/

Pongo-70B

wj2003 commited on Aug 4, 2023

Commit

7ff4fe9

•

1 Parent(s): 399aee3

Update README.md

Files changed (1) hide show

README.md CHANGED Viewed

@@ -20,7 +20,7 @@ model = AutoModelForCausalLM.from_pretrained(
     load_in_4bit=True,
     torch_dtype=torch.bfloat16,
     device_map="auto",
-    max_memory={i: '24000MB' for i in range(torch.cuda.device_count())},
     quantization_config=BitsAndBytesConfig(
         load_in_4bit=True,
         bnb_4bit_compute_dtype=torch.bfloat16,

     load_in_4bit=True,
     torch_dtype=torch.bfloat16,
     device_map="auto",
+    max_memory={i: '48000MB' for i in range(torch.cuda.device_count())},
     quantization_config=BitsAndBytesConfig(
         load_in_4bit=True,
         bnb_4bit_compute_dtype=torch.bfloat16,