adafactor

README.md

@@ -5,32 +5,47 @@ license: apache-2.0
 Краткая инструкция по установке
 Обновите систему и установите git-lfs:
 
-```apt update
-apt install git-lfs```
+```
+apt update
+apt install git-lfs
+```
 Обновите pip и установите требуемые пакеты:
 
-```python -m pip install --upgrade pip
-pip install -r requirements.txt```
+```
+python -m pip install --upgrade pip
+pip install -r requirements.txt
+```
 Настройте git-credentials:
 
-```git config --global credential.helper store```
+```
+git config --global credential.helper store
+```
 Клонируйте репозиторий:
 
-```git clone https://huggingface.co/AiArtLab/sdxs
-cd sdxs/```
+```
+git clone https://huggingface.co/AiArtLab/sdxs
+cd sdxs/
+```
 Подготовьте датасет:
 
-```mkdir datasets
+```
+mkdir datasets
 cd datasets
-git clone https://huggingface.co/datasets/AiArtLab/imagenet-1kk```
+git clone https://huggingface.co/datasets/AiArtLab/imagenet-1kk
+```
 Выполните вход в сервисы:
 
-```huggingface-cli login
-wandb login```
+```
+huggingface-cli login
+wandb login
+```
 Настройте accelerate для многокарточного обучения:
 
-```accelerate config```
-(выберите multigpu и bf16)
-Запустите обучение:
+```
+accelerate config
+```
+Запустите обучение!
 
-```nohup accelerate launch train.py &```
+```
+nohup accelerate launch train.py &
+```

train.py

@@ -22,12 +22,12 @@ from diffusers.models.attention_processor import AttnProcessor2_0
 
 # --------------------------- Параметры ---------------------------
 save_path = "datasets/dataset384_temp" #"datasets/dataset384" #"datasets/imagenet-1kk" #"datasets/siski576" #"datasets/siski384" #"datasets/siski64" #"datasets/mnist"
-batch_size = 30 #555 #35 #7
+batch_size = 33 #555 #35 #7
 base_learning_rate = 0 #5e-5 #8e-5
 min_learning_rate = 2.5e-5 #2e-5
-num_epochs = 1 #36 #18
+num_epochs = 5 #36 #18
 project = "sdxs"
-use_wandb = False
+use_wandb = True
 limit = 0
 save_model = True
 checkpoints_folder = ""
@@ -266,6 +266,7 @@ if lowram:
         from transformers.optimization import Adafactor
         
         # [1] Создаем словарь оптимизаторов (для каждого параметра)
+        base_learning_rate = 0
         optimizer_dict = {
             p: Adafactor(
                 [p],
@@ -476,7 +477,7 @@ for epoch in range(start_epoch, start_epoch + num_epochs):
     
     for step, (latents, embeddings) in enumerate(dataloader):
         with accelerator.accumulate(unet):
-            if step == 2:
+            if save_model == False and step == 3 :
                 used_gb = torch.cuda.max_memory_allocated() / 1024**3
                 print(f"Шаг {step}: {used_gb:.2f} GB")
             # Forward pass