Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +2 -0
- train/act_dice_art_run1/checkpoints/020000/pretrained_model/config.json +64 -0
- train/act_dice_art_run1/checkpoints/020000/pretrained_model/model.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/020000/pretrained_model/train_config.json +170 -0
- train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_param_groups.json +191 -0
- train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/020000/training_state/rng_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/020000/training_state/training_step.json +3 -0
- train/act_dice_art_run1/checkpoints/040000/pretrained_model/config.json +64 -0
- train/act_dice_art_run1/checkpoints/040000/pretrained_model/model.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json +170 -0
- train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_param_groups.json +191 -0
- train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/040000/training_state/rng_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/040000/training_state/training_step.json +3 -0
- train/act_dice_art_run1/checkpoints/060000/pretrained_model/config.json +64 -0
- train/act_dice_art_run1/checkpoints/060000/pretrained_model/model.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/060000/pretrained_model/train_config.json +170 -0
- train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_param_groups.json +191 -0
- train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/060000/training_state/rng_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/060000/training_state/training_step.json +3 -0
- train/act_dice_art_run1/checkpoints/080000/pretrained_model/config.json +64 -0
- train/act_dice_art_run1/checkpoints/080000/pretrained_model/model.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/080000/pretrained_model/train_config.json +170 -0
- train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_param_groups.json +191 -0
- train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/080000/training_state/rng_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/080000/training_state/training_step.json +3 -0
- train/act_dice_art_run1/checkpoints/100000/pretrained_model/config.json +64 -0
- train/act_dice_art_run1/checkpoints/100000/pretrained_model/model.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/100000/pretrained_model/train_config.json +170 -0
- train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_param_groups.json +191 -0
- train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/100000/training_state/rng_state.safetensors +3 -0
- train/act_dice_art_run1/checkpoints/100000/training_state/training_step.json +3 -0
- train/act_dice_art_run1/wandb/debug-internal.log +136 -0
- train/act_dice_art_run1/wandb/debug.log +23 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/config.yaml +161 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/output.log +301 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/requirements.txt +167 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-metadata.json +51 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-summary.json +1 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log +11 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log +11 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log +886 -0
- train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb +3 -0
- train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/config.yaml +186 -0
- train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/output.log +317 -0
- train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/requirements.txt +167 -0
.gitattributes
CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb filter=lfs diff=lfs merge=lfs -text
|
37 |
+
train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/run-fhdgxxjz.wandb filter=lfs diff=lfs merge=lfs -text
|
train/act_dice_art_run1/checkpoints/020000/pretrained_model/config.json
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"type": "act",
|
3 |
+
"n_obs_steps": 1,
|
4 |
+
"normalization_mapping": {
|
5 |
+
"VISUAL": "MEAN_STD",
|
6 |
+
"STATE": "MEAN_STD",
|
7 |
+
"ACTION": "MEAN_STD"
|
8 |
+
},
|
9 |
+
"input_features": {
|
10 |
+
"observation.state": {
|
11 |
+
"type": "STATE",
|
12 |
+
"shape": [
|
13 |
+
6
|
14 |
+
]
|
15 |
+
},
|
16 |
+
"observation.images.arm": {
|
17 |
+
"type": "VISUAL",
|
18 |
+
"shape": [
|
19 |
+
3,
|
20 |
+
480,
|
21 |
+
640
|
22 |
+
]
|
23 |
+
},
|
24 |
+
"observation.images.world": {
|
25 |
+
"type": "VISUAL",
|
26 |
+
"shape": [
|
27 |
+
3,
|
28 |
+
480,
|
29 |
+
640
|
30 |
+
]
|
31 |
+
}
|
32 |
+
},
|
33 |
+
"output_features": {
|
34 |
+
"action": {
|
35 |
+
"type": "ACTION",
|
36 |
+
"shape": [
|
37 |
+
6
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"device": "cuda",
|
42 |
+
"use_amp": false,
|
43 |
+
"chunk_size": 100,
|
44 |
+
"n_action_steps": 100,
|
45 |
+
"vision_backbone": "resnet18",
|
46 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
47 |
+
"replace_final_stride_with_dilation": false,
|
48 |
+
"pre_norm": false,
|
49 |
+
"dim_model": 512,
|
50 |
+
"n_heads": 8,
|
51 |
+
"dim_feedforward": 3200,
|
52 |
+
"feedforward_activation": "relu",
|
53 |
+
"n_encoder_layers": 4,
|
54 |
+
"n_decoder_layers": 1,
|
55 |
+
"use_vae": true,
|
56 |
+
"latent_dim": 32,
|
57 |
+
"n_vae_encoder_layers": 4,
|
58 |
+
"temporal_ensemble_coeff": null,
|
59 |
+
"dropout": 0.1,
|
60 |
+
"kl_weight": 10.0,
|
61 |
+
"optimizer_lr": 1e-05,
|
62 |
+
"optimizer_weight_decay": 0.0001,
|
63 |
+
"optimizer_lr_backbone": 1e-05
|
64 |
+
}
|
train/act_dice_art_run1/checkpoints/020000/pretrained_model/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3debbbbdc15ac24f2faf211ed999b8521d89fb2932125b067b08cfb77945cb86
|
3 |
+
size 206701064
|
train/act_dice_art_run1/checkpoints/020000/pretrained_model/train_config.json
ADDED
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset": {
|
3 |
+
"repo_id": "BoredMachinist/dice-art-run1",
|
4 |
+
"root": null,
|
5 |
+
"episodes": null,
|
6 |
+
"image_transforms": {
|
7 |
+
"enable": false,
|
8 |
+
"max_num_transforms": 3,
|
9 |
+
"random_order": false,
|
10 |
+
"tfs": {
|
11 |
+
"brightness": {
|
12 |
+
"weight": 1.0,
|
13 |
+
"type": "ColorJitter",
|
14 |
+
"kwargs": {
|
15 |
+
"brightness": [
|
16 |
+
0.8,
|
17 |
+
1.2
|
18 |
+
]
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"contrast": {
|
22 |
+
"weight": 1.0,
|
23 |
+
"type": "ColorJitter",
|
24 |
+
"kwargs": {
|
25 |
+
"contrast": [
|
26 |
+
0.8,
|
27 |
+
1.2
|
28 |
+
]
|
29 |
+
}
|
30 |
+
},
|
31 |
+
"saturation": {
|
32 |
+
"weight": 1.0,
|
33 |
+
"type": "ColorJitter",
|
34 |
+
"kwargs": {
|
35 |
+
"saturation": [
|
36 |
+
0.5,
|
37 |
+
1.5
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"hue": {
|
42 |
+
"weight": 1.0,
|
43 |
+
"type": "ColorJitter",
|
44 |
+
"kwargs": {
|
45 |
+
"hue": [
|
46 |
+
-0.05,
|
47 |
+
0.05
|
48 |
+
]
|
49 |
+
}
|
50 |
+
},
|
51 |
+
"sharpness": {
|
52 |
+
"weight": 1.0,
|
53 |
+
"type": "SharpnessJitter",
|
54 |
+
"kwargs": {
|
55 |
+
"sharpness": [
|
56 |
+
0.5,
|
57 |
+
1.5
|
58 |
+
]
|
59 |
+
}
|
60 |
+
}
|
61 |
+
}
|
62 |
+
},
|
63 |
+
"revision": null,
|
64 |
+
"use_imagenet_stats": true,
|
65 |
+
"video_backend": "pyav"
|
66 |
+
},
|
67 |
+
"env": null,
|
68 |
+
"policy": {
|
69 |
+
"type": "act",
|
70 |
+
"n_obs_steps": 1,
|
71 |
+
"normalization_mapping": {
|
72 |
+
"VISUAL": "MEAN_STD",
|
73 |
+
"STATE": "MEAN_STD",
|
74 |
+
"ACTION": "MEAN_STD"
|
75 |
+
},
|
76 |
+
"input_features": {
|
77 |
+
"observation.state": {
|
78 |
+
"type": "STATE",
|
79 |
+
"shape": [
|
80 |
+
6
|
81 |
+
]
|
82 |
+
},
|
83 |
+
"observation.images.arm": {
|
84 |
+
"type": "VISUAL",
|
85 |
+
"shape": [
|
86 |
+
3,
|
87 |
+
480,
|
88 |
+
640
|
89 |
+
]
|
90 |
+
},
|
91 |
+
"observation.images.world": {
|
92 |
+
"type": "VISUAL",
|
93 |
+
"shape": [
|
94 |
+
3,
|
95 |
+
480,
|
96 |
+
640
|
97 |
+
]
|
98 |
+
}
|
99 |
+
},
|
100 |
+
"output_features": {
|
101 |
+
"action": {
|
102 |
+
"type": "ACTION",
|
103 |
+
"shape": [
|
104 |
+
6
|
105 |
+
]
|
106 |
+
}
|
107 |
+
},
|
108 |
+
"device": "cuda",
|
109 |
+
"use_amp": false,
|
110 |
+
"chunk_size": 100,
|
111 |
+
"n_action_steps": 100,
|
112 |
+
"vision_backbone": "resnet18",
|
113 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
114 |
+
"replace_final_stride_with_dilation": false,
|
115 |
+
"pre_norm": false,
|
116 |
+
"dim_model": 512,
|
117 |
+
"n_heads": 8,
|
118 |
+
"dim_feedforward": 3200,
|
119 |
+
"feedforward_activation": "relu",
|
120 |
+
"n_encoder_layers": 4,
|
121 |
+
"n_decoder_layers": 1,
|
122 |
+
"use_vae": true,
|
123 |
+
"latent_dim": 32,
|
124 |
+
"n_vae_encoder_layers": 4,
|
125 |
+
"temporal_ensemble_coeff": null,
|
126 |
+
"dropout": 0.1,
|
127 |
+
"kl_weight": 10.0,
|
128 |
+
"optimizer_lr": 1e-05,
|
129 |
+
"optimizer_weight_decay": 0.0001,
|
130 |
+
"optimizer_lr_backbone": 1e-05
|
131 |
+
},
|
132 |
+
"output_dir": "outputs/train/act_dice_art_run1",
|
133 |
+
"job_name": "act_dice_art_run1",
|
134 |
+
"resume": false,
|
135 |
+
"seed": 1000,
|
136 |
+
"num_workers": 4,
|
137 |
+
"batch_size": 8,
|
138 |
+
"steps": 100000,
|
139 |
+
"eval_freq": 20000,
|
140 |
+
"log_freq": 200,
|
141 |
+
"save_checkpoint": true,
|
142 |
+
"save_freq": 20000,
|
143 |
+
"use_policy_training_preset": true,
|
144 |
+
"optimizer": {
|
145 |
+
"type": "adamw",
|
146 |
+
"lr": 1e-05,
|
147 |
+
"weight_decay": 0.0001,
|
148 |
+
"grad_clip_norm": 10.0,
|
149 |
+
"betas": [
|
150 |
+
0.9,
|
151 |
+
0.999
|
152 |
+
],
|
153 |
+
"eps": 1e-08
|
154 |
+
},
|
155 |
+
"scheduler": null,
|
156 |
+
"eval": {
|
157 |
+
"n_episodes": 50,
|
158 |
+
"batch_size": 50,
|
159 |
+
"use_async_envs": false
|
160 |
+
},
|
161 |
+
"wandb": {
|
162 |
+
"enable": true,
|
163 |
+
"disable_artifact": false,
|
164 |
+
"project": "lerobot",
|
165 |
+
"entity": null,
|
166 |
+
"notes": null,
|
167 |
+
"run_id": "fhdgxxjz",
|
168 |
+
"mode": null
|
169 |
+
}
|
170 |
+
}
|
train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_param_groups.json
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"lr": 1e-05,
|
4 |
+
"betas": [
|
5 |
+
0.9,
|
6 |
+
0.999
|
7 |
+
],
|
8 |
+
"eps": 1e-08,
|
9 |
+
"weight_decay": 0.0001,
|
10 |
+
"amsgrad": false,
|
11 |
+
"maximize": false,
|
12 |
+
"foreach": null,
|
13 |
+
"capturable": false,
|
14 |
+
"differentiable": false,
|
15 |
+
"fused": null,
|
16 |
+
"decoupled_weight_decay": true,
|
17 |
+
"params": [
|
18 |
+
0,
|
19 |
+
1,
|
20 |
+
2,
|
21 |
+
3,
|
22 |
+
4,
|
23 |
+
5,
|
24 |
+
6,
|
25 |
+
7,
|
26 |
+
8,
|
27 |
+
9,
|
28 |
+
10,
|
29 |
+
11,
|
30 |
+
12,
|
31 |
+
13,
|
32 |
+
14,
|
33 |
+
15,
|
34 |
+
16,
|
35 |
+
17,
|
36 |
+
18,
|
37 |
+
19,
|
38 |
+
20,
|
39 |
+
21,
|
40 |
+
22,
|
41 |
+
23,
|
42 |
+
24,
|
43 |
+
25,
|
44 |
+
26,
|
45 |
+
27,
|
46 |
+
28,
|
47 |
+
29,
|
48 |
+
30,
|
49 |
+
31,
|
50 |
+
32,
|
51 |
+
33,
|
52 |
+
34,
|
53 |
+
35,
|
54 |
+
36,
|
55 |
+
37,
|
56 |
+
38,
|
57 |
+
39,
|
58 |
+
40,
|
59 |
+
41,
|
60 |
+
42,
|
61 |
+
43,
|
62 |
+
44,
|
63 |
+
45,
|
64 |
+
46,
|
65 |
+
47,
|
66 |
+
48,
|
67 |
+
49,
|
68 |
+
50,
|
69 |
+
51,
|
70 |
+
52,
|
71 |
+
53,
|
72 |
+
54,
|
73 |
+
55,
|
74 |
+
56,
|
75 |
+
57,
|
76 |
+
58,
|
77 |
+
59,
|
78 |
+
60,
|
79 |
+
61,
|
80 |
+
62,
|
81 |
+
63,
|
82 |
+
64,
|
83 |
+
65,
|
84 |
+
66,
|
85 |
+
67,
|
86 |
+
68,
|
87 |
+
69,
|
88 |
+
70,
|
89 |
+
71,
|
90 |
+
72,
|
91 |
+
73,
|
92 |
+
74,
|
93 |
+
75,
|
94 |
+
76,
|
95 |
+
77,
|
96 |
+
78,
|
97 |
+
79,
|
98 |
+
80,
|
99 |
+
81,
|
100 |
+
82,
|
101 |
+
83,
|
102 |
+
84,
|
103 |
+
85,
|
104 |
+
86,
|
105 |
+
87,
|
106 |
+
88,
|
107 |
+
89,
|
108 |
+
90,
|
109 |
+
91,
|
110 |
+
92,
|
111 |
+
93,
|
112 |
+
94,
|
113 |
+
95,
|
114 |
+
96,
|
115 |
+
97,
|
116 |
+
98,
|
117 |
+
99,
|
118 |
+
100,
|
119 |
+
101,
|
120 |
+
102,
|
121 |
+
103,
|
122 |
+
104,
|
123 |
+
105,
|
124 |
+
106,
|
125 |
+
107,
|
126 |
+
108,
|
127 |
+
109,
|
128 |
+
110,
|
129 |
+
111,
|
130 |
+
112,
|
131 |
+
113,
|
132 |
+
114,
|
133 |
+
115,
|
134 |
+
116,
|
135 |
+
117,
|
136 |
+
118,
|
137 |
+
119,
|
138 |
+
120,
|
139 |
+
121,
|
140 |
+
122,
|
141 |
+
123,
|
142 |
+
124,
|
143 |
+
125,
|
144 |
+
126,
|
145 |
+
127,
|
146 |
+
128,
|
147 |
+
129,
|
148 |
+
130,
|
149 |
+
131,
|
150 |
+
132
|
151 |
+
]
|
152 |
+
},
|
153 |
+
{
|
154 |
+
"lr": 1e-05,
|
155 |
+
"betas": [
|
156 |
+
0.9,
|
157 |
+
0.999
|
158 |
+
],
|
159 |
+
"eps": 1e-08,
|
160 |
+
"weight_decay": 0.0001,
|
161 |
+
"amsgrad": false,
|
162 |
+
"maximize": false,
|
163 |
+
"foreach": null,
|
164 |
+
"capturable": false,
|
165 |
+
"differentiable": false,
|
166 |
+
"fused": null,
|
167 |
+
"decoupled_weight_decay": true,
|
168 |
+
"params": [
|
169 |
+
133,
|
170 |
+
134,
|
171 |
+
135,
|
172 |
+
136,
|
173 |
+
137,
|
174 |
+
138,
|
175 |
+
139,
|
176 |
+
140,
|
177 |
+
141,
|
178 |
+
142,
|
179 |
+
143,
|
180 |
+
144,
|
181 |
+
145,
|
182 |
+
146,
|
183 |
+
147,
|
184 |
+
148,
|
185 |
+
149,
|
186 |
+
150,
|
187 |
+
151,
|
188 |
+
152
|
189 |
+
]
|
190 |
+
}
|
191 |
+
]
|
train/act_dice_art_run1/checkpoints/020000/training_state/optimizer_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:12ecb7bf1376a08e87a23aa01f2d51a65997f0bda2a544f19ac22e147babcfe8
|
3 |
+
size 412817652
|
train/act_dice_art_run1/checkpoints/020000/training_state/rng_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f1ba88f4abbb0f4a60c28ce5166e14f1de15326d12c483f0def3d50e7c105a13
|
3 |
+
size 15708
|
train/act_dice_art_run1/checkpoints/020000/training_state/training_step.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"step": 20000
|
3 |
+
}
|
train/act_dice_art_run1/checkpoints/040000/pretrained_model/config.json
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"type": "act",
|
3 |
+
"n_obs_steps": 1,
|
4 |
+
"normalization_mapping": {
|
5 |
+
"VISUAL": "MEAN_STD",
|
6 |
+
"STATE": "MEAN_STD",
|
7 |
+
"ACTION": "MEAN_STD"
|
8 |
+
},
|
9 |
+
"input_features": {
|
10 |
+
"observation.state": {
|
11 |
+
"type": "STATE",
|
12 |
+
"shape": [
|
13 |
+
6
|
14 |
+
]
|
15 |
+
},
|
16 |
+
"observation.images.arm": {
|
17 |
+
"type": "VISUAL",
|
18 |
+
"shape": [
|
19 |
+
3,
|
20 |
+
480,
|
21 |
+
640
|
22 |
+
]
|
23 |
+
},
|
24 |
+
"observation.images.world": {
|
25 |
+
"type": "VISUAL",
|
26 |
+
"shape": [
|
27 |
+
3,
|
28 |
+
480,
|
29 |
+
640
|
30 |
+
]
|
31 |
+
}
|
32 |
+
},
|
33 |
+
"output_features": {
|
34 |
+
"action": {
|
35 |
+
"type": "ACTION",
|
36 |
+
"shape": [
|
37 |
+
6
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"device": "cuda",
|
42 |
+
"use_amp": false,
|
43 |
+
"chunk_size": 100,
|
44 |
+
"n_action_steps": 100,
|
45 |
+
"vision_backbone": "resnet18",
|
46 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
47 |
+
"replace_final_stride_with_dilation": false,
|
48 |
+
"pre_norm": false,
|
49 |
+
"dim_model": 512,
|
50 |
+
"n_heads": 8,
|
51 |
+
"dim_feedforward": 3200,
|
52 |
+
"feedforward_activation": "relu",
|
53 |
+
"n_encoder_layers": 4,
|
54 |
+
"n_decoder_layers": 1,
|
55 |
+
"use_vae": true,
|
56 |
+
"latent_dim": 32,
|
57 |
+
"n_vae_encoder_layers": 4,
|
58 |
+
"temporal_ensemble_coeff": null,
|
59 |
+
"dropout": 0.1,
|
60 |
+
"kl_weight": 10.0,
|
61 |
+
"optimizer_lr": 1e-05,
|
62 |
+
"optimizer_weight_decay": 0.0001,
|
63 |
+
"optimizer_lr_backbone": 1e-05
|
64 |
+
}
|
train/act_dice_art_run1/checkpoints/040000/pretrained_model/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:792c237d0a9e0b491fac49010bb7e7861279cb0420934bb46fdd550f8cf7e6f0
|
3 |
+
size 206701064
|
train/act_dice_art_run1/checkpoints/040000/pretrained_model/train_config.json
ADDED
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset": {
|
3 |
+
"repo_id": "BoredMachinist/dice-art-run1",
|
4 |
+
"root": null,
|
5 |
+
"episodes": null,
|
6 |
+
"image_transforms": {
|
7 |
+
"enable": false,
|
8 |
+
"max_num_transforms": 3,
|
9 |
+
"random_order": false,
|
10 |
+
"tfs": {
|
11 |
+
"brightness": {
|
12 |
+
"weight": 1.0,
|
13 |
+
"type": "ColorJitter",
|
14 |
+
"kwargs": {
|
15 |
+
"brightness": [
|
16 |
+
0.8,
|
17 |
+
1.2
|
18 |
+
]
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"contrast": {
|
22 |
+
"weight": 1.0,
|
23 |
+
"type": "ColorJitter",
|
24 |
+
"kwargs": {
|
25 |
+
"contrast": [
|
26 |
+
0.8,
|
27 |
+
1.2
|
28 |
+
]
|
29 |
+
}
|
30 |
+
},
|
31 |
+
"saturation": {
|
32 |
+
"weight": 1.0,
|
33 |
+
"type": "ColorJitter",
|
34 |
+
"kwargs": {
|
35 |
+
"saturation": [
|
36 |
+
0.5,
|
37 |
+
1.5
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"hue": {
|
42 |
+
"weight": 1.0,
|
43 |
+
"type": "ColorJitter",
|
44 |
+
"kwargs": {
|
45 |
+
"hue": [
|
46 |
+
-0.05,
|
47 |
+
0.05
|
48 |
+
]
|
49 |
+
}
|
50 |
+
},
|
51 |
+
"sharpness": {
|
52 |
+
"weight": 1.0,
|
53 |
+
"type": "SharpnessJitter",
|
54 |
+
"kwargs": {
|
55 |
+
"sharpness": [
|
56 |
+
0.5,
|
57 |
+
1.5
|
58 |
+
]
|
59 |
+
}
|
60 |
+
}
|
61 |
+
}
|
62 |
+
},
|
63 |
+
"revision": null,
|
64 |
+
"use_imagenet_stats": true,
|
65 |
+
"video_backend": "pyav"
|
66 |
+
},
|
67 |
+
"env": null,
|
68 |
+
"policy": {
|
69 |
+
"type": "act",
|
70 |
+
"n_obs_steps": 1,
|
71 |
+
"normalization_mapping": {
|
72 |
+
"VISUAL": "MEAN_STD",
|
73 |
+
"STATE": "MEAN_STD",
|
74 |
+
"ACTION": "MEAN_STD"
|
75 |
+
},
|
76 |
+
"input_features": {
|
77 |
+
"observation.state": {
|
78 |
+
"type": "STATE",
|
79 |
+
"shape": [
|
80 |
+
6
|
81 |
+
]
|
82 |
+
},
|
83 |
+
"observation.images.arm": {
|
84 |
+
"type": "VISUAL",
|
85 |
+
"shape": [
|
86 |
+
3,
|
87 |
+
480,
|
88 |
+
640
|
89 |
+
]
|
90 |
+
},
|
91 |
+
"observation.images.world": {
|
92 |
+
"type": "VISUAL",
|
93 |
+
"shape": [
|
94 |
+
3,
|
95 |
+
480,
|
96 |
+
640
|
97 |
+
]
|
98 |
+
}
|
99 |
+
},
|
100 |
+
"output_features": {
|
101 |
+
"action": {
|
102 |
+
"type": "ACTION",
|
103 |
+
"shape": [
|
104 |
+
6
|
105 |
+
]
|
106 |
+
}
|
107 |
+
},
|
108 |
+
"device": "cuda",
|
109 |
+
"use_amp": false,
|
110 |
+
"chunk_size": 100,
|
111 |
+
"n_action_steps": 100,
|
112 |
+
"vision_backbone": "resnet18",
|
113 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
114 |
+
"replace_final_stride_with_dilation": false,
|
115 |
+
"pre_norm": false,
|
116 |
+
"dim_model": 512,
|
117 |
+
"n_heads": 8,
|
118 |
+
"dim_feedforward": 3200,
|
119 |
+
"feedforward_activation": "relu",
|
120 |
+
"n_encoder_layers": 4,
|
121 |
+
"n_decoder_layers": 1,
|
122 |
+
"use_vae": true,
|
123 |
+
"latent_dim": 32,
|
124 |
+
"n_vae_encoder_layers": 4,
|
125 |
+
"temporal_ensemble_coeff": null,
|
126 |
+
"dropout": 0.1,
|
127 |
+
"kl_weight": 10.0,
|
128 |
+
"optimizer_lr": 1e-05,
|
129 |
+
"optimizer_weight_decay": 0.0001,
|
130 |
+
"optimizer_lr_backbone": 1e-05
|
131 |
+
},
|
132 |
+
"output_dir": "outputs/train/act_dice_art_run1",
|
133 |
+
"job_name": "act_dice_art_run1",
|
134 |
+
"resume": false,
|
135 |
+
"seed": 1000,
|
136 |
+
"num_workers": 4,
|
137 |
+
"batch_size": 8,
|
138 |
+
"steps": 100000,
|
139 |
+
"eval_freq": 20000,
|
140 |
+
"log_freq": 200,
|
141 |
+
"save_checkpoint": true,
|
142 |
+
"save_freq": 20000,
|
143 |
+
"use_policy_training_preset": true,
|
144 |
+
"optimizer": {
|
145 |
+
"type": "adamw",
|
146 |
+
"lr": 1e-05,
|
147 |
+
"weight_decay": 0.0001,
|
148 |
+
"grad_clip_norm": 10.0,
|
149 |
+
"betas": [
|
150 |
+
0.9,
|
151 |
+
0.999
|
152 |
+
],
|
153 |
+
"eps": 1e-08
|
154 |
+
},
|
155 |
+
"scheduler": null,
|
156 |
+
"eval": {
|
157 |
+
"n_episodes": 50,
|
158 |
+
"batch_size": 50,
|
159 |
+
"use_async_envs": false
|
160 |
+
},
|
161 |
+
"wandb": {
|
162 |
+
"enable": true,
|
163 |
+
"disable_artifact": false,
|
164 |
+
"project": "lerobot",
|
165 |
+
"entity": null,
|
166 |
+
"notes": null,
|
167 |
+
"run_id": "fhdgxxjz",
|
168 |
+
"mode": null
|
169 |
+
}
|
170 |
+
}
|
train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_param_groups.json
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"lr": 1e-05,
|
4 |
+
"betas": [
|
5 |
+
0.9,
|
6 |
+
0.999
|
7 |
+
],
|
8 |
+
"eps": 1e-08,
|
9 |
+
"weight_decay": 0.0001,
|
10 |
+
"amsgrad": false,
|
11 |
+
"maximize": false,
|
12 |
+
"foreach": null,
|
13 |
+
"capturable": false,
|
14 |
+
"differentiable": false,
|
15 |
+
"fused": null,
|
16 |
+
"decoupled_weight_decay": true,
|
17 |
+
"params": [
|
18 |
+
0,
|
19 |
+
1,
|
20 |
+
2,
|
21 |
+
3,
|
22 |
+
4,
|
23 |
+
5,
|
24 |
+
6,
|
25 |
+
7,
|
26 |
+
8,
|
27 |
+
9,
|
28 |
+
10,
|
29 |
+
11,
|
30 |
+
12,
|
31 |
+
13,
|
32 |
+
14,
|
33 |
+
15,
|
34 |
+
16,
|
35 |
+
17,
|
36 |
+
18,
|
37 |
+
19,
|
38 |
+
20,
|
39 |
+
21,
|
40 |
+
22,
|
41 |
+
23,
|
42 |
+
24,
|
43 |
+
25,
|
44 |
+
26,
|
45 |
+
27,
|
46 |
+
28,
|
47 |
+
29,
|
48 |
+
30,
|
49 |
+
31,
|
50 |
+
32,
|
51 |
+
33,
|
52 |
+
34,
|
53 |
+
35,
|
54 |
+
36,
|
55 |
+
37,
|
56 |
+
38,
|
57 |
+
39,
|
58 |
+
40,
|
59 |
+
41,
|
60 |
+
42,
|
61 |
+
43,
|
62 |
+
44,
|
63 |
+
45,
|
64 |
+
46,
|
65 |
+
47,
|
66 |
+
48,
|
67 |
+
49,
|
68 |
+
50,
|
69 |
+
51,
|
70 |
+
52,
|
71 |
+
53,
|
72 |
+
54,
|
73 |
+
55,
|
74 |
+
56,
|
75 |
+
57,
|
76 |
+
58,
|
77 |
+
59,
|
78 |
+
60,
|
79 |
+
61,
|
80 |
+
62,
|
81 |
+
63,
|
82 |
+
64,
|
83 |
+
65,
|
84 |
+
66,
|
85 |
+
67,
|
86 |
+
68,
|
87 |
+
69,
|
88 |
+
70,
|
89 |
+
71,
|
90 |
+
72,
|
91 |
+
73,
|
92 |
+
74,
|
93 |
+
75,
|
94 |
+
76,
|
95 |
+
77,
|
96 |
+
78,
|
97 |
+
79,
|
98 |
+
80,
|
99 |
+
81,
|
100 |
+
82,
|
101 |
+
83,
|
102 |
+
84,
|
103 |
+
85,
|
104 |
+
86,
|
105 |
+
87,
|
106 |
+
88,
|
107 |
+
89,
|
108 |
+
90,
|
109 |
+
91,
|
110 |
+
92,
|
111 |
+
93,
|
112 |
+
94,
|
113 |
+
95,
|
114 |
+
96,
|
115 |
+
97,
|
116 |
+
98,
|
117 |
+
99,
|
118 |
+
100,
|
119 |
+
101,
|
120 |
+
102,
|
121 |
+
103,
|
122 |
+
104,
|
123 |
+
105,
|
124 |
+
106,
|
125 |
+
107,
|
126 |
+
108,
|
127 |
+
109,
|
128 |
+
110,
|
129 |
+
111,
|
130 |
+
112,
|
131 |
+
113,
|
132 |
+
114,
|
133 |
+
115,
|
134 |
+
116,
|
135 |
+
117,
|
136 |
+
118,
|
137 |
+
119,
|
138 |
+
120,
|
139 |
+
121,
|
140 |
+
122,
|
141 |
+
123,
|
142 |
+
124,
|
143 |
+
125,
|
144 |
+
126,
|
145 |
+
127,
|
146 |
+
128,
|
147 |
+
129,
|
148 |
+
130,
|
149 |
+
131,
|
150 |
+
132
|
151 |
+
]
|
152 |
+
},
|
153 |
+
{
|
154 |
+
"lr": 1e-05,
|
155 |
+
"betas": [
|
156 |
+
0.9,
|
157 |
+
0.999
|
158 |
+
],
|
159 |
+
"eps": 1e-08,
|
160 |
+
"weight_decay": 0.0001,
|
161 |
+
"amsgrad": false,
|
162 |
+
"maximize": false,
|
163 |
+
"foreach": null,
|
164 |
+
"capturable": false,
|
165 |
+
"differentiable": false,
|
166 |
+
"fused": null,
|
167 |
+
"decoupled_weight_decay": true,
|
168 |
+
"params": [
|
169 |
+
133,
|
170 |
+
134,
|
171 |
+
135,
|
172 |
+
136,
|
173 |
+
137,
|
174 |
+
138,
|
175 |
+
139,
|
176 |
+
140,
|
177 |
+
141,
|
178 |
+
142,
|
179 |
+
143,
|
180 |
+
144,
|
181 |
+
145,
|
182 |
+
146,
|
183 |
+
147,
|
184 |
+
148,
|
185 |
+
149,
|
186 |
+
150,
|
187 |
+
151,
|
188 |
+
152
|
189 |
+
]
|
190 |
+
}
|
191 |
+
]
|
train/act_dice_art_run1/checkpoints/040000/training_state/optimizer_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:746228ecd60295447cdae7097efd335d0499dbb811eb7c26d694e61c9e937598
|
3 |
+
size 412817652
|
train/act_dice_art_run1/checkpoints/040000/training_state/rng_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:88ccf929509382665c94e8858d70d7af7646ca677014ead574a2440444698955
|
3 |
+
size 15708
|
train/act_dice_art_run1/checkpoints/040000/training_state/training_step.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"step": 40000
|
3 |
+
}
|
train/act_dice_art_run1/checkpoints/060000/pretrained_model/config.json
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"type": "act",
|
3 |
+
"n_obs_steps": 1,
|
4 |
+
"normalization_mapping": {
|
5 |
+
"VISUAL": "MEAN_STD",
|
6 |
+
"STATE": "MEAN_STD",
|
7 |
+
"ACTION": "MEAN_STD"
|
8 |
+
},
|
9 |
+
"input_features": {
|
10 |
+
"observation.state": {
|
11 |
+
"type": "STATE",
|
12 |
+
"shape": [
|
13 |
+
6
|
14 |
+
]
|
15 |
+
},
|
16 |
+
"observation.images.arm": {
|
17 |
+
"type": "VISUAL",
|
18 |
+
"shape": [
|
19 |
+
3,
|
20 |
+
480,
|
21 |
+
640
|
22 |
+
]
|
23 |
+
},
|
24 |
+
"observation.images.world": {
|
25 |
+
"type": "VISUAL",
|
26 |
+
"shape": [
|
27 |
+
3,
|
28 |
+
480,
|
29 |
+
640
|
30 |
+
]
|
31 |
+
}
|
32 |
+
},
|
33 |
+
"output_features": {
|
34 |
+
"action": {
|
35 |
+
"type": "ACTION",
|
36 |
+
"shape": [
|
37 |
+
6
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"device": "cuda",
|
42 |
+
"use_amp": false,
|
43 |
+
"chunk_size": 100,
|
44 |
+
"n_action_steps": 100,
|
45 |
+
"vision_backbone": "resnet18",
|
46 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
47 |
+
"replace_final_stride_with_dilation": 0,
|
48 |
+
"pre_norm": false,
|
49 |
+
"dim_model": 512,
|
50 |
+
"n_heads": 8,
|
51 |
+
"dim_feedforward": 3200,
|
52 |
+
"feedforward_activation": "relu",
|
53 |
+
"n_encoder_layers": 4,
|
54 |
+
"n_decoder_layers": 1,
|
55 |
+
"use_vae": true,
|
56 |
+
"latent_dim": 32,
|
57 |
+
"n_vae_encoder_layers": 4,
|
58 |
+
"temporal_ensemble_coeff": null,
|
59 |
+
"dropout": 0.1,
|
60 |
+
"kl_weight": 10.0,
|
61 |
+
"optimizer_lr": 1e-05,
|
62 |
+
"optimizer_weight_decay": 0.0001,
|
63 |
+
"optimizer_lr_backbone": 1e-05
|
64 |
+
}
|
train/act_dice_art_run1/checkpoints/060000/pretrained_model/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a83d53866360c1dbf05f9a3ebd73d3ab77dadfbee6dcca054c480b5bb4b83524
|
3 |
+
size 206701064
|
train/act_dice_art_run1/checkpoints/060000/pretrained_model/train_config.json
ADDED
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset": {
|
3 |
+
"repo_id": "BoredMachinist/dice-art-run1",
|
4 |
+
"root": null,
|
5 |
+
"episodes": null,
|
6 |
+
"image_transforms": {
|
7 |
+
"enable": false,
|
8 |
+
"max_num_transforms": 3,
|
9 |
+
"random_order": false,
|
10 |
+
"tfs": {
|
11 |
+
"brightness": {
|
12 |
+
"weight": 1.0,
|
13 |
+
"type": "ColorJitter",
|
14 |
+
"kwargs": {
|
15 |
+
"brightness": [
|
16 |
+
0.8,
|
17 |
+
1.2
|
18 |
+
]
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"contrast": {
|
22 |
+
"weight": 1.0,
|
23 |
+
"type": "ColorJitter",
|
24 |
+
"kwargs": {
|
25 |
+
"contrast": [
|
26 |
+
0.8,
|
27 |
+
1.2
|
28 |
+
]
|
29 |
+
}
|
30 |
+
},
|
31 |
+
"saturation": {
|
32 |
+
"weight": 1.0,
|
33 |
+
"type": "ColorJitter",
|
34 |
+
"kwargs": {
|
35 |
+
"saturation": [
|
36 |
+
0.5,
|
37 |
+
1.5
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"hue": {
|
42 |
+
"weight": 1.0,
|
43 |
+
"type": "ColorJitter",
|
44 |
+
"kwargs": {
|
45 |
+
"hue": [
|
46 |
+
-0.05,
|
47 |
+
0.05
|
48 |
+
]
|
49 |
+
}
|
50 |
+
},
|
51 |
+
"sharpness": {
|
52 |
+
"weight": 1.0,
|
53 |
+
"type": "SharpnessJitter",
|
54 |
+
"kwargs": {
|
55 |
+
"sharpness": [
|
56 |
+
0.5,
|
57 |
+
1.5
|
58 |
+
]
|
59 |
+
}
|
60 |
+
}
|
61 |
+
}
|
62 |
+
},
|
63 |
+
"revision": null,
|
64 |
+
"use_imagenet_stats": true,
|
65 |
+
"video_backend": "pyav"
|
66 |
+
},
|
67 |
+
"env": null,
|
68 |
+
"policy": {
|
69 |
+
"type": "act",
|
70 |
+
"n_obs_steps": 1,
|
71 |
+
"normalization_mapping": {
|
72 |
+
"VISUAL": "MEAN_STD",
|
73 |
+
"STATE": "MEAN_STD",
|
74 |
+
"ACTION": "MEAN_STD"
|
75 |
+
},
|
76 |
+
"input_features": {
|
77 |
+
"observation.state": {
|
78 |
+
"type": "STATE",
|
79 |
+
"shape": [
|
80 |
+
6
|
81 |
+
]
|
82 |
+
},
|
83 |
+
"observation.images.arm": {
|
84 |
+
"type": "VISUAL",
|
85 |
+
"shape": [
|
86 |
+
3,
|
87 |
+
480,
|
88 |
+
640
|
89 |
+
]
|
90 |
+
},
|
91 |
+
"observation.images.world": {
|
92 |
+
"type": "VISUAL",
|
93 |
+
"shape": [
|
94 |
+
3,
|
95 |
+
480,
|
96 |
+
640
|
97 |
+
]
|
98 |
+
}
|
99 |
+
},
|
100 |
+
"output_features": {
|
101 |
+
"action": {
|
102 |
+
"type": "ACTION",
|
103 |
+
"shape": [
|
104 |
+
6
|
105 |
+
]
|
106 |
+
}
|
107 |
+
},
|
108 |
+
"device": "cuda",
|
109 |
+
"use_amp": false,
|
110 |
+
"chunk_size": 100,
|
111 |
+
"n_action_steps": 100,
|
112 |
+
"vision_backbone": "resnet18",
|
113 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
114 |
+
"replace_final_stride_with_dilation": 0,
|
115 |
+
"pre_norm": false,
|
116 |
+
"dim_model": 512,
|
117 |
+
"n_heads": 8,
|
118 |
+
"dim_feedforward": 3200,
|
119 |
+
"feedforward_activation": "relu",
|
120 |
+
"n_encoder_layers": 4,
|
121 |
+
"n_decoder_layers": 1,
|
122 |
+
"use_vae": true,
|
123 |
+
"latent_dim": 32,
|
124 |
+
"n_vae_encoder_layers": 4,
|
125 |
+
"temporal_ensemble_coeff": null,
|
126 |
+
"dropout": 0.1,
|
127 |
+
"kl_weight": 10.0,
|
128 |
+
"optimizer_lr": 1e-05,
|
129 |
+
"optimizer_weight_decay": 0.0001,
|
130 |
+
"optimizer_lr_backbone": 1e-05
|
131 |
+
},
|
132 |
+
"output_dir": "outputs/train/act_dice_art_run1",
|
133 |
+
"job_name": "act_dice_art_run1",
|
134 |
+
"resume": true,
|
135 |
+
"seed": 1000,
|
136 |
+
"num_workers": 4,
|
137 |
+
"batch_size": 8,
|
138 |
+
"steps": 100000,
|
139 |
+
"eval_freq": 20000,
|
140 |
+
"log_freq": 200,
|
141 |
+
"save_checkpoint": true,
|
142 |
+
"save_freq": 20000,
|
143 |
+
"use_policy_training_preset": true,
|
144 |
+
"optimizer": {
|
145 |
+
"type": "adamw",
|
146 |
+
"lr": 1e-05,
|
147 |
+
"weight_decay": 0.0001,
|
148 |
+
"grad_clip_norm": 10.0,
|
149 |
+
"betas": [
|
150 |
+
0.9,
|
151 |
+
0.999
|
152 |
+
],
|
153 |
+
"eps": 1e-08
|
154 |
+
},
|
155 |
+
"scheduler": null,
|
156 |
+
"eval": {
|
157 |
+
"n_episodes": 50,
|
158 |
+
"batch_size": 50,
|
159 |
+
"use_async_envs": false
|
160 |
+
},
|
161 |
+
"wandb": {
|
162 |
+
"enable": true,
|
163 |
+
"disable_artifact": false,
|
164 |
+
"project": "lerobot",
|
165 |
+
"entity": null,
|
166 |
+
"notes": null,
|
167 |
+
"run_id": "fhdgxxjz",
|
168 |
+
"mode": null
|
169 |
+
}
|
170 |
+
}
|
train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_param_groups.json
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"lr": 1e-05,
|
4 |
+
"betas": [
|
5 |
+
0.9,
|
6 |
+
0.999
|
7 |
+
],
|
8 |
+
"eps": 1e-08,
|
9 |
+
"weight_decay": 0.0001,
|
10 |
+
"amsgrad": false,
|
11 |
+
"maximize": false,
|
12 |
+
"foreach": null,
|
13 |
+
"capturable": false,
|
14 |
+
"differentiable": false,
|
15 |
+
"fused": null,
|
16 |
+
"decoupled_weight_decay": true,
|
17 |
+
"params": [
|
18 |
+
0,
|
19 |
+
1,
|
20 |
+
2,
|
21 |
+
3,
|
22 |
+
4,
|
23 |
+
5,
|
24 |
+
6,
|
25 |
+
7,
|
26 |
+
8,
|
27 |
+
9,
|
28 |
+
10,
|
29 |
+
11,
|
30 |
+
12,
|
31 |
+
13,
|
32 |
+
14,
|
33 |
+
15,
|
34 |
+
16,
|
35 |
+
17,
|
36 |
+
18,
|
37 |
+
19,
|
38 |
+
20,
|
39 |
+
21,
|
40 |
+
22,
|
41 |
+
23,
|
42 |
+
24,
|
43 |
+
25,
|
44 |
+
26,
|
45 |
+
27,
|
46 |
+
28,
|
47 |
+
29,
|
48 |
+
30,
|
49 |
+
31,
|
50 |
+
32,
|
51 |
+
33,
|
52 |
+
34,
|
53 |
+
35,
|
54 |
+
36,
|
55 |
+
37,
|
56 |
+
38,
|
57 |
+
39,
|
58 |
+
40,
|
59 |
+
41,
|
60 |
+
42,
|
61 |
+
43,
|
62 |
+
44,
|
63 |
+
45,
|
64 |
+
46,
|
65 |
+
47,
|
66 |
+
48,
|
67 |
+
49,
|
68 |
+
50,
|
69 |
+
51,
|
70 |
+
52,
|
71 |
+
53,
|
72 |
+
54,
|
73 |
+
55,
|
74 |
+
56,
|
75 |
+
57,
|
76 |
+
58,
|
77 |
+
59,
|
78 |
+
60,
|
79 |
+
61,
|
80 |
+
62,
|
81 |
+
63,
|
82 |
+
64,
|
83 |
+
65,
|
84 |
+
66,
|
85 |
+
67,
|
86 |
+
68,
|
87 |
+
69,
|
88 |
+
70,
|
89 |
+
71,
|
90 |
+
72,
|
91 |
+
73,
|
92 |
+
74,
|
93 |
+
75,
|
94 |
+
76,
|
95 |
+
77,
|
96 |
+
78,
|
97 |
+
79,
|
98 |
+
80,
|
99 |
+
81,
|
100 |
+
82,
|
101 |
+
83,
|
102 |
+
84,
|
103 |
+
85,
|
104 |
+
86,
|
105 |
+
87,
|
106 |
+
88,
|
107 |
+
89,
|
108 |
+
90,
|
109 |
+
91,
|
110 |
+
92,
|
111 |
+
93,
|
112 |
+
94,
|
113 |
+
95,
|
114 |
+
96,
|
115 |
+
97,
|
116 |
+
98,
|
117 |
+
99,
|
118 |
+
100,
|
119 |
+
101,
|
120 |
+
102,
|
121 |
+
103,
|
122 |
+
104,
|
123 |
+
105,
|
124 |
+
106,
|
125 |
+
107,
|
126 |
+
108,
|
127 |
+
109,
|
128 |
+
110,
|
129 |
+
111,
|
130 |
+
112,
|
131 |
+
113,
|
132 |
+
114,
|
133 |
+
115,
|
134 |
+
116,
|
135 |
+
117,
|
136 |
+
118,
|
137 |
+
119,
|
138 |
+
120,
|
139 |
+
121,
|
140 |
+
122,
|
141 |
+
123,
|
142 |
+
124,
|
143 |
+
125,
|
144 |
+
126,
|
145 |
+
127,
|
146 |
+
128,
|
147 |
+
129,
|
148 |
+
130,
|
149 |
+
131,
|
150 |
+
132
|
151 |
+
]
|
152 |
+
},
|
153 |
+
{
|
154 |
+
"lr": 1e-05,
|
155 |
+
"betas": [
|
156 |
+
0.9,
|
157 |
+
0.999
|
158 |
+
],
|
159 |
+
"eps": 1e-08,
|
160 |
+
"weight_decay": 0.0001,
|
161 |
+
"amsgrad": false,
|
162 |
+
"maximize": false,
|
163 |
+
"foreach": null,
|
164 |
+
"capturable": false,
|
165 |
+
"differentiable": false,
|
166 |
+
"fused": null,
|
167 |
+
"decoupled_weight_decay": true,
|
168 |
+
"params": [
|
169 |
+
133,
|
170 |
+
134,
|
171 |
+
135,
|
172 |
+
136,
|
173 |
+
137,
|
174 |
+
138,
|
175 |
+
139,
|
176 |
+
140,
|
177 |
+
141,
|
178 |
+
142,
|
179 |
+
143,
|
180 |
+
144,
|
181 |
+
145,
|
182 |
+
146,
|
183 |
+
147,
|
184 |
+
148,
|
185 |
+
149,
|
186 |
+
150,
|
187 |
+
151,
|
188 |
+
152
|
189 |
+
]
|
190 |
+
}
|
191 |
+
]
|
train/act_dice_art_run1/checkpoints/060000/training_state/optimizer_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:cb566c648187f7eccdbee8d3807fc64c73381d396bbc5554457617b6c290513c
|
3 |
+
size 412817652
|
train/act_dice_art_run1/checkpoints/060000/training_state/rng_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:da386b82654b6929f5157bdf6e8fcef1d9f55e70ddbb0d97a5b6432abd607501
|
3 |
+
size 15708
|
train/act_dice_art_run1/checkpoints/060000/training_state/training_step.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"step": 60000
|
3 |
+
}
|
train/act_dice_art_run1/checkpoints/080000/pretrained_model/config.json
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"type": "act",
|
3 |
+
"n_obs_steps": 1,
|
4 |
+
"normalization_mapping": {
|
5 |
+
"VISUAL": "MEAN_STD",
|
6 |
+
"STATE": "MEAN_STD",
|
7 |
+
"ACTION": "MEAN_STD"
|
8 |
+
},
|
9 |
+
"input_features": {
|
10 |
+
"observation.state": {
|
11 |
+
"type": "STATE",
|
12 |
+
"shape": [
|
13 |
+
6
|
14 |
+
]
|
15 |
+
},
|
16 |
+
"observation.images.arm": {
|
17 |
+
"type": "VISUAL",
|
18 |
+
"shape": [
|
19 |
+
3,
|
20 |
+
480,
|
21 |
+
640
|
22 |
+
]
|
23 |
+
},
|
24 |
+
"observation.images.world": {
|
25 |
+
"type": "VISUAL",
|
26 |
+
"shape": [
|
27 |
+
3,
|
28 |
+
480,
|
29 |
+
640
|
30 |
+
]
|
31 |
+
}
|
32 |
+
},
|
33 |
+
"output_features": {
|
34 |
+
"action": {
|
35 |
+
"type": "ACTION",
|
36 |
+
"shape": [
|
37 |
+
6
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"device": "cuda",
|
42 |
+
"use_amp": false,
|
43 |
+
"chunk_size": 100,
|
44 |
+
"n_action_steps": 100,
|
45 |
+
"vision_backbone": "resnet18",
|
46 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
47 |
+
"replace_final_stride_with_dilation": 0,
|
48 |
+
"pre_norm": false,
|
49 |
+
"dim_model": 512,
|
50 |
+
"n_heads": 8,
|
51 |
+
"dim_feedforward": 3200,
|
52 |
+
"feedforward_activation": "relu",
|
53 |
+
"n_encoder_layers": 4,
|
54 |
+
"n_decoder_layers": 1,
|
55 |
+
"use_vae": true,
|
56 |
+
"latent_dim": 32,
|
57 |
+
"n_vae_encoder_layers": 4,
|
58 |
+
"temporal_ensemble_coeff": null,
|
59 |
+
"dropout": 0.1,
|
60 |
+
"kl_weight": 10.0,
|
61 |
+
"optimizer_lr": 1e-05,
|
62 |
+
"optimizer_weight_decay": 0.0001,
|
63 |
+
"optimizer_lr_backbone": 1e-05
|
64 |
+
}
|
train/act_dice_art_run1/checkpoints/080000/pretrained_model/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:eee379243296b72fed0528a86a041e05dcca13533c0dbd482784ea97da3e543c
|
3 |
+
size 206701064
|
train/act_dice_art_run1/checkpoints/080000/pretrained_model/train_config.json
ADDED
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset": {
|
3 |
+
"repo_id": "BoredMachinist/dice-art-run1",
|
4 |
+
"root": null,
|
5 |
+
"episodes": null,
|
6 |
+
"image_transforms": {
|
7 |
+
"enable": false,
|
8 |
+
"max_num_transforms": 3,
|
9 |
+
"random_order": false,
|
10 |
+
"tfs": {
|
11 |
+
"brightness": {
|
12 |
+
"weight": 1.0,
|
13 |
+
"type": "ColorJitter",
|
14 |
+
"kwargs": {
|
15 |
+
"brightness": [
|
16 |
+
0.8,
|
17 |
+
1.2
|
18 |
+
]
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"contrast": {
|
22 |
+
"weight": 1.0,
|
23 |
+
"type": "ColorJitter",
|
24 |
+
"kwargs": {
|
25 |
+
"contrast": [
|
26 |
+
0.8,
|
27 |
+
1.2
|
28 |
+
]
|
29 |
+
}
|
30 |
+
},
|
31 |
+
"saturation": {
|
32 |
+
"weight": 1.0,
|
33 |
+
"type": "ColorJitter",
|
34 |
+
"kwargs": {
|
35 |
+
"saturation": [
|
36 |
+
0.5,
|
37 |
+
1.5
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"hue": {
|
42 |
+
"weight": 1.0,
|
43 |
+
"type": "ColorJitter",
|
44 |
+
"kwargs": {
|
45 |
+
"hue": [
|
46 |
+
-0.05,
|
47 |
+
0.05
|
48 |
+
]
|
49 |
+
}
|
50 |
+
},
|
51 |
+
"sharpness": {
|
52 |
+
"weight": 1.0,
|
53 |
+
"type": "SharpnessJitter",
|
54 |
+
"kwargs": {
|
55 |
+
"sharpness": [
|
56 |
+
0.5,
|
57 |
+
1.5
|
58 |
+
]
|
59 |
+
}
|
60 |
+
}
|
61 |
+
}
|
62 |
+
},
|
63 |
+
"revision": null,
|
64 |
+
"use_imagenet_stats": true,
|
65 |
+
"video_backend": "pyav"
|
66 |
+
},
|
67 |
+
"env": null,
|
68 |
+
"policy": {
|
69 |
+
"type": "act",
|
70 |
+
"n_obs_steps": 1,
|
71 |
+
"normalization_mapping": {
|
72 |
+
"VISUAL": "MEAN_STD",
|
73 |
+
"STATE": "MEAN_STD",
|
74 |
+
"ACTION": "MEAN_STD"
|
75 |
+
},
|
76 |
+
"input_features": {
|
77 |
+
"observation.state": {
|
78 |
+
"type": "STATE",
|
79 |
+
"shape": [
|
80 |
+
6
|
81 |
+
]
|
82 |
+
},
|
83 |
+
"observation.images.arm": {
|
84 |
+
"type": "VISUAL",
|
85 |
+
"shape": [
|
86 |
+
3,
|
87 |
+
480,
|
88 |
+
640
|
89 |
+
]
|
90 |
+
},
|
91 |
+
"observation.images.world": {
|
92 |
+
"type": "VISUAL",
|
93 |
+
"shape": [
|
94 |
+
3,
|
95 |
+
480,
|
96 |
+
640
|
97 |
+
]
|
98 |
+
}
|
99 |
+
},
|
100 |
+
"output_features": {
|
101 |
+
"action": {
|
102 |
+
"type": "ACTION",
|
103 |
+
"shape": [
|
104 |
+
6
|
105 |
+
]
|
106 |
+
}
|
107 |
+
},
|
108 |
+
"device": "cuda",
|
109 |
+
"use_amp": false,
|
110 |
+
"chunk_size": 100,
|
111 |
+
"n_action_steps": 100,
|
112 |
+
"vision_backbone": "resnet18",
|
113 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
114 |
+
"replace_final_stride_with_dilation": 0,
|
115 |
+
"pre_norm": false,
|
116 |
+
"dim_model": 512,
|
117 |
+
"n_heads": 8,
|
118 |
+
"dim_feedforward": 3200,
|
119 |
+
"feedforward_activation": "relu",
|
120 |
+
"n_encoder_layers": 4,
|
121 |
+
"n_decoder_layers": 1,
|
122 |
+
"use_vae": true,
|
123 |
+
"latent_dim": 32,
|
124 |
+
"n_vae_encoder_layers": 4,
|
125 |
+
"temporal_ensemble_coeff": null,
|
126 |
+
"dropout": 0.1,
|
127 |
+
"kl_weight": 10.0,
|
128 |
+
"optimizer_lr": 1e-05,
|
129 |
+
"optimizer_weight_decay": 0.0001,
|
130 |
+
"optimizer_lr_backbone": 1e-05
|
131 |
+
},
|
132 |
+
"output_dir": "outputs/train/act_dice_art_run1",
|
133 |
+
"job_name": "act_dice_art_run1",
|
134 |
+
"resume": true,
|
135 |
+
"seed": 1000,
|
136 |
+
"num_workers": 4,
|
137 |
+
"batch_size": 8,
|
138 |
+
"steps": 100000,
|
139 |
+
"eval_freq": 20000,
|
140 |
+
"log_freq": 200,
|
141 |
+
"save_checkpoint": true,
|
142 |
+
"save_freq": 20000,
|
143 |
+
"use_policy_training_preset": true,
|
144 |
+
"optimizer": {
|
145 |
+
"type": "adamw",
|
146 |
+
"lr": 1e-05,
|
147 |
+
"weight_decay": 0.0001,
|
148 |
+
"grad_clip_norm": 10.0,
|
149 |
+
"betas": [
|
150 |
+
0.9,
|
151 |
+
0.999
|
152 |
+
],
|
153 |
+
"eps": 1e-08
|
154 |
+
},
|
155 |
+
"scheduler": null,
|
156 |
+
"eval": {
|
157 |
+
"n_episodes": 50,
|
158 |
+
"batch_size": 50,
|
159 |
+
"use_async_envs": false
|
160 |
+
},
|
161 |
+
"wandb": {
|
162 |
+
"enable": true,
|
163 |
+
"disable_artifact": false,
|
164 |
+
"project": "lerobot",
|
165 |
+
"entity": null,
|
166 |
+
"notes": null,
|
167 |
+
"run_id": "fhdgxxjz",
|
168 |
+
"mode": null
|
169 |
+
}
|
170 |
+
}
|
train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_param_groups.json
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"lr": 1e-05,
|
4 |
+
"betas": [
|
5 |
+
0.9,
|
6 |
+
0.999
|
7 |
+
],
|
8 |
+
"eps": 1e-08,
|
9 |
+
"weight_decay": 0.0001,
|
10 |
+
"amsgrad": false,
|
11 |
+
"maximize": false,
|
12 |
+
"foreach": null,
|
13 |
+
"capturable": false,
|
14 |
+
"differentiable": false,
|
15 |
+
"fused": null,
|
16 |
+
"decoupled_weight_decay": true,
|
17 |
+
"params": [
|
18 |
+
0,
|
19 |
+
1,
|
20 |
+
2,
|
21 |
+
3,
|
22 |
+
4,
|
23 |
+
5,
|
24 |
+
6,
|
25 |
+
7,
|
26 |
+
8,
|
27 |
+
9,
|
28 |
+
10,
|
29 |
+
11,
|
30 |
+
12,
|
31 |
+
13,
|
32 |
+
14,
|
33 |
+
15,
|
34 |
+
16,
|
35 |
+
17,
|
36 |
+
18,
|
37 |
+
19,
|
38 |
+
20,
|
39 |
+
21,
|
40 |
+
22,
|
41 |
+
23,
|
42 |
+
24,
|
43 |
+
25,
|
44 |
+
26,
|
45 |
+
27,
|
46 |
+
28,
|
47 |
+
29,
|
48 |
+
30,
|
49 |
+
31,
|
50 |
+
32,
|
51 |
+
33,
|
52 |
+
34,
|
53 |
+
35,
|
54 |
+
36,
|
55 |
+
37,
|
56 |
+
38,
|
57 |
+
39,
|
58 |
+
40,
|
59 |
+
41,
|
60 |
+
42,
|
61 |
+
43,
|
62 |
+
44,
|
63 |
+
45,
|
64 |
+
46,
|
65 |
+
47,
|
66 |
+
48,
|
67 |
+
49,
|
68 |
+
50,
|
69 |
+
51,
|
70 |
+
52,
|
71 |
+
53,
|
72 |
+
54,
|
73 |
+
55,
|
74 |
+
56,
|
75 |
+
57,
|
76 |
+
58,
|
77 |
+
59,
|
78 |
+
60,
|
79 |
+
61,
|
80 |
+
62,
|
81 |
+
63,
|
82 |
+
64,
|
83 |
+
65,
|
84 |
+
66,
|
85 |
+
67,
|
86 |
+
68,
|
87 |
+
69,
|
88 |
+
70,
|
89 |
+
71,
|
90 |
+
72,
|
91 |
+
73,
|
92 |
+
74,
|
93 |
+
75,
|
94 |
+
76,
|
95 |
+
77,
|
96 |
+
78,
|
97 |
+
79,
|
98 |
+
80,
|
99 |
+
81,
|
100 |
+
82,
|
101 |
+
83,
|
102 |
+
84,
|
103 |
+
85,
|
104 |
+
86,
|
105 |
+
87,
|
106 |
+
88,
|
107 |
+
89,
|
108 |
+
90,
|
109 |
+
91,
|
110 |
+
92,
|
111 |
+
93,
|
112 |
+
94,
|
113 |
+
95,
|
114 |
+
96,
|
115 |
+
97,
|
116 |
+
98,
|
117 |
+
99,
|
118 |
+
100,
|
119 |
+
101,
|
120 |
+
102,
|
121 |
+
103,
|
122 |
+
104,
|
123 |
+
105,
|
124 |
+
106,
|
125 |
+
107,
|
126 |
+
108,
|
127 |
+
109,
|
128 |
+
110,
|
129 |
+
111,
|
130 |
+
112,
|
131 |
+
113,
|
132 |
+
114,
|
133 |
+
115,
|
134 |
+
116,
|
135 |
+
117,
|
136 |
+
118,
|
137 |
+
119,
|
138 |
+
120,
|
139 |
+
121,
|
140 |
+
122,
|
141 |
+
123,
|
142 |
+
124,
|
143 |
+
125,
|
144 |
+
126,
|
145 |
+
127,
|
146 |
+
128,
|
147 |
+
129,
|
148 |
+
130,
|
149 |
+
131,
|
150 |
+
132
|
151 |
+
]
|
152 |
+
},
|
153 |
+
{
|
154 |
+
"lr": 1e-05,
|
155 |
+
"betas": [
|
156 |
+
0.9,
|
157 |
+
0.999
|
158 |
+
],
|
159 |
+
"eps": 1e-08,
|
160 |
+
"weight_decay": 0.0001,
|
161 |
+
"amsgrad": false,
|
162 |
+
"maximize": false,
|
163 |
+
"foreach": null,
|
164 |
+
"capturable": false,
|
165 |
+
"differentiable": false,
|
166 |
+
"fused": null,
|
167 |
+
"decoupled_weight_decay": true,
|
168 |
+
"params": [
|
169 |
+
133,
|
170 |
+
134,
|
171 |
+
135,
|
172 |
+
136,
|
173 |
+
137,
|
174 |
+
138,
|
175 |
+
139,
|
176 |
+
140,
|
177 |
+
141,
|
178 |
+
142,
|
179 |
+
143,
|
180 |
+
144,
|
181 |
+
145,
|
182 |
+
146,
|
183 |
+
147,
|
184 |
+
148,
|
185 |
+
149,
|
186 |
+
150,
|
187 |
+
151,
|
188 |
+
152
|
189 |
+
]
|
190 |
+
}
|
191 |
+
]
|
train/act_dice_art_run1/checkpoints/080000/training_state/optimizer_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f39a5ae696e01a4b70ac917d1295cef22a61c3f432b5943573aa4de5b89d48ca
|
3 |
+
size 412817652
|
train/act_dice_art_run1/checkpoints/080000/training_state/rng_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b0d91358c7f3d433e71afcdfebe0a2c8a4d876563ececf5fd9939d8e950b4ca8
|
3 |
+
size 15708
|
train/act_dice_art_run1/checkpoints/080000/training_state/training_step.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"step": 80000
|
3 |
+
}
|
train/act_dice_art_run1/checkpoints/100000/pretrained_model/config.json
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"type": "act",
|
3 |
+
"n_obs_steps": 1,
|
4 |
+
"normalization_mapping": {
|
5 |
+
"VISUAL": "MEAN_STD",
|
6 |
+
"STATE": "MEAN_STD",
|
7 |
+
"ACTION": "MEAN_STD"
|
8 |
+
},
|
9 |
+
"input_features": {
|
10 |
+
"observation.state": {
|
11 |
+
"type": "STATE",
|
12 |
+
"shape": [
|
13 |
+
6
|
14 |
+
]
|
15 |
+
},
|
16 |
+
"observation.images.arm": {
|
17 |
+
"type": "VISUAL",
|
18 |
+
"shape": [
|
19 |
+
3,
|
20 |
+
480,
|
21 |
+
640
|
22 |
+
]
|
23 |
+
},
|
24 |
+
"observation.images.world": {
|
25 |
+
"type": "VISUAL",
|
26 |
+
"shape": [
|
27 |
+
3,
|
28 |
+
480,
|
29 |
+
640
|
30 |
+
]
|
31 |
+
}
|
32 |
+
},
|
33 |
+
"output_features": {
|
34 |
+
"action": {
|
35 |
+
"type": "ACTION",
|
36 |
+
"shape": [
|
37 |
+
6
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"device": "cuda",
|
42 |
+
"use_amp": false,
|
43 |
+
"chunk_size": 100,
|
44 |
+
"n_action_steps": 100,
|
45 |
+
"vision_backbone": "resnet18",
|
46 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
47 |
+
"replace_final_stride_with_dilation": 0,
|
48 |
+
"pre_norm": false,
|
49 |
+
"dim_model": 512,
|
50 |
+
"n_heads": 8,
|
51 |
+
"dim_feedforward": 3200,
|
52 |
+
"feedforward_activation": "relu",
|
53 |
+
"n_encoder_layers": 4,
|
54 |
+
"n_decoder_layers": 1,
|
55 |
+
"use_vae": true,
|
56 |
+
"latent_dim": 32,
|
57 |
+
"n_vae_encoder_layers": 4,
|
58 |
+
"temporal_ensemble_coeff": null,
|
59 |
+
"dropout": 0.1,
|
60 |
+
"kl_weight": 10.0,
|
61 |
+
"optimizer_lr": 1e-05,
|
62 |
+
"optimizer_weight_decay": 0.0001,
|
63 |
+
"optimizer_lr_backbone": 1e-05
|
64 |
+
}
|
train/act_dice_art_run1/checkpoints/100000/pretrained_model/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c2a9f83a2296309d6e87d2f62b202909cfc48794f29b82cf943333c0c5211945
|
3 |
+
size 206701064
|
train/act_dice_art_run1/checkpoints/100000/pretrained_model/train_config.json
ADDED
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset": {
|
3 |
+
"repo_id": "BoredMachinist/dice-art-run1",
|
4 |
+
"root": null,
|
5 |
+
"episodes": null,
|
6 |
+
"image_transforms": {
|
7 |
+
"enable": false,
|
8 |
+
"max_num_transforms": 3,
|
9 |
+
"random_order": false,
|
10 |
+
"tfs": {
|
11 |
+
"brightness": {
|
12 |
+
"weight": 1.0,
|
13 |
+
"type": "ColorJitter",
|
14 |
+
"kwargs": {
|
15 |
+
"brightness": [
|
16 |
+
0.8,
|
17 |
+
1.2
|
18 |
+
]
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"contrast": {
|
22 |
+
"weight": 1.0,
|
23 |
+
"type": "ColorJitter",
|
24 |
+
"kwargs": {
|
25 |
+
"contrast": [
|
26 |
+
0.8,
|
27 |
+
1.2
|
28 |
+
]
|
29 |
+
}
|
30 |
+
},
|
31 |
+
"saturation": {
|
32 |
+
"weight": 1.0,
|
33 |
+
"type": "ColorJitter",
|
34 |
+
"kwargs": {
|
35 |
+
"saturation": [
|
36 |
+
0.5,
|
37 |
+
1.5
|
38 |
+
]
|
39 |
+
}
|
40 |
+
},
|
41 |
+
"hue": {
|
42 |
+
"weight": 1.0,
|
43 |
+
"type": "ColorJitter",
|
44 |
+
"kwargs": {
|
45 |
+
"hue": [
|
46 |
+
-0.05,
|
47 |
+
0.05
|
48 |
+
]
|
49 |
+
}
|
50 |
+
},
|
51 |
+
"sharpness": {
|
52 |
+
"weight": 1.0,
|
53 |
+
"type": "SharpnessJitter",
|
54 |
+
"kwargs": {
|
55 |
+
"sharpness": [
|
56 |
+
0.5,
|
57 |
+
1.5
|
58 |
+
]
|
59 |
+
}
|
60 |
+
}
|
61 |
+
}
|
62 |
+
},
|
63 |
+
"revision": null,
|
64 |
+
"use_imagenet_stats": true,
|
65 |
+
"video_backend": "pyav"
|
66 |
+
},
|
67 |
+
"env": null,
|
68 |
+
"policy": {
|
69 |
+
"type": "act",
|
70 |
+
"n_obs_steps": 1,
|
71 |
+
"normalization_mapping": {
|
72 |
+
"VISUAL": "MEAN_STD",
|
73 |
+
"STATE": "MEAN_STD",
|
74 |
+
"ACTION": "MEAN_STD"
|
75 |
+
},
|
76 |
+
"input_features": {
|
77 |
+
"observation.state": {
|
78 |
+
"type": "STATE",
|
79 |
+
"shape": [
|
80 |
+
6
|
81 |
+
]
|
82 |
+
},
|
83 |
+
"observation.images.arm": {
|
84 |
+
"type": "VISUAL",
|
85 |
+
"shape": [
|
86 |
+
3,
|
87 |
+
480,
|
88 |
+
640
|
89 |
+
]
|
90 |
+
},
|
91 |
+
"observation.images.world": {
|
92 |
+
"type": "VISUAL",
|
93 |
+
"shape": [
|
94 |
+
3,
|
95 |
+
480,
|
96 |
+
640
|
97 |
+
]
|
98 |
+
}
|
99 |
+
},
|
100 |
+
"output_features": {
|
101 |
+
"action": {
|
102 |
+
"type": "ACTION",
|
103 |
+
"shape": [
|
104 |
+
6
|
105 |
+
]
|
106 |
+
}
|
107 |
+
},
|
108 |
+
"device": "cuda",
|
109 |
+
"use_amp": false,
|
110 |
+
"chunk_size": 100,
|
111 |
+
"n_action_steps": 100,
|
112 |
+
"vision_backbone": "resnet18",
|
113 |
+
"pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
|
114 |
+
"replace_final_stride_with_dilation": 0,
|
115 |
+
"pre_norm": false,
|
116 |
+
"dim_model": 512,
|
117 |
+
"n_heads": 8,
|
118 |
+
"dim_feedforward": 3200,
|
119 |
+
"feedforward_activation": "relu",
|
120 |
+
"n_encoder_layers": 4,
|
121 |
+
"n_decoder_layers": 1,
|
122 |
+
"use_vae": true,
|
123 |
+
"latent_dim": 32,
|
124 |
+
"n_vae_encoder_layers": 4,
|
125 |
+
"temporal_ensemble_coeff": null,
|
126 |
+
"dropout": 0.1,
|
127 |
+
"kl_weight": 10.0,
|
128 |
+
"optimizer_lr": 1e-05,
|
129 |
+
"optimizer_weight_decay": 0.0001,
|
130 |
+
"optimizer_lr_backbone": 1e-05
|
131 |
+
},
|
132 |
+
"output_dir": "outputs/train/act_dice_art_run1",
|
133 |
+
"job_name": "act_dice_art_run1",
|
134 |
+
"resume": true,
|
135 |
+
"seed": 1000,
|
136 |
+
"num_workers": 4,
|
137 |
+
"batch_size": 8,
|
138 |
+
"steps": 100000,
|
139 |
+
"eval_freq": 20000,
|
140 |
+
"log_freq": 200,
|
141 |
+
"save_checkpoint": true,
|
142 |
+
"save_freq": 20000,
|
143 |
+
"use_policy_training_preset": true,
|
144 |
+
"optimizer": {
|
145 |
+
"type": "adamw",
|
146 |
+
"lr": 1e-05,
|
147 |
+
"weight_decay": 0.0001,
|
148 |
+
"grad_clip_norm": 10.0,
|
149 |
+
"betas": [
|
150 |
+
0.9,
|
151 |
+
0.999
|
152 |
+
],
|
153 |
+
"eps": 1e-08
|
154 |
+
},
|
155 |
+
"scheduler": null,
|
156 |
+
"eval": {
|
157 |
+
"n_episodes": 50,
|
158 |
+
"batch_size": 50,
|
159 |
+
"use_async_envs": false
|
160 |
+
},
|
161 |
+
"wandb": {
|
162 |
+
"enable": true,
|
163 |
+
"disable_artifact": false,
|
164 |
+
"project": "lerobot",
|
165 |
+
"entity": null,
|
166 |
+
"notes": null,
|
167 |
+
"run_id": "fhdgxxjz",
|
168 |
+
"mode": null
|
169 |
+
}
|
170 |
+
}
|
train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_param_groups.json
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[
|
2 |
+
{
|
3 |
+
"lr": 1e-05,
|
4 |
+
"betas": [
|
5 |
+
0.9,
|
6 |
+
0.999
|
7 |
+
],
|
8 |
+
"eps": 1e-08,
|
9 |
+
"weight_decay": 0.0001,
|
10 |
+
"amsgrad": false,
|
11 |
+
"maximize": false,
|
12 |
+
"foreach": null,
|
13 |
+
"capturable": false,
|
14 |
+
"differentiable": false,
|
15 |
+
"fused": null,
|
16 |
+
"decoupled_weight_decay": true,
|
17 |
+
"params": [
|
18 |
+
0,
|
19 |
+
1,
|
20 |
+
2,
|
21 |
+
3,
|
22 |
+
4,
|
23 |
+
5,
|
24 |
+
6,
|
25 |
+
7,
|
26 |
+
8,
|
27 |
+
9,
|
28 |
+
10,
|
29 |
+
11,
|
30 |
+
12,
|
31 |
+
13,
|
32 |
+
14,
|
33 |
+
15,
|
34 |
+
16,
|
35 |
+
17,
|
36 |
+
18,
|
37 |
+
19,
|
38 |
+
20,
|
39 |
+
21,
|
40 |
+
22,
|
41 |
+
23,
|
42 |
+
24,
|
43 |
+
25,
|
44 |
+
26,
|
45 |
+
27,
|
46 |
+
28,
|
47 |
+
29,
|
48 |
+
30,
|
49 |
+
31,
|
50 |
+
32,
|
51 |
+
33,
|
52 |
+
34,
|
53 |
+
35,
|
54 |
+
36,
|
55 |
+
37,
|
56 |
+
38,
|
57 |
+
39,
|
58 |
+
40,
|
59 |
+
41,
|
60 |
+
42,
|
61 |
+
43,
|
62 |
+
44,
|
63 |
+
45,
|
64 |
+
46,
|
65 |
+
47,
|
66 |
+
48,
|
67 |
+
49,
|
68 |
+
50,
|
69 |
+
51,
|
70 |
+
52,
|
71 |
+
53,
|
72 |
+
54,
|
73 |
+
55,
|
74 |
+
56,
|
75 |
+
57,
|
76 |
+
58,
|
77 |
+
59,
|
78 |
+
60,
|
79 |
+
61,
|
80 |
+
62,
|
81 |
+
63,
|
82 |
+
64,
|
83 |
+
65,
|
84 |
+
66,
|
85 |
+
67,
|
86 |
+
68,
|
87 |
+
69,
|
88 |
+
70,
|
89 |
+
71,
|
90 |
+
72,
|
91 |
+
73,
|
92 |
+
74,
|
93 |
+
75,
|
94 |
+
76,
|
95 |
+
77,
|
96 |
+
78,
|
97 |
+
79,
|
98 |
+
80,
|
99 |
+
81,
|
100 |
+
82,
|
101 |
+
83,
|
102 |
+
84,
|
103 |
+
85,
|
104 |
+
86,
|
105 |
+
87,
|
106 |
+
88,
|
107 |
+
89,
|
108 |
+
90,
|
109 |
+
91,
|
110 |
+
92,
|
111 |
+
93,
|
112 |
+
94,
|
113 |
+
95,
|
114 |
+
96,
|
115 |
+
97,
|
116 |
+
98,
|
117 |
+
99,
|
118 |
+
100,
|
119 |
+
101,
|
120 |
+
102,
|
121 |
+
103,
|
122 |
+
104,
|
123 |
+
105,
|
124 |
+
106,
|
125 |
+
107,
|
126 |
+
108,
|
127 |
+
109,
|
128 |
+
110,
|
129 |
+
111,
|
130 |
+
112,
|
131 |
+
113,
|
132 |
+
114,
|
133 |
+
115,
|
134 |
+
116,
|
135 |
+
117,
|
136 |
+
118,
|
137 |
+
119,
|
138 |
+
120,
|
139 |
+
121,
|
140 |
+
122,
|
141 |
+
123,
|
142 |
+
124,
|
143 |
+
125,
|
144 |
+
126,
|
145 |
+
127,
|
146 |
+
128,
|
147 |
+
129,
|
148 |
+
130,
|
149 |
+
131,
|
150 |
+
132
|
151 |
+
]
|
152 |
+
},
|
153 |
+
{
|
154 |
+
"lr": 1e-05,
|
155 |
+
"betas": [
|
156 |
+
0.9,
|
157 |
+
0.999
|
158 |
+
],
|
159 |
+
"eps": 1e-08,
|
160 |
+
"weight_decay": 0.0001,
|
161 |
+
"amsgrad": false,
|
162 |
+
"maximize": false,
|
163 |
+
"foreach": null,
|
164 |
+
"capturable": false,
|
165 |
+
"differentiable": false,
|
166 |
+
"fused": null,
|
167 |
+
"decoupled_weight_decay": true,
|
168 |
+
"params": [
|
169 |
+
133,
|
170 |
+
134,
|
171 |
+
135,
|
172 |
+
136,
|
173 |
+
137,
|
174 |
+
138,
|
175 |
+
139,
|
176 |
+
140,
|
177 |
+
141,
|
178 |
+
142,
|
179 |
+
143,
|
180 |
+
144,
|
181 |
+
145,
|
182 |
+
146,
|
183 |
+
147,
|
184 |
+
148,
|
185 |
+
149,
|
186 |
+
150,
|
187 |
+
151,
|
188 |
+
152
|
189 |
+
]
|
190 |
+
}
|
191 |
+
]
|
train/act_dice_art_run1/checkpoints/100000/training_state/optimizer_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a44c4e5523045bb226a12412edb707541d251e609999ad307432db6ec677981c
|
3 |
+
size 412817652
|
train/act_dice_art_run1/checkpoints/100000/training_state/rng_state.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6edd2b0eb5ffb0556ac59e5b8b208d9ec2a740d79c3c8bc18c8d0fc37afcde74
|
3 |
+
size 15708
|
train/act_dice_art_run1/checkpoints/100000/training_state/training_step.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"step": 100000
|
3 |
+
}
|
train/act_dice_art_run1/wandb/debug-internal.log
ADDED
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{"time":"2025-06-15T01:25:25.992015864-07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-core.log"}
|
2 |
+
{"time":"2025-06-15T01:25:26.634032319-07:00","level":"INFO","msg":"stream: created new stream","id":"fhdgxxjz"}
|
3 |
+
{"time":"2025-06-15T01:25:26.634083631-07:00","level":"INFO","msg":"stream: started","id":"fhdgxxjz"}
|
4 |
+
{"time":"2025-06-15T01:25:26.634215949-07:00","level":"INFO","msg":"writer: Do: started","stream_id":"fhdgxxjz"}
|
5 |
+
{"time":"2025-06-15T01:25:26.634288668-07:00","level":"INFO","msg":"handler: started","stream_id":"fhdgxxjz"}
|
6 |
+
{"time":"2025-06-15T01:25:26.634389474-07:00","level":"INFO","msg":"sender: started","stream_id":"fhdgxxjz"}
|
7 |
+
{"time":"2025-06-15T01:25:27.320758871-07:00","level":"INFO","msg":"Starting system monitor"}
|
8 |
+
{"time":"2025-06-15T01:25:46.568983039-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
9 |
+
{"time":"2025-06-15T01:25:46.568997155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
10 |
+
{"time":"2025-06-15T01:25:46.568999143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
11 |
+
{"time":"2025-06-15T01:25:46.569000545-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
12 |
+
{"time":"2025-06-15T01:25:46.569001878-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
13 |
+
{"time":"2025-06-15T01:25:46.569003382-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
14 |
+
{"time":"2025-06-15T01:25:46.569004827-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
15 |
+
{"time":"2025-06-15T01:25:46.569006116-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
16 |
+
{"time":"2025-06-15T01:25:46.569007422-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
17 |
+
{"time":"2025-06-15T01:25:46.569008882-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
18 |
+
{"time":"2025-06-15T01:25:46.569048108-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40200,"current":42201}
|
19 |
+
{"time":"2025-06-15T01:26:02.367793397-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
20 |
+
{"time":"2025-06-15T01:26:02.372335004-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
21 |
+
{"time":"2025-06-15T01:26:02.372337971-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
22 |
+
{"time":"2025-06-15T01:26:02.372339512-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
23 |
+
{"time":"2025-06-15T01:26:02.372340951-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
24 |
+
{"time":"2025-06-15T01:26:02.372342409-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
25 |
+
{"time":"2025-06-15T01:26:02.372343769-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
26 |
+
{"time":"2025-06-15T01:26:02.372345043-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
27 |
+
{"time":"2025-06-15T01:26:02.3723463-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
28 |
+
{"time":"2025-06-15T01:26:02.372348347-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
29 |
+
{"time":"2025-06-15T01:26:02.372349688-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40400,"current":42201}
|
30 |
+
{"time":"2025-06-15T01:26:17.685821042-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
31 |
+
{"time":"2025-06-15T01:26:17.685849081-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
32 |
+
{"time":"2025-06-15T01:26:17.685857431-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
33 |
+
{"time":"2025-06-15T01:26:17.685858732-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
34 |
+
{"time":"2025-06-15T01:26:17.685859866-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
35 |
+
{"time":"2025-06-15T01:26:17.685861139-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
36 |
+
{"time":"2025-06-15T01:26:17.685862255-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
37 |
+
{"time":"2025-06-15T01:26:17.685863394-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
38 |
+
{"time":"2025-06-15T01:26:17.685864601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
39 |
+
{"time":"2025-06-15T01:26:17.68586591-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
40 |
+
{"time":"2025-06-15T01:26:17.685867074-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40600,"current":42201}
|
41 |
+
{"time":"2025-06-15T01:26:32.130793723-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
42 |
+
{"time":"2025-06-15T01:26:32.130813145-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
43 |
+
{"time":"2025-06-15T01:26:32.130815054-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
44 |
+
{"time":"2025-06-15T01:26:32.130816448-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
45 |
+
{"time":"2025-06-15T01:26:32.130817763-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
46 |
+
{"time":"2025-06-15T01:26:32.130819273-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
47 |
+
{"time":"2025-06-15T01:26:32.130820563-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
48 |
+
{"time":"2025-06-15T01:26:32.130821885-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
49 |
+
{"time":"2025-06-15T01:26:32.130823187-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
50 |
+
{"time":"2025-06-15T01:26:32.13082474-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
51 |
+
{"time":"2025-06-15T01:26:32.130826044-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":40800,"current":42201}
|
52 |
+
{"time":"2025-06-15T01:26:46.825477529-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
53 |
+
{"time":"2025-06-15T01:26:46.825530193-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
54 |
+
{"time":"2025-06-15T01:26:46.825558155-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
55 |
+
{"time":"2025-06-15T01:26:46.825582435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
56 |
+
{"time":"2025-06-15T01:26:46.825603491-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
57 |
+
{"time":"2025-06-15T01:26:46.825629646-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
58 |
+
{"time":"2025-06-15T01:26:46.825656804-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
59 |
+
{"time":"2025-06-15T01:26:46.825682964-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
60 |
+
{"time":"2025-06-15T01:26:46.825707833-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
61 |
+
{"time":"2025-06-15T01:26:46.825737502-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
62 |
+
{"time":"2025-06-15T01:26:46.825760265-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41000,"current":42201}
|
63 |
+
{"time":"2025-06-15T01:27:02.16185199-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
64 |
+
{"time":"2025-06-15T01:27:02.161878645-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
65 |
+
{"time":"2025-06-15T01:27:02.161884483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
66 |
+
{"time":"2025-06-15T01:27:02.161888247-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
67 |
+
{"time":"2025-06-15T01:27:02.161891557-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
68 |
+
{"time":"2025-06-15T01:27:02.161894938-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
69 |
+
{"time":"2025-06-15T01:27:02.161898154-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
70 |
+
{"time":"2025-06-15T01:27:02.161901259-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
71 |
+
{"time":"2025-06-15T01:27:02.161904335-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
72 |
+
{"time":"2025-06-15T01:27:02.161907587-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
73 |
+
{"time":"2025-06-15T01:27:02.161910751-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41200,"current":42201}
|
74 |
+
{"time":"2025-06-15T01:27:16.797906839-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
75 |
+
{"time":"2025-06-15T01:27:16.7979248-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
76 |
+
{"time":"2025-06-15T01:27:16.797928888-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
77 |
+
{"time":"2025-06-15T01:27:16.797955435-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
78 |
+
{"time":"2025-06-15T01:27:16.797965742-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
79 |
+
{"time":"2025-06-15T01:27:16.797986483-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
80 |
+
{"time":"2025-06-15T01:27:16.798010312-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
81 |
+
{"time":"2025-06-15T01:27:16.798035412-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
82 |
+
{"time":"2025-06-15T01:27:16.798056372-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
83 |
+
{"time":"2025-06-15T01:27:16.798085178-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
84 |
+
{"time":"2025-06-15T01:27:16.798107623-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41400,"current":42201}
|
85 |
+
{"time":"2025-06-15T01:27:31.191814532-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
86 |
+
{"time":"2025-06-15T01:27:31.19183601-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
87 |
+
{"time":"2025-06-15T01:27:31.191838508-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
88 |
+
{"time":"2025-06-15T01:27:31.19184059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
89 |
+
{"time":"2025-06-15T01:27:31.19184258-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
90 |
+
{"time":"2025-06-15T01:27:31.191850218-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
91 |
+
{"time":"2025-06-15T01:27:31.191851983-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
92 |
+
{"time":"2025-06-15T01:27:31.191854215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
93 |
+
{"time":"2025-06-15T01:27:31.191855837-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
94 |
+
{"time":"2025-06-15T01:27:31.191857573-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
95 |
+
{"time":"2025-06-15T01:27:31.191859143-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41600,"current":42201}
|
96 |
+
{"time":"2025-06-15T01:27:45.982376932-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
97 |
+
{"time":"2025-06-15T01:27:45.982418361-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
98 |
+
{"time":"2025-06-15T01:27:45.982429595-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
99 |
+
{"time":"2025-06-15T01:27:45.982466168-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
100 |
+
{"time":"2025-06-15T01:27:45.982474892-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
101 |
+
{"time":"2025-06-15T01:27:45.982495778-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
102 |
+
{"time":"2025-06-15T01:27:45.982520859-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
103 |
+
{"time":"2025-06-15T01:27:45.982542092-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
104 |
+
{"time":"2025-06-15T01:27:45.982564589-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
105 |
+
{"time":"2025-06-15T01:27:45.982593443-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
106 |
+
{"time":"2025-06-15T01:27:45.982610475-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":41800,"current":42201}
|
107 |
+
{"time":"2025-06-15T01:28:01.229859045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
108 |
+
{"time":"2025-06-15T01:28:01.229889059-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
109 |
+
{"time":"2025-06-15T01:28:01.229894087-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
110 |
+
{"time":"2025-06-15T01:28:01.229897505-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
111 |
+
{"time":"2025-06-15T01:28:01.229900368-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
112 |
+
{"time":"2025-06-15T01:28:01.229903215-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
113 |
+
{"time":"2025-06-15T01:28:01.229905919-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
114 |
+
{"time":"2025-06-15T01:28:01.229908651-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
115 |
+
{"time":"2025-06-15T01:28:01.229911346-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
116 |
+
{"time":"2025-06-15T01:28:01.22991437-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
117 |
+
{"time":"2025-06-15T01:28:01.229917398-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42000,"current":42201}
|
118 |
+
{"time":"2025-06-15T01:28:15.6350119-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
119 |
+
{"time":"2025-06-15T01:28:15.635042909-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
120 |
+
{"time":"2025-06-15T01:28:15.635078209-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
121 |
+
{"time":"2025-06-15T01:28:15.635093855-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
122 |
+
{"time":"2025-06-15T01:28:15.635112045-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
123 |
+
{"time":"2025-06-15T01:28:15.635150996-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
124 |
+
{"time":"2025-06-15T01:28:15.6351606-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
125 |
+
{"time":"2025-06-15T01:28:15.635184442-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
126 |
+
{"time":"2025-06-15T01:28:15.63520897-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
127 |
+
{"time":"2025-06-15T01:28:15.635243681-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
128 |
+
{"time":"2025-06-15T01:28:15.635260056-07:00","level":"WARN","msg":"handler: ignoring partial history record","step":42200,"current":42201}
|
129 |
+
{"time":"2025-06-15T02:36:42.45420484-07:00","level":"INFO","msg":"stream: closing","id":"fhdgxxjz"}
|
130 |
+
{"time":"2025-06-15T02:36:42.454249855-07:00","level":"INFO","msg":"Stopping system monitor"}
|
131 |
+
{"time":"2025-06-15T02:36:42.454306083-07:00","level":"INFO","msg":"Stopped system monitor"}
|
132 |
+
{"time":"2025-06-15T02:37:31.874312646-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
133 |
+
{"time":"2025-06-15T02:37:32.062233809-07:00","level":"INFO","msg":"handler: closed","stream_id":"fhdgxxjz"}
|
134 |
+
{"time":"2025-06-15T02:37:32.062277604-07:00","level":"INFO","msg":"writer: Close: closed","stream_id":"fhdgxxjz"}
|
135 |
+
{"time":"2025-06-15T02:37:32.062318929-07:00","level":"INFO","msg":"sender: closed","stream_id":"fhdgxxjz"}
|
136 |
+
{"time":"2025-06-15T02:37:32.06237074-07:00","level":"INFO","msg":"stream: closed","id":"fhdgxxjz"}
|
train/act_dice_art_run1/wandb/debug.log
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
|
2 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Configure stats pid to 4339
|
3 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/.config/wandb/settings
|
4 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/wandb/settings
|
5 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
6 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug.log
|
7 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/logs/debug-internal.log
|
8 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():831] calling init triggers
|
9 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
|
10 |
+
config: {'dataset': {'repo_id': 'BoredMachinist/dice-art-run1', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {'observation.state': {'type': <FeatureType.STATE: 'STATE'>, 'shape': [6]}, 'observation.images.arm': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}, 'observation.images.world': {'type': <FeatureType.VISUAL: 'VISUAL'>, 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': <FeatureType.ACTION: 'ACTION'>, 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_dice_art_run1', 'job_name': 'act_dice_art_run1', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'fhdgxxjz', 'mode': None}, '_wandb': {}}
|
11 |
+
2025-06-15 01:25:25,773 INFO MainThread:4339 [wandb_init.py:init():872] starting backend
|
12 |
+
2025-06-15 01:25:25,979 INFO MainThread:4339 [wandb_init.py:init():875] sending inform_init request
|
13 |
+
2025-06-15 01:25:25,986 INFO MainThread:4339 [wandb_init.py:init():883] backend started and connected
|
14 |
+
2025-06-15 01:25:25,989 INFO MainThread:4339 [wandb_init.py:init():956] updated telemetry
|
15 |
+
2025-06-15 01:25:26,005 INFO MainThread:4339 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
|
16 |
+
2025-06-15 01:25:27,275 INFO MainThread:4339 [wandb_init.py:init():1027] run resumed
|
17 |
+
2025-06-15 01:25:27,279 INFO MainThread:4339 [wandb_init.py:init():1032] starting run threads in backend
|
18 |
+
2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_console_start():2453] atexit reg
|
19 |
+
2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2301] redirect: wrap_raw
|
20 |
+
2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2370] Wrapping output streams.
|
21 |
+
2025-06-15 01:25:27,405 INFO MainThread:4339 [wandb_run.py:_redirect():2393] Redirects installed.
|
22 |
+
2025-06-15 01:25:27,406 INFO MainThread:4339 [wandb_init.py:init():1078] run started, returning control to user process
|
23 |
+
2025-06-15 02:36:42,453 INFO MsgRouterThr:4339 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 3 handles.
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/config.yaml
ADDED
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
_wandb:
|
2 |
+
value:
|
3 |
+
cli_version: 0.20.1
|
4 |
+
m: []
|
5 |
+
python_version: 3.10.13
|
6 |
+
t:
|
7 |
+
"1":
|
8 |
+
- 1
|
9 |
+
- 41
|
10 |
+
- 49
|
11 |
+
- 51
|
12 |
+
- 105
|
13 |
+
"2":
|
14 |
+
- 1
|
15 |
+
- 41
|
16 |
+
- 49
|
17 |
+
- 51
|
18 |
+
- 105
|
19 |
+
"3":
|
20 |
+
- 13
|
21 |
+
- 15
|
22 |
+
- 16
|
23 |
+
- 55
|
24 |
+
- 61
|
25 |
+
"4": 3.10.13
|
26 |
+
"5": 0.20.1
|
27 |
+
"10":
|
28 |
+
- 21
|
29 |
+
"12": 0.20.1
|
30 |
+
"13": linux-x86_64
|
31 |
+
batch_size:
|
32 |
+
value: 8
|
33 |
+
dataset:
|
34 |
+
value:
|
35 |
+
episodes: null
|
36 |
+
image_transforms:
|
37 |
+
enable: false
|
38 |
+
max_num_transforms: 3
|
39 |
+
random_order: false
|
40 |
+
tfs:
|
41 |
+
brightness:
|
42 |
+
kwargs:
|
43 |
+
brightness:
|
44 |
+
- 0.8
|
45 |
+
- 1.2
|
46 |
+
type: ColorJitter
|
47 |
+
weight: 1
|
48 |
+
contrast:
|
49 |
+
kwargs:
|
50 |
+
contrast:
|
51 |
+
- 0.8
|
52 |
+
- 1.2
|
53 |
+
type: ColorJitter
|
54 |
+
weight: 1
|
55 |
+
hue:
|
56 |
+
kwargs:
|
57 |
+
hue:
|
58 |
+
- -0.05
|
59 |
+
- 0.05
|
60 |
+
type: ColorJitter
|
61 |
+
weight: 1
|
62 |
+
saturation:
|
63 |
+
kwargs:
|
64 |
+
saturation:
|
65 |
+
- 0.5
|
66 |
+
- 1.5
|
67 |
+
type: ColorJitter
|
68 |
+
weight: 1
|
69 |
+
sharpness:
|
70 |
+
kwargs:
|
71 |
+
sharpness:
|
72 |
+
- 0.5
|
73 |
+
- 1.5
|
74 |
+
type: SharpnessJitter
|
75 |
+
weight: 1
|
76 |
+
repo_id: BoredMachinist/dice-art-run1
|
77 |
+
revision: null
|
78 |
+
root: null
|
79 |
+
use_imagenet_stats: true
|
80 |
+
video_backend: pyav
|
81 |
+
env:
|
82 |
+
value: null
|
83 |
+
eval:
|
84 |
+
value:
|
85 |
+
batch_size: 50
|
86 |
+
n_episodes: 50
|
87 |
+
use_async_envs: false
|
88 |
+
eval_freq:
|
89 |
+
value: 20000
|
90 |
+
job_name:
|
91 |
+
value: act_dice_art_run1
|
92 |
+
log_freq:
|
93 |
+
value: 200
|
94 |
+
num_workers:
|
95 |
+
value: 4
|
96 |
+
optimizer:
|
97 |
+
value:
|
98 |
+
betas:
|
99 |
+
- 0.9
|
100 |
+
- 0.999
|
101 |
+
eps: 1e-08
|
102 |
+
grad_clip_norm: 10
|
103 |
+
lr: 1e-05
|
104 |
+
type: adamw
|
105 |
+
weight_decay: 0.0001
|
106 |
+
output_dir:
|
107 |
+
value: outputs/train/act_dice_art_run1
|
108 |
+
policy:
|
109 |
+
value:
|
110 |
+
chunk_size: 100
|
111 |
+
device: cuda
|
112 |
+
dim_feedforward: 3200
|
113 |
+
dim_model: 512
|
114 |
+
dropout: 0.1
|
115 |
+
feedforward_activation: relu
|
116 |
+
kl_weight: 10
|
117 |
+
latent_dim: 32
|
118 |
+
n_action_steps: 100
|
119 |
+
n_decoder_layers: 1
|
120 |
+
n_encoder_layers: 4
|
121 |
+
n_heads: 8
|
122 |
+
n_obs_steps: 1
|
123 |
+
n_vae_encoder_layers: 4
|
124 |
+
normalization_mapping:
|
125 |
+
ACTION: MEAN_STD
|
126 |
+
STATE: MEAN_STD
|
127 |
+
VISUAL: MEAN_STD
|
128 |
+
optimizer_lr: 1e-05
|
129 |
+
optimizer_lr_backbone: 1e-05
|
130 |
+
optimizer_weight_decay: 0.0001
|
131 |
+
pre_norm: false
|
132 |
+
pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
|
133 |
+
replace_final_stride_with_dilation: false
|
134 |
+
temporal_ensemble_coeff: null
|
135 |
+
type: act
|
136 |
+
use_amp: false
|
137 |
+
use_vae: true
|
138 |
+
vision_backbone: resnet18
|
139 |
+
resume:
|
140 |
+
value: false
|
141 |
+
save_checkpoint:
|
142 |
+
value: true
|
143 |
+
save_freq:
|
144 |
+
value: 20000
|
145 |
+
scheduler:
|
146 |
+
value: null
|
147 |
+
seed:
|
148 |
+
value: 1000
|
149 |
+
steps:
|
150 |
+
value: 100000
|
151 |
+
use_policy_training_preset:
|
152 |
+
value: true
|
153 |
+
wandb:
|
154 |
+
value:
|
155 |
+
disable_artifact: false
|
156 |
+
enable: true
|
157 |
+
entity: null
|
158 |
+
mode: null
|
159 |
+
notes: null
|
160 |
+
project: lerobot
|
161 |
+
run_id: null
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/output.log
ADDED
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[1m[34mLogs will be synced with wandb.[0m
|
2 |
+
INFO 2025-06-14 20:09:52 db_utils.py:103 Track this run --> [1m[33mhttps://wandb.ai/ametind-o/lerobot/runs/fhdgxxjz[0m
|
3 |
+
INFO 2025-06-14 20:09:52 ts/train.py:127 Creating dataset
|
4 |
+
INFO 2025-06-14 20:09:52 ts/train.py:138 Creating policy
|
5 |
+
INFO 2025-06-14 20:09:53 ts/train.py:144 Creating optimizer and scheduler
|
6 |
+
INFO 2025-06-14 20:09:53 ts/train.py:156 [1m[33mOutput dir:[0m outputs/train/act_dice_art_run1
|
7 |
+
INFO 2025-06-14 20:09:53 ts/train.py:159 cfg.steps=100000 (100K)
|
8 |
+
INFO 2025-06-14 20:09:53 ts/train.py:160 dataset.num_frames=2244 (2K)
|
9 |
+
INFO 2025-06-14 20:09:53 ts/train.py:161 dataset.num_episodes=9
|
10 |
+
INFO 2025-06-14 20:09:53 ts/train.py:162 num_learnable_params=51597190 (52M)
|
11 |
+
INFO 2025-06-14 20:09:53 ts/train.py:163 num_total_params=51597238 (52M)
|
12 |
+
INFO 2025-06-14 20:09:53 ts/train.py:202 Start offline training on a fixed dataset
|
13 |
+
INFO 2025-06-14 20:10:09 ts/train.py:232 step:200 smpl:2K ep:6 epch:0.71 loss:6.515 grdn:149.668 lr:1.0e-05 updt_s:0.079 data_s:0.002
|
14 |
+
INFO 2025-06-14 20:10:25 ts/train.py:232 step:400 smpl:3K ep:13 epch:1.43 loss:2.868 grdn:82.381 lr:1.0e-05 updt_s:0.076 data_s:0.002
|
15 |
+
INFO 2025-06-14 20:10:40 ts/train.py:232 step:600 smpl:5K ep:19 epch:2.14 loss:2.408 grdn:72.525 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
16 |
+
INFO 2025-06-14 20:10:55 ts/train.py:232 step:800 smpl:6K ep:26 epch:2.85 loss:2.109 grdn:67.732 lr:1.0e-05 updt_s:0.074 data_s:0.000
|
17 |
+
INFO 2025-06-14 20:11:10 ts/train.py:232 step:1K smpl:8K ep:32 epch:3.57 loss:1.891 grdn:65.419 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
18 |
+
INFO 2025-06-14 20:11:25 ts/train.py:232 step:1K smpl:10K ep:39 epch:4.28 loss:1.706 grdn:62.129 lr:1.0e-05 updt_s:0.074 data_s:0.001
|
19 |
+
INFO 2025-06-14 20:11:40 ts/train.py:232 step:1K smpl:11K ep:45 epch:4.99 loss:1.534 grdn:58.982 lr:1.0e-05 updt_s:0.074 data_s:0.000
|
20 |
+
INFO 2025-06-14 20:11:55 ts/train.py:232 step:2K smpl:13K ep:51 epch:5.70 loss:1.392 grdn:56.874 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
21 |
+
INFO 2025-06-14 20:12:10 ts/train.py:232 step:2K smpl:14K ep:58 epch:6.42 loss:1.269 grdn:55.147 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
22 |
+
INFO 2025-06-14 20:12:26 ts/train.py:232 step:2K smpl:16K ep:64 epch:7.13 loss:1.161 grdn:52.125 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
23 |
+
INFO 2025-06-14 20:12:41 ts/train.py:232 step:2K smpl:18K ep:71 epch:7.84 loss:1.062 grdn:50.324 lr:1.0e-05 updt_s:0.074 data_s:0.000
|
24 |
+
INFO 2025-06-14 20:12:56 ts/train.py:232 step:2K smpl:19K ep:77 epch:8.56 loss:0.960 grdn:47.189 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
25 |
+
INFO 2025-06-14 20:13:11 ts/train.py:232 step:3K smpl:21K ep:83 epch:9.27 loss:0.870 grdn:45.187 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
26 |
+
INFO 2025-06-14 20:13:26 ts/train.py:232 step:3K smpl:22K ep:90 epch:9.98 loss:0.793 grdn:42.965 lr:1.0e-05 updt_s:0.075 data_s:0.000
|
27 |
+
INFO 2025-06-14 20:13:42 ts/train.py:232 step:3K smpl:24K ep:96 epch:10.70 loss:0.725 grdn:42.298 lr:1.0e-05 updt_s:0.075 data_s:0.002
|
28 |
+
INFO 2025-06-14 20:13:57 ts/train.py:232 step:3K smpl:26K ep:103 epch:11.41 loss:0.659 grdn:40.367 lr:1.0e-05 updt_s:0.075 data_s:0.002
|
29 |
+
INFO 2025-06-14 20:14:12 ts/train.py:232 step:3K smpl:27K ep:109 epch:12.12 loss:0.604 grdn:37.984 lr:1.0e-05 updt_s:0.075 data_s:0.002
|
30 |
+
INFO 2025-06-14 20:14:27 ts/train.py:232 step:4K smpl:29K ep:116 epch:12.83 loss:0.555 grdn:36.595 lr:1.0e-05 updt_s:0.075 data_s:0.000
|
31 |
+
INFO 2025-06-14 20:14:43 ts/train.py:232 step:4K smpl:30K ep:122 epch:13.55 loss:0.511 grdn:35.474 lr:1.0e-05 updt_s:0.075 data_s:0.002
|
32 |
+
INFO 2025-06-14 20:14:58 ts/train.py:232 step:4K smpl:32K ep:128 epch:14.26 loss:0.473 grdn:33.822 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
33 |
+
INFO 2025-06-14 20:15:13 ts/train.py:232 step:4K smpl:34K ep:135 epch:14.97 loss:0.440 grdn:34.830 lr:1.0e-05 updt_s:0.075 data_s:0.000
|
34 |
+
INFO 2025-06-14 20:15:28 ts/train.py:232 step:4K smpl:35K ep:141 epch:15.69 loss:0.411 grdn:30.655 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
35 |
+
INFO 2025-06-14 20:15:43 ts/train.py:232 step:5K smpl:37K ep:148 epch:16.40 loss:0.388 grdn:30.633 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
36 |
+
INFO 2025-06-14 20:15:57 ts/train.py:232 step:5K smpl:38K ep:154 epch:17.11 loss:0.364 grdn:29.262 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
37 |
+
INFO 2025-06-14 20:16:11 ts/train.py:232 step:5K smpl:40K ep:160 epch:17.83 loss:0.338 grdn:28.092 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
38 |
+
INFO 2025-06-14 20:16:26 ts/train.py:232 step:5K smpl:42K ep:167 epch:18.54 loss:0.324 grdn:27.827 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
39 |
+
INFO 2025-06-14 20:16:40 ts/train.py:232 step:5K smpl:43K ep:173 epch:19.25 loss:0.311 grdn:28.218 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
40 |
+
INFO 2025-06-14 20:16:54 ts/train.py:232 step:6K smpl:45K ep:180 epch:19.96 loss:0.294 grdn:26.132 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
41 |
+
INFO 2025-06-14 20:17:08 ts/train.py:232 step:6K smpl:46K ep:186 epch:20.68 loss:0.284 grdn:26.785 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
42 |
+
INFO 2025-06-14 20:17:23 ts/train.py:232 step:6K smpl:48K ep:193 epch:21.39 loss:0.274 grdn:24.581 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
43 |
+
INFO 2025-06-14 20:17:38 ts/train.py:232 step:6K smpl:50K ep:199 epch:22.10 loss:0.262 grdn:24.783 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
44 |
+
INFO 2025-06-14 20:17:52 ts/train.py:232 step:6K smpl:51K ep:205 epch:22.82 loss:0.256 grdn:24.414 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
45 |
+
INFO 2025-06-14 20:18:07 ts/train.py:232 step:7K smpl:53K ep:212 epch:23.53 loss:0.247 grdn:23.247 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
46 |
+
INFO 2025-06-14 20:18:22 ts/train.py:232 step:7K smpl:54K ep:218 epch:24.24 loss:0.236 grdn:23.459 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
47 |
+
INFO 2025-06-14 20:18:37 ts/train.py:232 step:7K smpl:56K ep:225 epch:24.96 loss:0.230 grdn:22.174 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
48 |
+
INFO 2025-06-14 20:18:51 ts/train.py:232 step:7K smpl:58K ep:231 epch:25.67 loss:0.222 grdn:22.406 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
49 |
+
INFO 2025-06-14 20:19:06 ts/train.py:232 step:7K smpl:59K ep:237 epch:26.38 loss:0.220 grdn:22.310 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
50 |
+
INFO 2025-06-14 20:19:20 ts/train.py:232 step:8K smpl:61K ep:244 epch:27.09 loss:0.212 grdn:21.087 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
51 |
+
INFO 2025-06-14 20:19:34 ts/train.py:232 step:8K smpl:62K ep:250 epch:27.81 loss:0.210 grdn:21.643 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
52 |
+
INFO 2025-06-14 20:19:49 ts/train.py:232 step:8K smpl:64K ep:257 epch:28.52 loss:0.205 grdn:20.799 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
53 |
+
INFO 2025-06-14 20:20:03 ts/train.py:232 step:8K smpl:66K ep:263 epch:29.23 loss:0.201 grdn:21.118 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
54 |
+
INFO 2025-06-14 20:20:17 ts/train.py:232 step:8K smpl:67K ep:270 epch:29.95 loss:0.193 grdn:20.314 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
55 |
+
INFO 2025-06-14 20:20:32 ts/train.py:232 step:9K smpl:69K ep:276 epch:30.66 loss:0.188 grdn:19.721 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
56 |
+
INFO 2025-06-14 20:20:46 ts/train.py:232 step:9K smpl:70K ep:282 epch:31.37 loss:0.187 grdn:20.583 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
57 |
+
INFO 2025-06-14 20:21:01 ts/train.py:232 step:9K smpl:72K ep:289 epch:32.09 loss:0.181 grdn:20.067 lr:1.0e-05 updt_s:0.071 data_s:0.001
|
58 |
+
INFO 2025-06-14 20:21:15 ts/train.py:232 step:9K smpl:74K ep:295 epch:32.80 loss:0.181 grdn:19.900 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
59 |
+
INFO 2025-06-14 20:21:30 ts/train.py:232 step:9K smpl:75K ep:302 epch:33.51 loss:0.175 grdn:19.306 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
60 |
+
INFO 2025-06-14 20:21:44 ts/train.py:232 step:10K smpl:77K ep:308 epch:34.22 loss:0.175 grdn:18.802 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
61 |
+
INFO 2025-06-14 20:21:58 ts/train.py:232 step:10K smpl:78K ep:314 epch:34.94 loss:0.171 grdn:18.738 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
62 |
+
INFO 2025-06-14 20:22:13 ts/train.py:232 step:10K smpl:80K ep:321 epch:35.65 loss:0.166 grdn:18.788 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
63 |
+
INFO 2025-06-14 20:22:27 ts/train.py:232 step:10K smpl:82K ep:327 epch:36.36 loss:0.165 grdn:17.641 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
64 |
+
INFO 2025-06-14 20:22:42 ts/train.py:232 step:10K smpl:83K ep:334 epch:37.08 loss:0.164 grdn:19.923 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
65 |
+
INFO 2025-06-14 20:22:56 ts/train.py:232 step:11K smpl:85K ep:340 epch:37.79 loss:0.163 grdn:19.528 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
66 |
+
INFO 2025-06-14 20:23:10 ts/train.py:232 step:11K smpl:86K ep:347 epch:38.50 loss:0.158 grdn:17.752 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
67 |
+
INFO 2025-06-14 20:23:25 ts/train.py:232 step:11K smpl:88K ep:353 epch:39.22 loss:0.157 grdn:17.456 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
68 |
+
INFO 2025-06-14 20:23:39 ts/train.py:232 step:11K smpl:90K ep:359 epch:39.93 loss:0.153 grdn:17.560 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
69 |
+
INFO 2025-06-14 20:23:53 ts/train.py:232 step:11K smpl:91K ep:366 epch:40.64 loss:0.153 grdn:17.448 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
70 |
+
INFO 2025-06-14 20:24:07 ts/train.py:232 step:12K smpl:93K ep:372 epch:41.35 loss:0.150 grdn:17.562 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
71 |
+
INFO 2025-06-14 20:24:21 ts/train.py:232 step:12K smpl:94K ep:379 epch:42.07 loss:0.148 grdn:17.039 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
72 |
+
INFO 2025-06-14 20:24:36 ts/train.py:232 step:12K smpl:96K ep:385 epch:42.78 loss:0.146 grdn:16.688 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
73 |
+
INFO 2025-06-14 20:24:50 ts/train.py:232 step:12K smpl:98K ep:391 epch:43.49 loss:0.145 grdn:17.900 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
74 |
+
INFO 2025-06-14 20:25:04 ts/train.py:232 step:12K smpl:99K ep:398 epch:44.21 loss:0.143 grdn:16.788 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
75 |
+
INFO 2025-06-14 20:25:18 ts/train.py:232 step:13K smpl:101K ep:404 epch:44.92 loss:0.141 grdn:16.224 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
76 |
+
INFO 2025-06-14 20:25:32 ts/train.py:232 step:13K smpl:102K ep:411 epch:45.63 loss:0.140 grdn:16.345 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
77 |
+
INFO 2025-06-14 20:25:47 ts/train.py:232 step:13K smpl:104K ep:417 epch:46.35 loss:0.139 grdn:16.433 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
78 |
+
INFO 2025-06-14 20:26:01 ts/train.py:232 step:13K smpl:106K ep:424 epch:47.06 loss:0.137 grdn:16.710 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
79 |
+
INFO 2025-06-14 20:26:15 ts/train.py:232 step:13K smpl:107K ep:430 epch:47.77 loss:0.135 grdn:15.966 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
80 |
+
INFO 2025-06-14 20:26:29 ts/train.py:232 step:14K smpl:109K ep:436 epch:48.48 loss:0.137 grdn:16.677 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
81 |
+
INFO 2025-06-14 20:26:43 ts/train.py:232 step:14K smpl:110K ep:443 epch:49.20 loss:0.131 grdn:15.750 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
82 |
+
INFO 2025-06-14 20:26:57 ts/train.py:232 step:14K smpl:112K ep:449 epch:49.91 loss:0.129 grdn:15.559 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
83 |
+
INFO 2025-06-14 20:27:12 ts/train.py:232 step:14K smpl:114K ep:456 epch:50.62 loss:0.130 grdn:15.826 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
84 |
+
INFO 2025-06-14 20:27:26 ts/train.py:232 step:14K smpl:115K ep:462 epch:51.34 loss:0.128 grdn:15.333 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
85 |
+
INFO 2025-06-14 20:27:40 ts/train.py:232 step:15K smpl:117K ep:468 epch:52.05 loss:0.126 grdn:14.943 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
86 |
+
INFO 2025-06-14 20:27:54 ts/train.py:232 step:15K smpl:118K ep:475 epch:52.76 loss:0.126 grdn:15.906 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
87 |
+
INFO 2025-06-14 20:28:09 ts/train.py:232 step:15K smpl:120K ep:481 epch:53.48 loss:0.123 grdn:15.554 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
88 |
+
INFO 2025-06-14 20:28:23 ts/train.py:232 step:15K smpl:122K ep:488 epch:54.19 loss:0.125 grdn:15.625 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
89 |
+
INFO 2025-06-14 20:28:37 ts/train.py:232 step:15K smpl:123K ep:494 epch:54.90 loss:0.122 grdn:14.467 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
90 |
+
INFO 2025-06-14 20:28:51 ts/train.py:232 step:16K smpl:125K ep:501 epch:55.61 loss:0.121 grdn:14.501 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
91 |
+
INFO 2025-06-14 20:29:05 ts/train.py:232 step:16K smpl:126K ep:507 epch:56.33 loss:0.121 grdn:15.215 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
92 |
+
INFO 2025-06-14 20:29:19 ts/train.py:232 step:16K smpl:128K ep:513 epch:57.04 loss:0.118 grdn:14.298 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
93 |
+
INFO 2025-06-14 20:29:34 ts/train.py:232 step:16K smpl:130K ep:520 epch:57.75 loss:0.119 grdn:14.430 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
94 |
+
INFO 2025-06-14 20:29:48 ts/train.py:232 step:16K smpl:131K ep:526 epch:58.47 loss:0.115 grdn:14.649 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
95 |
+
INFO 2025-06-14 20:30:02 ts/train.py:232 step:17K smpl:133K ep:533 epch:59.18 loss:0.114 grdn:14.037 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
96 |
+
INFO 2025-06-14 20:30:16 ts/train.py:232 step:17K smpl:134K ep:539 epch:59.89 loss:0.114 grdn:14.603 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
97 |
+
INFO 2025-06-14 20:30:30 ts/train.py:232 step:17K smpl:136K ep:545 epch:60.61 loss:0.114 grdn:13.872 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
98 |
+
INFO 2025-06-14 20:30:45 ts/train.py:232 step:17K smpl:138K ep:552 epch:61.32 loss:0.112 grdn:14.225 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
99 |
+
INFO 2025-06-14 20:30:59 ts/train.py:232 step:17K smpl:139K ep:558 epch:62.03 loss:0.114 grdn:14.691 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
100 |
+
INFO 2025-06-14 20:31:13 ts/train.py:232 step:18K smpl:141K ep:565 epch:62.75 loss:0.111 grdn:13.451 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
101 |
+
INFO 2025-06-14 20:31:27 ts/train.py:232 step:18K smpl:142K ep:571 epch:63.46 loss:0.110 grdn:14.047 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
102 |
+
INFO 2025-06-14 20:31:41 ts/train.py:232 step:18K smpl:144K ep:578 epch:64.17 loss:0.108 grdn:13.818 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
103 |
+
INFO 2025-06-14 20:31:55 ts/train.py:232 step:18K smpl:146K ep:584 epch:64.88 loss:0.108 grdn:13.908 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
104 |
+
INFO 2025-06-14 20:32:10 ts/train.py:232 step:18K smpl:147K ep:590 epch:65.60 loss:0.107 grdn:13.497 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
105 |
+
INFO 2025-06-14 20:32:24 ts/train.py:232 step:19K smpl:149K ep:597 epch:66.31 loss:0.107 grdn:13.179 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
106 |
+
INFO 2025-06-14 20:32:38 ts/train.py:232 step:19K smpl:150K ep:603 epch:67.02 loss:0.107 grdn:13.837 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
107 |
+
INFO 2025-06-14 20:32:52 ts/train.py:232 step:19K smpl:152K ep:610 epch:67.74 loss:0.104 grdn:13.106 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
108 |
+
INFO 2025-06-14 20:33:07 ts/train.py:232 step:19K smpl:154K ep:616 epch:68.45 loss:0.104 grdn:12.692 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
109 |
+
INFO 2025-06-14 20:33:21 ts/train.py:232 step:19K smpl:155K ep:622 epch:69.16 loss:0.103 grdn:13.085 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
110 |
+
INFO 2025-06-14 20:33:35 ts/train.py:232 step:20K smpl:157K ep:629 epch:69.88 loss:0.102 grdn:12.937 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
111 |
+
INFO 2025-06-14 20:33:49 ts/train.py:232 step:20K smpl:158K ep:635 epch:70.59 loss:0.101 grdn:12.779 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
112 |
+
INFO 2025-06-14 20:34:04 ts/train.py:232 step:20K smpl:160K ep:642 epch:71.30 loss:0.100 grdn:12.397 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
113 |
+
INFO 2025-06-14 20:34:04 ts/train.py:241 Checkpoint policy after step 20000
|
114 |
+
INFO 2025-06-14 20:34:20 ts/train.py:232 step:20K smpl:162K ep:648 epch:72.01 loss:0.102 grdn:13.082 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
115 |
+
INFO 2025-06-14 20:34:34 ts/train.py:232 step:20K smpl:163K ep:655 epch:72.73 loss:0.101 grdn:13.668 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
116 |
+
INFO 2025-06-14 20:34:49 ts/train.py:232 step:21K smpl:165K ep:661 epch:73.44 loss:0.098 grdn:12.644 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
117 |
+
INFO 2025-06-14 20:35:04 ts/train.py:232 step:21K smpl:166K ep:667 epch:74.15 loss:0.098 grdn:12.694 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
118 |
+
INFO 2025-06-14 20:35:18 ts/train.py:232 step:21K smpl:168K ep:674 epch:74.87 loss:0.095 grdn:11.722 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
119 |
+
INFO 2025-06-14 20:35:32 ts/train.py:232 step:21K smpl:170K ep:680 epch:75.58 loss:0.096 grdn:12.236 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
120 |
+
INFO 2025-06-14 20:35:47 ts/train.py:232 step:21K smpl:171K ep:687 epch:76.29 loss:0.096 grdn:12.080 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
121 |
+
INFO 2025-06-14 20:36:02 ts/train.py:232 step:22K smpl:173K ep:693 epch:77.01 loss:0.097 grdn:12.912 lr:1.0e-05 updt_s:0.075 data_s:0.000
|
122 |
+
INFO 2025-06-14 20:36:18 ts/train.py:232 step:22K smpl:174K ep:699 epch:77.72 loss:0.096 grdn:12.410 lr:1.0e-05 updt_s:0.075 data_s:0.002
|
123 |
+
INFO 2025-06-14 20:36:33 ts/train.py:232 step:22K smpl:176K ep:706 epch:78.43 loss:0.094 grdn:12.101 lr:1.0e-05 updt_s:0.074 data_s:0.001
|
124 |
+
INFO 2025-06-14 20:36:48 ts/train.py:232 step:22K smpl:178K ep:712 epch:79.14 loss:0.093 grdn:11.906 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
125 |
+
INFO 2025-06-14 20:37:02 ts/train.py:232 step:22K smpl:179K ep:719 epch:79.86 loss:0.094 grdn:11.773 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
126 |
+
INFO 2025-06-14 20:37:17 ts/train.py:232 step:23K smpl:181K ep:725 epch:80.57 loss:0.092 grdn:12.889 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
127 |
+
INFO 2025-06-14 20:37:32 ts/train.py:232 step:23K smpl:182K ep:732 epch:81.28 loss:0.093 grdn:11.871 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
128 |
+
INFO 2025-06-14 20:37:46 ts/train.py:232 step:23K smpl:184K ep:738 epch:82.00 loss:0.089 grdn:11.595 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
129 |
+
INFO 2025-06-14 20:38:01 ts/train.py:232 step:23K smpl:186K ep:744 epch:82.71 loss:0.092 grdn:12.292 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
130 |
+
INFO 2025-06-14 20:38:15 ts/train.py:232 step:23K smpl:187K ep:751 epch:83.42 loss:0.090 grdn:12.134 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
131 |
+
INFO 2025-06-14 20:38:30 ts/train.py:232 step:24K smpl:189K ep:757 epch:84.14 loss:0.089 grdn:11.247 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
132 |
+
INFO 2025-06-14 20:38:45 ts/train.py:232 step:24K smpl:190K ep:764 epch:84.85 loss:0.087 grdn:11.122 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
133 |
+
INFO 2025-06-14 20:39:00 ts/train.py:232 step:24K smpl:192K ep:770 epch:85.56 loss:0.089 grdn:11.528 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
134 |
+
INFO 2025-06-14 20:39:15 ts/train.py:232 step:24K smpl:194K ep:776 epch:86.27 loss:0.088 grdn:12.144 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
135 |
+
INFO 2025-06-14 20:39:29 ts/train.py:232 step:24K smpl:195K ep:783 epch:86.99 loss:0.086 grdn:10.970 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
136 |
+
INFO 2025-06-14 20:39:44 ts/train.py:232 step:25K smpl:197K ep:789 epch:87.70 loss:0.086 grdn:11.029 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
137 |
+
INFO 2025-06-14 20:39:59 ts/train.py:232 step:25K smpl:198K ep:796 epch:88.41 loss:0.086 grdn:11.238 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
138 |
+
INFO 2025-06-14 20:40:13 ts/train.py:232 step:25K smpl:200K ep:802 epch:89.13 loss:0.086 grdn:10.944 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
139 |
+
INFO 2025-06-14 20:40:28 ts/train.py:232 step:25K smpl:202K ep:809 epch:89.84 loss:0.085 grdn:11.207 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
140 |
+
INFO 2025-06-14 20:40:42 ts/train.py:232 step:25K smpl:203K ep:815 epch:90.55 loss:0.086 grdn:11.160 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
141 |
+
INFO 2025-06-14 20:40:56 ts/train.py:232 step:26K smpl:205K ep:821 epch:91.27 loss:0.085 grdn:11.252 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
142 |
+
INFO 2025-06-14 20:41:10 ts/train.py:232 step:26K smpl:206K ep:828 epch:91.98 loss:0.082 grdn:10.840 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
143 |
+
INFO 2025-06-14 20:41:25 ts/train.py:232 step:26K smpl:208K ep:834 epch:92.69 loss:0.084 grdn:10.840 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
144 |
+
INFO 2025-06-14 20:41:39 ts/train.py:232 step:26K smpl:210K ep:841 epch:93.40 loss:0.081 grdn:11.170 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
145 |
+
INFO 2025-06-14 20:41:53 ts/train.py:232 step:26K smpl:211K ep:847 epch:94.12 loss:0.083 grdn:10.977 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
146 |
+
INFO 2025-06-14 20:42:08 ts/train.py:232 step:27K smpl:213K ep:853 epch:94.83 loss:0.083 grdn:11.122 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
147 |
+
INFO 2025-06-14 20:42:22 ts/train.py:232 step:27K smpl:214K ep:860 epch:95.54 loss:0.082 grdn:10.623 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
148 |
+
INFO 2025-06-14 20:42:36 ts/train.py:232 step:27K smpl:216K ep:866 epch:96.26 loss:0.081 grdn:10.906 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
149 |
+
INFO 2025-06-14 20:42:51 ts/train.py:232 step:27K smpl:218K ep:873 epch:96.97 loss:0.079 grdn:10.305 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
150 |
+
INFO 2025-06-14 20:43:05 ts/train.py:232 step:27K smpl:219K ep:879 epch:97.68 loss:0.080 grdn:10.587 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
151 |
+
INFO 2025-06-14 20:43:19 ts/train.py:232 step:28K smpl:221K ep:886 epch:98.40 loss:0.078 grdn:9.818 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
152 |
+
INFO 2025-06-14 20:43:33 ts/train.py:232 step:28K smpl:222K ep:892 epch:99.11 loss:0.081 grdn:10.856 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
153 |
+
INFO 2025-06-14 20:43:48 ts/train.py:232 step:28K smpl:224K ep:898 epch:99.82 loss:0.079 grdn:10.619 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
154 |
+
INFO 2025-06-14 20:44:02 ts/train.py:232 step:28K smpl:226K ep:905 epch:100.53 loss:0.078 grdn:10.213 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
155 |
+
INFO 2025-06-14 20:44:16 ts/train.py:232 step:28K smpl:227K ep:911 epch:101.25 loss:0.078 grdn:10.132 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
156 |
+
INFO 2025-06-14 20:44:31 ts/train.py:232 step:29K smpl:229K ep:918 epch:101.96 loss:0.078 grdn:10.275 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
157 |
+
INFO 2025-06-14 20:44:45 ts/train.py:232 step:29K smpl:230K ep:924 epch:102.67 loss:0.076 grdn:9.912 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
158 |
+
INFO 2025-06-14 20:44:59 ts/train.py:232 step:29K smpl:232K ep:930 epch:103.39 loss:0.077 grdn:9.954 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
159 |
+
INFO 2025-06-14 20:45:13 ts/train.py:232 step:29K smpl:234K ep:937 epch:104.10 loss:0.076 grdn:9.673 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
160 |
+
INFO 2025-06-14 20:45:28 ts/train.py:232 step:29K smpl:235K ep:943 epch:104.81 loss:0.075 grdn:9.531 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
161 |
+
INFO 2025-06-14 20:45:42 ts/train.py:232 step:30K smpl:237K ep:950 epch:105.53 loss:0.076 grdn:10.207 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
162 |
+
INFO 2025-06-14 20:45:56 ts/train.py:232 step:30K smpl:238K ep:956 epch:106.24 loss:0.076 grdn:9.426 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
163 |
+
INFO 2025-06-14 20:46:10 ts/train.py:232 step:30K smpl:240K ep:963 epch:106.95 loss:0.076 grdn:10.054 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
164 |
+
INFO 2025-06-14 20:46:25 ts/train.py:232 step:30K smpl:242K ep:969 epch:107.66 loss:0.076 grdn:9.912 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
165 |
+
INFO 2025-06-14 20:46:39 ts/train.py:232 step:30K smpl:243K ep:975 epch:108.38 loss:0.074 grdn:9.162 lr:1.0e-05 updt_s:0.071 data_s:0.001
|
166 |
+
INFO 2025-06-14 20:46:54 ts/train.py:232 step:31K smpl:245K ep:982 epch:109.09 loss:0.074 grdn:10.086 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
167 |
+
INFO 2025-06-14 20:47:08 ts/train.py:232 step:31K smpl:246K ep:988 epch:109.80 loss:0.073 grdn:9.689 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
168 |
+
INFO 2025-06-14 20:47:23 ts/train.py:232 step:31K smpl:248K ep:995 epch:110.52 loss:0.074 grdn:9.903 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
169 |
+
INFO 2025-06-14 20:47:37 ts/train.py:232 step:31K smpl:250K ep:1K epch:111.23 loss:0.072 grdn:9.448 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
170 |
+
INFO 2025-06-14 20:47:51 ts/train.py:232 step:31K smpl:251K ep:1K epch:111.94 loss:0.073 grdn:9.866 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
171 |
+
INFO 2025-06-14 20:48:06 ts/train.py:232 step:32K smpl:253K ep:1K epch:112.66 loss:0.073 grdn:9.879 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
172 |
+
INFO 2025-06-14 20:48:20 ts/train.py:232 step:32K smpl:254K ep:1K epch:113.37 loss:0.072 grdn:9.268 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
173 |
+
INFO 2025-06-14 20:48:35 ts/train.py:232 step:32K smpl:256K ep:1K epch:114.08 loss:0.072 grdn:9.532 lr:1.0e-05 updt_s:0.071 data_s:0.000
|
174 |
+
INFO 2025-06-14 20:48:49 ts/train.py:232 step:32K smpl:258K ep:1K epch:114.80 loss:0.071 grdn:9.071 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
175 |
+
INFO 2025-06-14 20:49:03 ts/train.py:232 step:32K smpl:259K ep:1K epch:115.51 loss:0.071 grdn:9.304 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
176 |
+
INFO 2025-06-14 20:49:18 ts/train.py:232 step:33K smpl:261K ep:1K epch:116.22 loss:0.071 grdn:9.134 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
177 |
+
INFO 2025-06-14 20:49:32 ts/train.py:232 step:33K smpl:262K ep:1K epch:116.93 loss:0.071 grdn:9.155 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
178 |
+
INFO 2025-06-14 20:49:46 ts/train.py:232 step:33K smpl:264K ep:1K epch:117.65 loss:0.071 grdn:9.559 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
179 |
+
INFO 2025-06-14 20:50:01 ts/train.py:232 step:33K smpl:266K ep:1K epch:118.36 loss:0.071 grdn:9.151 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
180 |
+
INFO 2025-06-14 20:50:15 ts/train.py:232 step:33K smpl:267K ep:1K epch:119.07 loss:0.071 grdn:9.337 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
181 |
+
INFO 2025-06-14 20:50:29 ts/train.py:232 step:34K smpl:269K ep:1K epch:119.79 loss:0.070 grdn:8.993 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
182 |
+
INFO 2025-06-14 20:50:44 ts/train.py:232 step:34K smpl:270K ep:1K epch:120.50 loss:0.069 grdn:9.001 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
183 |
+
INFO 2025-06-14 20:50:58 ts/train.py:232 step:34K smpl:272K ep:1K epch:121.21 loss:0.068 grdn:9.136 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
184 |
+
INFO 2025-06-14 20:51:13 ts/train.py:232 step:34K smpl:274K ep:1K epch:121.93 loss:0.068 grdn:9.193 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
185 |
+
INFO 2025-06-14 20:51:27 ts/train.py:232 step:34K smpl:275K ep:1K epch:122.64 loss:0.069 grdn:9.335 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
186 |
+
INFO 2025-06-14 20:51:42 ts/train.py:232 step:35K smpl:277K ep:1K epch:123.35 loss:0.068 grdn:9.611 lr:1.0e-05 updt_s:0.071 data_s:0.002
|
187 |
+
INFO 2025-06-14 20:51:56 ts/train.py:232 step:35K smpl:278K ep:1K epch:124.06 loss:0.068 grdn:9.089 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
188 |
+
INFO 2025-06-14 20:52:10 ts/train.py:232 step:35K smpl:280K ep:1K epch:124.78 loss:0.068 grdn:9.464 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
189 |
+
INFO 2025-06-14 20:52:24 ts/train.py:232 step:35K smpl:282K ep:1K epch:125.49 loss:0.068 grdn:8.916 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
190 |
+
INFO 2025-06-14 20:52:38 ts/train.py:232 step:35K smpl:283K ep:1K epch:126.20 loss:0.067 grdn:8.464 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
191 |
+
INFO 2025-06-14 20:52:52 ts/train.py:232 step:36K smpl:285K ep:1K epch:126.92 loss:0.067 grdn:9.242 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
192 |
+
INFO 2025-06-14 20:53:07 ts/train.py:232 step:36K smpl:286K ep:1K epch:127.63 loss:0.068 grdn:8.885 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
193 |
+
INFO 2025-06-14 20:53:21 ts/train.py:232 step:36K smpl:288K ep:1K epch:128.34 loss:0.065 grdn:8.411 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
194 |
+
INFO 2025-06-14 20:53:35 ts/train.py:232 step:36K smpl:290K ep:1K epch:129.06 loss:0.065 grdn:9.023 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
195 |
+
INFO 2025-06-14 20:53:49 ts/train.py:232 step:36K smpl:291K ep:1K epch:129.77 loss:0.066 grdn:8.840 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
196 |
+
INFO 2025-06-14 20:54:04 ts/train.py:232 step:37K smpl:293K ep:1K epch:130.48 loss:0.065 grdn:8.796 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
197 |
+
INFO 2025-06-14 20:54:18 ts/train.py:232 step:37K smpl:294K ep:1K epch:131.19 loss:0.066 grdn:9.151 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
198 |
+
INFO 2025-06-14 20:54:32 ts/train.py:232 step:37K smpl:296K ep:1K epch:131.91 loss:0.065 grdn:8.574 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
199 |
+
INFO 2025-06-14 20:54:46 ts/train.py:232 step:37K smpl:298K ep:1K epch:132.62 loss:0.065 grdn:8.830 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
200 |
+
INFO 2025-06-14 20:55:00 ts/train.py:232 step:37K smpl:299K ep:1K epch:133.33 loss:0.064 grdn:8.787 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
201 |
+
INFO 2025-06-14 20:55:14 ts/train.py:232 step:38K smpl:301K ep:1K epch:134.05 loss:0.066 grdn:8.757 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
202 |
+
INFO 2025-06-14 20:55:29 ts/train.py:232 step:38K smpl:302K ep:1K epch:134.76 loss:0.064 grdn:8.601 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
203 |
+
INFO 2025-06-14 20:55:43 ts/train.py:232 step:38K smpl:304K ep:1K epch:135.47 loss:0.064 grdn:8.619 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
204 |
+
INFO 2025-06-14 20:55:57 ts/train.py:232 step:38K smpl:306K ep:1K epch:136.19 loss:0.064 grdn:8.578 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
205 |
+
INFO 2025-06-14 20:56:11 ts/train.py:232 step:38K smpl:307K ep:1K epch:136.90 loss:0.064 grdn:8.725 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
206 |
+
INFO 2025-06-14 20:56:26 ts/train.py:232 step:39K smpl:309K ep:1K epch:137.61 loss:0.064 grdn:8.557 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
207 |
+
INFO 2025-06-14 20:56:40 ts/train.py:232 step:39K smpl:310K ep:1K epch:138.32 loss:0.064 grdn:8.590 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
208 |
+
INFO 2025-06-14 20:56:54 ts/train.py:232 step:39K smpl:312K ep:1K epch:139.04 loss:0.063 grdn:8.656 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
209 |
+
INFO 2025-06-14 20:57:08 ts/train.py:232 step:39K smpl:314K ep:1K epch:139.75 loss:0.063 grdn:8.554 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
210 |
+
INFO 2025-06-14 20:57:22 ts/train.py:232 step:39K smpl:315K ep:1K epch:140.46 loss:0.061 grdn:7.760 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
211 |
+
INFO 2025-06-14 20:57:36 ts/train.py:232 step:40K smpl:317K ep:1K epch:141.18 loss:0.062 grdn:8.565 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
212 |
+
INFO 2025-06-14 20:57:51 ts/train.py:232 step:40K smpl:318K ep:1K epch:141.89 loss:0.064 grdn:8.273 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
213 |
+
INFO 2025-06-14 20:58:05 ts/train.py:232 step:40K smpl:320K ep:1K epch:142.60 loss:0.062 grdn:8.144 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
214 |
+
INFO 2025-06-14 20:58:05 ts/train.py:241 Checkpoint policy after step 40000
|
215 |
+
INFO 2025-06-14 20:58:20 ts/train.py:232 step:40K smpl:322K ep:1K epch:143.32 loss:0.063 grdn:8.797 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
216 |
+
INFO 2025-06-14 20:58:35 ts/train.py:232 step:40K smpl:323K ep:1K epch:144.03 loss:0.061 grdn:8.157 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
217 |
+
INFO 2025-06-14 20:58:49 ts/train.py:232 step:41K smpl:325K ep:1K epch:144.74 loss:0.063 grdn:8.521 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
218 |
+
INFO 2025-06-14 20:59:03 ts/train.py:232 step:41K smpl:326K ep:1K epch:145.45 loss:0.062 grdn:8.052 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
219 |
+
INFO 2025-06-14 20:59:17 ts/train.py:232 step:41K smpl:328K ep:1K epch:146.17 loss:0.060 grdn:8.172 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
220 |
+
INFO 2025-06-14 20:59:31 ts/train.py:232 step:41K smpl:330K ep:1K epch:146.88 loss:0.061 grdn:8.706 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
221 |
+
INFO 2025-06-14 20:59:46 ts/train.py:232 step:41K smpl:331K ep:1K epch:147.59 loss:0.062 grdn:8.565 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
222 |
+
INFO 2025-06-14 21:00:00 ts/train.py:232 step:42K smpl:333K ep:1K epch:148.31 loss:0.060 grdn:8.123 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
223 |
+
INFO 2025-06-14 21:00:14 ts/train.py:232 step:42K smpl:334K ep:1K epch:149.02 loss:0.060 grdn:8.211 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
224 |
+
INFO 2025-06-14 21:00:28 ts/train.py:232 step:42K smpl:336K ep:1K epch:149.73 loss:0.062 grdn:8.367 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
225 |
+
INFO 2025-06-14 21:00:43 ts/train.py:232 step:42K smpl:338K ep:1K epch:150.45 loss:0.060 grdn:8.225 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
226 |
+
Traceback (most recent call last):
|
227 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 288, in <module>
|
228 |
+
train()
|
229 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/configs/parser.py", line 226, in wrapper_inner
|
230 |
+
response = fn(cfg, *args, **kwargs)
|
231 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 212, in train
|
232 |
+
train_tracker, output_dict = update_policy(
|
233 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 71, in update_policy
|
234 |
+
loss, output_dict = policy.forward(batch)
|
235 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 153, in forward
|
236 |
+
actions_hat, (mu_hat, log_sigma_x2_hat) = self.model(batch)
|
237 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
238 |
+
return self._call_impl(*args, **kwargs)
|
239 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
240 |
+
return forward_call(*args, **kwargs)
|
241 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 452, in forward
|
242 |
+
cls_token_out = self.vae_encoder(
|
243 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
244 |
+
return self._call_impl(*args, **kwargs)
|
245 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
246 |
+
return forward_call(*args, **kwargs)
|
247 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 546, in forward
|
248 |
+
x = layer(x, pos_embed=pos_embed, key_padding_mask=key_padding_mask)
|
249 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
250 |
+
return self._call_impl(*args, **kwargs)
|
251 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
252 |
+
return forward_call(*args, **kwargs)
|
253 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 584, in forward
|
254 |
+
x = skip + self.dropout2(x)
|
255 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
256 |
+
return self._call_impl(*args, **kwargs)
|
257 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
258 |
+
return forward_call(*args, **kwargs)
|
259 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/dropout.py", line 70, in forward
|
260 |
+
return F.dropout(input, self.p, self.training, self.inplace)
|
261 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/functional.py", line 1425, in dropout
|
262 |
+
_VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)
|
263 |
+
KeyboardInterrupt
|
264 |
+
Traceback (most recent call last):
|
265 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 288, in <module>
|
266 |
+
train()
|
267 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/configs/parser.py", line 226, in wrapper_inner
|
268 |
+
response = fn(cfg, *args, **kwargs)
|
269 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 212, in train
|
270 |
+
train_tracker, output_dict = update_policy(
|
271 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py", line 71, in update_policy
|
272 |
+
loss, output_dict = policy.forward(batch)
|
273 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 153, in forward
|
274 |
+
actions_hat, (mu_hat, log_sigma_x2_hat) = self.model(batch)
|
275 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
276 |
+
return self._call_impl(*args, **kwargs)
|
277 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
278 |
+
return forward_call(*args, **kwargs)
|
279 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 452, in forward
|
280 |
+
cls_token_out = self.vae_encoder(
|
281 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
282 |
+
return self._call_impl(*args, **kwargs)
|
283 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
284 |
+
return forward_call(*args, **kwargs)
|
285 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 546, in forward
|
286 |
+
x = layer(x, pos_embed=pos_embed, key_padding_mask=key_padding_mask)
|
287 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
288 |
+
return self._call_impl(*args, **kwargs)
|
289 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
290 |
+
return forward_call(*args, **kwargs)
|
291 |
+
File "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/common/policies/act/modeling_act.py", line 584, in forward
|
292 |
+
x = skip + self.dropout2(x)
|
293 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
|
294 |
+
return self._call_impl(*args, **kwargs)
|
295 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
|
296 |
+
return forward_call(*args, **kwargs)
|
297 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/dropout.py", line 70, in forward
|
298 |
+
return F.dropout(input, self.p, self.training, self.inplace)
|
299 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/nn/functional.py", line 1425, in dropout
|
300 |
+
_VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training)
|
301 |
+
KeyboardInterrupt
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/requirements.txt
ADDED
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
starlette==0.46.2
|
2 |
+
httpx-sse==0.4.0
|
3 |
+
regex==2024.11.6
|
4 |
+
markdown-it-py==3.0.0
|
5 |
+
mdurl==0.1.2
|
6 |
+
groovy==0.1.2
|
7 |
+
tqdm==4.67.1
|
8 |
+
joblib==1.5.1
|
9 |
+
httpcore==1.0.9
|
10 |
+
typer==0.16.0
|
11 |
+
pydantic_core==2.33.2
|
12 |
+
typing_extensions==4.14.0
|
13 |
+
mcp==1.9.0
|
14 |
+
aiofiles==24.1.0
|
15 |
+
ruff==0.11.13
|
16 |
+
packaging==25.0
|
17 |
+
pydantic-settings==2.9.1
|
18 |
+
exceptiongroup==1.3.0
|
19 |
+
pandas==2.3.0
|
20 |
+
Jinja2==3.1.6
|
21 |
+
nltk==3.9.1
|
22 |
+
websockets==15.0.1
|
23 |
+
tzdata==2025.2
|
24 |
+
semantic-version==2.10.0
|
25 |
+
fastapi==0.115.12
|
26 |
+
orjson==3.10.18
|
27 |
+
textblob==0.19.0
|
28 |
+
h11==0.16.0
|
29 |
+
python-dateutil==2.9.0.post0
|
30 |
+
pydub==0.25.1
|
31 |
+
sse-starlette==2.3.6
|
32 |
+
tomlkit==0.13.3
|
33 |
+
Pygments==2.19.1
|
34 |
+
anyio==4.9.0
|
35 |
+
shellingham==1.5.4
|
36 |
+
typing-inspection==0.4.1
|
37 |
+
pydantic==2.11.5
|
38 |
+
rich==14.0.0
|
39 |
+
numpy==2.2.6
|
40 |
+
uvicorn==0.34.3
|
41 |
+
huggingface-hub==0.32.4
|
42 |
+
annotated-types==0.7.0
|
43 |
+
python-dotenv==1.1.0
|
44 |
+
safehttpx==0.1.6
|
45 |
+
ffmpy==0.6.0
|
46 |
+
httpx==0.28.1
|
47 |
+
gradio_client==1.10.2
|
48 |
+
python-multipart==0.0.20
|
49 |
+
gradio==5.33.0
|
50 |
+
filelock==3.18.0
|
51 |
+
hf-xet==1.1.3
|
52 |
+
sniffio==1.3.1
|
53 |
+
itsdangerous==2.2.0
|
54 |
+
toml==0.10.2
|
55 |
+
multidict==6.4.4
|
56 |
+
mpmath==1.3.0
|
57 |
+
protobuf==6.31.1
|
58 |
+
cloudpickle==3.1.1
|
59 |
+
torchvision==0.22.1
|
60 |
+
prompt_toolkit==3.0.51
|
61 |
+
blinker==1.9.0
|
62 |
+
PySocks==1.7.1
|
63 |
+
beautifulsoup4==4.13.4
|
64 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
65 |
+
sentry-sdk==2.30.0
|
66 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
67 |
+
h5py==3.14.0
|
68 |
+
aiohttp==3.12.13
|
69 |
+
gitdb==4.0.12
|
70 |
+
setproctitle==1.3.6
|
71 |
+
GitPython==3.1.44
|
72 |
+
six==1.17.0
|
73 |
+
einops==0.8.1
|
74 |
+
Farama-Notifications==0.0.4
|
75 |
+
diffusers==0.33.1
|
76 |
+
cffi==1.17.1
|
77 |
+
pyarrow==20.0.0
|
78 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
79 |
+
zipp==3.23.0
|
80 |
+
pycparser==2.22
|
81 |
+
nvidia-nccl-cu12==2.26.2
|
82 |
+
zarr==2.18.3
|
83 |
+
antlr4-python3-runtime==4.9.3
|
84 |
+
numcodecs==0.13.1
|
85 |
+
MarkupSafe==3.0.2
|
86 |
+
pip==25.1
|
87 |
+
mergedeep==1.3.4
|
88 |
+
certifi==2025.6.15
|
89 |
+
soupsieve==2.7
|
90 |
+
numba==0.61.2
|
91 |
+
safetensors==0.5.3
|
92 |
+
gymnasium==0.29.1
|
93 |
+
multiprocess==0.70.16
|
94 |
+
idna==3.10
|
95 |
+
orderly-set==5.4.1
|
96 |
+
networkx==3.4.2
|
97 |
+
sympy==1.14.0
|
98 |
+
xxhash==3.5.0
|
99 |
+
pillow==11.2.1
|
100 |
+
draccus==0.10.0
|
101 |
+
platformdirs==4.3.8
|
102 |
+
pyserial==3.5
|
103 |
+
requests==2.32.4
|
104 |
+
fasteners==0.19
|
105 |
+
pyzmq==27.0.0
|
106 |
+
nvidia-cudnn-cu12==9.5.1.17
|
107 |
+
torch==2.7.1
|
108 |
+
nvidia-cusparse-cu12==12.5.4.2
|
109 |
+
psutil==7.0.0
|
110 |
+
scipy==1.15.3
|
111 |
+
async-timeout==5.0.1
|
112 |
+
urllib3==2.4.0
|
113 |
+
nvidia-curand-cu12==10.3.7.77
|
114 |
+
av==14.4.0
|
115 |
+
attrs==25.3.0
|
116 |
+
smmap==5.0.2
|
117 |
+
wandb==0.20.1
|
118 |
+
hf_transfer==0.1.9
|
119 |
+
propcache==0.3.2
|
120 |
+
setuptools==78.1.1
|
121 |
+
imageio-ffmpeg==0.6.0
|
122 |
+
pynput==1.8.1
|
123 |
+
pytz==2025.2
|
124 |
+
pfzy==0.3.4
|
125 |
+
torchcodec==0.4.0
|
126 |
+
imageio==2.37.0
|
127 |
+
nvidia-cublas-cu12==12.6.4.1
|
128 |
+
triton==3.3.1
|
129 |
+
pymunk==6.11.1
|
130 |
+
wcwidth==0.2.13
|
131 |
+
jsonlines==4.0.0
|
132 |
+
python-xlib==0.33
|
133 |
+
mypy_extensions==1.1.0
|
134 |
+
cmake==4.0.3
|
135 |
+
evdev==1.9.2
|
136 |
+
omegaconf==2.3.0
|
137 |
+
asciitree==0.3.3
|
138 |
+
nvidia-cufft-cu12==11.3.0.4
|
139 |
+
charset-normalizer==3.4.2
|
140 |
+
deepdiff==8.5.0
|
141 |
+
click==8.2.1
|
142 |
+
termcolor==3.1.0
|
143 |
+
rerun-sdk==0.23.3
|
144 |
+
aiosignal==1.3.2
|
145 |
+
llvmlite==0.44.0
|
146 |
+
PyYAML==6.0.2
|
147 |
+
importlib_metadata==8.7.0
|
148 |
+
lerobot==0.1.0
|
149 |
+
frozenlist==1.7.0
|
150 |
+
Werkzeug==3.1.3
|
151 |
+
nvidia-nvjitlink-cu12==12.6.85
|
152 |
+
typing-inspect==0.9.0
|
153 |
+
nvidia-cusolver-cu12==11.7.1.2
|
154 |
+
dill==0.3.8
|
155 |
+
pyyaml-include==1.4.1
|
156 |
+
gdown==5.2.0
|
157 |
+
datasets==3.6.0
|
158 |
+
fsspec==2025.3.0
|
159 |
+
yarl==1.20.1
|
160 |
+
inquirerpy==0.3.4
|
161 |
+
nvidia-nvtx-cu12==12.6.77
|
162 |
+
Flask==3.1.1
|
163 |
+
aiohappyeyeballs==2.6.1
|
164 |
+
opencv-python-headless==4.11.0.86
|
165 |
+
wheel==0.45.1
|
166 |
+
nvidia-cufile-cu12==1.11.1.6
|
167 |
+
nvidia-cusparselt-cu12==0.6.3
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-metadata.json
ADDED
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-6.8.0-60-generic-x86_64-with-glibc2.35",
|
3 |
+
"python": "CPython 3.10.13",
|
4 |
+
"startedAt": "2025-06-15T03:09:51.326969Z",
|
5 |
+
"args": [
|
6 |
+
"--dataset.repo_id=BoredMachinist/dice-art-run1",
|
7 |
+
"--dataset.video_backend=pyav",
|
8 |
+
"--policy.type=act",
|
9 |
+
"--output_dir=outputs/train/act_dice_art_run1",
|
10 |
+
"--job_name=act_dice_art_run1",
|
11 |
+
"--policy.device=cuda",
|
12 |
+
"--wandb.enable=true"
|
13 |
+
],
|
14 |
+
"program": "/home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/lerobot/scripts/train.py",
|
15 |
+
"codePath": "lerobot/scripts/train.py",
|
16 |
+
"git": {
|
17 |
+
"remote": "https://github.com/huggingface/lerobot.git",
|
18 |
+
"commit": "b8637c09ec93ca30d7901f5fdcf47aa4020cfe89"
|
19 |
+
},
|
20 |
+
"root": "outputs/train/act_dice_art_run1",
|
21 |
+
"host": "hongyi-J",
|
22 |
+
"executable": "/home/hshi-j-4090/miniconda3/envs/lerobot/bin/python",
|
23 |
+
"codePathLocal": "lerobot/scripts/train.py",
|
24 |
+
"cpu_count": 24,
|
25 |
+
"cpu_count_logical": 32,
|
26 |
+
"gpu": "NVIDIA GeForce RTX 4090",
|
27 |
+
"gpu_count": 1,
|
28 |
+
"disk": {
|
29 |
+
"/": {
|
30 |
+
"total": "3739429154816",
|
31 |
+
"used": "152715747328"
|
32 |
+
}
|
33 |
+
},
|
34 |
+
"memory": {
|
35 |
+
"total": "67168518144"
|
36 |
+
},
|
37 |
+
"cpu": {
|
38 |
+
"count": 24,
|
39 |
+
"countLogical": 32
|
40 |
+
},
|
41 |
+
"gpu_nvidia": [
|
42 |
+
{
|
43 |
+
"name": "NVIDIA GeForce RTX 4090",
|
44 |
+
"memoryTotal": "25757220864",
|
45 |
+
"cudaCores": 16384,
|
46 |
+
"architecture": "Ada",
|
47 |
+
"uuid": "GPU-5b318751-a95f-5805-6c0e-a4b86dcd442d"
|
48 |
+
}
|
49 |
+
],
|
50 |
+
"cudaVersion": "12.2"
|
51 |
+
}
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"train/steps":42200,"train/episodes":1354.0106951871658,"_timestamp":1.7499600435914388e+09,"train/kld_loss":0.0003022812306880951,"_runtime":3059.972341236,"train/samples":337600,"train/grad_norm":8.225247728824616,"train/l1_loss":0.05051584169268608,"train/dataloading_s":0.0016573248049689938,"_step":42200,"train/update_s":0.07279860169501717,"train/loss":0.059818110205233095,"train/lr":1.0000000000000021e-05,"_wandb":{"runtime":3059},"train/epochs":150.44563279857397}
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{"time":"2025-06-14T20:09:51.124778953-07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_d1l9fr6/port-11388.txt","pid":11388,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
2 |
+
{"time":"2025-06-14T20:09:51.126955456-07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37013,"Zone":""}}
|
3 |
+
{"time":"2025-06-14T20:09:51.127005411-07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":11388}
|
4 |
+
{"time":"2025-06-14T20:09:51.318865438-07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:41090"}
|
5 |
+
{"time":"2025-06-14T20:09:51.330293507-07:00","level":"INFO","msg":"handleInformInit: received","streamId":"fhdgxxjz","id":"127.0.0.1:41090"}
|
6 |
+
{"time":"2025-06-14T20:09:51.792331002-07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fhdgxxjz","id":"127.0.0.1:41090"}
|
7 |
+
{"time":"2025-06-14T21:00:51.299091378-07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:41090"}
|
8 |
+
{"time":"2025-06-14T21:00:51.299209042-07:00","level":"INFO","msg":"server is shutting down"}
|
9 |
+
{"time":"2025-06-14T21:00:51.299173348-07:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:41090"}
|
10 |
+
{"time":"2025-06-14T21:00:51.29937174-07:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:41090"}
|
11 |
+
{"time":"2025-06-14T21:00:52.297381201-07:00","level":"INFO","msg":"Parent process exited, terminating service process."}
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{"time":"2025-06-14T20:09:51.33052201-07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-core.log"}
|
2 |
+
{"time":"2025-06-14T20:09:51.792313834-07:00","level":"INFO","msg":"stream: created new stream","id":"fhdgxxjz"}
|
3 |
+
{"time":"2025-06-14T20:09:51.792328297-07:00","level":"INFO","msg":"stream: started","id":"fhdgxxjz"}
|
4 |
+
{"time":"2025-06-14T20:09:51.792427231-07:00","level":"INFO","msg":"sender: started","stream_id":"fhdgxxjz"}
|
5 |
+
{"time":"2025-06-14T20:09:51.792424884-07:00","level":"INFO","msg":"writer: Do: started","stream_id":"fhdgxxjz"}
|
6 |
+
{"time":"2025-06-14T20:09:51.792446588-07:00","level":"INFO","msg":"handler: started","stream_id":"fhdgxxjz"}
|
7 |
+
{"time":"2025-06-14T20:09:52.128559874-07:00","level":"INFO","msg":"Starting system monitor"}
|
8 |
+
{"time":"2025-06-14T21:00:51.299276318-07:00","level":"INFO","msg":"stream: closing","id":"fhdgxxjz"}
|
9 |
+
{"time":"2025-06-14T21:00:51.299350521-07:00","level":"INFO","msg":"Stopping system monitor"}
|
10 |
+
{"time":"2025-06-14T21:00:51.299409681-07:00","level":"INFO","msg":"Stopped system monitor"}
|
11 |
+
{"time":"2025-06-14T21:00:51.872219071-07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log
ADDED
@@ -0,0 +1,886 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
|
2 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Configure stats pid to 11388
|
3 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/.config/wandb/settings
|
4 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from /home/hshi-j-4090/Desktop/SO-ARM-Art/lerobot/wandb/settings
|
5 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
6 |
+
2025-06-14 20:09:51,114 INFO MainThread:11388 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug.log
|
7 |
+
2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/logs/debug-internal.log
|
8 |
+
2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():831] calling init triggers
|
9 |
+
2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
|
10 |
+
config: {'dataset': {'repo_id': 'BoredMachinist/dice-art-run1', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/act_dice_art_run1', 'job_name': 'act_dice_art_run1', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 100000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
|
11 |
+
2025-06-14 20:09:51,115 INFO MainThread:11388 [wandb_init.py:init():872] starting backend
|
12 |
+
2025-06-14 20:09:51,319 INFO MainThread:11388 [wandb_init.py:init():875] sending inform_init request
|
13 |
+
2025-06-14 20:09:51,326 INFO MainThread:11388 [wandb_init.py:init():883] backend started and connected
|
14 |
+
2025-06-14 20:09:51,331 INFO MainThread:11388 [wandb_init.py:init():956] updated telemetry
|
15 |
+
2025-06-14 20:09:51,343 INFO MainThread:11388 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
|
16 |
+
2025-06-14 20:09:52,123 INFO MainThread:11388 [wandb_init.py:init():1032] starting run threads in backend
|
17 |
+
2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_console_start():2453] atexit reg
|
18 |
+
2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2301] redirect: wrap_raw
|
19 |
+
2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2370] Wrapping output streams.
|
20 |
+
2025-06-14 20:09:52,192 INFO MainThread:11388 [wandb_run.py:_redirect():2393] Redirects installed.
|
21 |
+
2025-06-14 20:09:52,193 INFO MainThread:11388 [wandb_init.py:init():1078] run started, returning control to user process
|
22 |
+
2025-06-14 21:00:51,298 INFO MsgRouterThr:11388 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
|
23 |
+
2025-06-14 21:00:51,942 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
24 |
+
Traceback (most recent call last):
|
25 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
26 |
+
cb(written_data)
|
27 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
28 |
+
lambda data: self._console_raw_callback("stderr", data),
|
29 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
30 |
+
return func(self, *args, **kwargs)
|
31 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
32 |
+
return func(self, *args, **kwargs)
|
33 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
34 |
+
self._backend.interface.publish_output_raw(name, data)
|
35 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
36 |
+
self._publish_output_raw(o)
|
37 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
38 |
+
self._publish(rec)
|
39 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
40 |
+
self._sock_client.send_record_publish(record)
|
41 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
42 |
+
self.send_server_request(server_req)
|
43 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
44 |
+
self._send_message(msg)
|
45 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
46 |
+
self._sendall_with_error_handle(header + data)
|
47 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
48 |
+
sent = self._sock.send(data)
|
49 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
50 |
+
2025-06-14 21:00:51,946 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
51 |
+
Traceback (most recent call last):
|
52 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
53 |
+
cb(written_data)
|
54 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
55 |
+
lambda data: self._console_raw_callback("stderr", data),
|
56 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
57 |
+
return func(self, *args, **kwargs)
|
58 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
59 |
+
return func(self, *args, **kwargs)
|
60 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
61 |
+
self._backend.interface.publish_output_raw(name, data)
|
62 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
63 |
+
self._publish_output_raw(o)
|
64 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
65 |
+
self._publish(rec)
|
66 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
67 |
+
self._sock_client.send_record_publish(record)
|
68 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
69 |
+
self.send_server_request(server_req)
|
70 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
71 |
+
self._send_message(msg)
|
72 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
73 |
+
self._sendall_with_error_handle(header + data)
|
74 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
75 |
+
sent = self._sock.send(data)
|
76 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
77 |
+
2025-06-14 21:00:51,946 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
78 |
+
Traceback (most recent call last):
|
79 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
80 |
+
cb(written_data)
|
81 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
82 |
+
lambda data: self._console_raw_callback("stderr", data),
|
83 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
84 |
+
return func(self, *args, **kwargs)
|
85 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
86 |
+
return func(self, *args, **kwargs)
|
87 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
88 |
+
self._backend.interface.publish_output_raw(name, data)
|
89 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
90 |
+
self._publish_output_raw(o)
|
91 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
92 |
+
self._publish(rec)
|
93 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
94 |
+
self._sock_client.send_record_publish(record)
|
95 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
96 |
+
self.send_server_request(server_req)
|
97 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
98 |
+
self._send_message(msg)
|
99 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
100 |
+
self._sendall_with_error_handle(header + data)
|
101 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
102 |
+
sent = self._sock.send(data)
|
103 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
104 |
+
2025-06-14 21:00:51,948 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
105 |
+
Traceback (most recent call last):
|
106 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
107 |
+
cb(written_data)
|
108 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
109 |
+
lambda data: self._console_raw_callback("stderr", data),
|
110 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
111 |
+
return func(self, *args, **kwargs)
|
112 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
113 |
+
return func(self, *args, **kwargs)
|
114 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
115 |
+
self._backend.interface.publish_output_raw(name, data)
|
116 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
117 |
+
self._publish_output_raw(o)
|
118 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
119 |
+
self._publish(rec)
|
120 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
121 |
+
self._sock_client.send_record_publish(record)
|
122 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
123 |
+
self.send_server_request(server_req)
|
124 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
125 |
+
self._send_message(msg)
|
126 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
127 |
+
self._sendall_with_error_handle(header + data)
|
128 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
129 |
+
sent = self._sock.send(data)
|
130 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
131 |
+
2025-06-14 21:00:51,949 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
132 |
+
Traceback (most recent call last):
|
133 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
134 |
+
cb(written_data)
|
135 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
136 |
+
lambda data: self._console_raw_callback("stderr", data),
|
137 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
138 |
+
return func(self, *args, **kwargs)
|
139 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
140 |
+
return func(self, *args, **kwargs)
|
141 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
142 |
+
self._backend.interface.publish_output_raw(name, data)
|
143 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
144 |
+
self._publish_output_raw(o)
|
145 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
146 |
+
self._publish(rec)
|
147 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
148 |
+
self._sock_client.send_record_publish(record)
|
149 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
150 |
+
self.send_server_request(server_req)
|
151 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
152 |
+
self._send_message(msg)
|
153 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
154 |
+
self._sendall_with_error_handle(header + data)
|
155 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
156 |
+
sent = self._sock.send(data)
|
157 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
158 |
+
2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
159 |
+
Traceback (most recent call last):
|
160 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
161 |
+
cb(written_data)
|
162 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
163 |
+
lambda data: self._console_raw_callback("stderr", data),
|
164 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
165 |
+
return func(self, *args, **kwargs)
|
166 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
167 |
+
return func(self, *args, **kwargs)
|
168 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
169 |
+
self._backend.interface.publish_output_raw(name, data)
|
170 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
171 |
+
self._publish_output_raw(o)
|
172 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
173 |
+
self._publish(rec)
|
174 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
175 |
+
self._sock_client.send_record_publish(record)
|
176 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
177 |
+
self.send_server_request(server_req)
|
178 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
179 |
+
self._send_message(msg)
|
180 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
181 |
+
self._sendall_with_error_handle(header + data)
|
182 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
183 |
+
sent = self._sock.send(data)
|
184 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
185 |
+
2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
186 |
+
Traceback (most recent call last):
|
187 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
188 |
+
cb(written_data)
|
189 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
190 |
+
lambda data: self._console_raw_callback("stderr", data),
|
191 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
192 |
+
return func(self, *args, **kwargs)
|
193 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
194 |
+
return func(self, *args, **kwargs)
|
195 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
196 |
+
self._backend.interface.publish_output_raw(name, data)
|
197 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
198 |
+
self._publish_output_raw(o)
|
199 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
200 |
+
self._publish(rec)
|
201 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
202 |
+
self._sock_client.send_record_publish(record)
|
203 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
204 |
+
self.send_server_request(server_req)
|
205 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
206 |
+
self._send_message(msg)
|
207 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
208 |
+
self._sendall_with_error_handle(header + data)
|
209 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
210 |
+
sent = self._sock.send(data)
|
211 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
212 |
+
2025-06-14 21:00:51,950 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
213 |
+
Traceback (most recent call last):
|
214 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
215 |
+
cb(written_data)
|
216 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
217 |
+
lambda data: self._console_raw_callback("stderr", data),
|
218 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
219 |
+
return func(self, *args, **kwargs)
|
220 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
221 |
+
return func(self, *args, **kwargs)
|
222 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
223 |
+
self._backend.interface.publish_output_raw(name, data)
|
224 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
225 |
+
self._publish_output_raw(o)
|
226 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
227 |
+
self._publish(rec)
|
228 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
229 |
+
self._sock_client.send_record_publish(record)
|
230 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
231 |
+
self.send_server_request(server_req)
|
232 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
233 |
+
self._send_message(msg)
|
234 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
235 |
+
self._sendall_with_error_handle(header + data)
|
236 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
237 |
+
sent = self._sock.send(data)
|
238 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
239 |
+
2025-06-14 21:00:51,951 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
240 |
+
Traceback (most recent call last):
|
241 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
242 |
+
cb(written_data)
|
243 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
244 |
+
lambda data: self._console_raw_callback("stderr", data),
|
245 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
246 |
+
return func(self, *args, **kwargs)
|
247 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
248 |
+
return func(self, *args, **kwargs)
|
249 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
250 |
+
self._backend.interface.publish_output_raw(name, data)
|
251 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
252 |
+
self._publish_output_raw(o)
|
253 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
254 |
+
self._publish(rec)
|
255 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
256 |
+
self._sock_client.send_record_publish(record)
|
257 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
258 |
+
self.send_server_request(server_req)
|
259 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
260 |
+
self._send_message(msg)
|
261 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
262 |
+
self._sendall_with_error_handle(header + data)
|
263 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
264 |
+
sent = self._sock.send(data)
|
265 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
266 |
+
2025-06-14 21:00:51,951 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
267 |
+
Traceback (most recent call last):
|
268 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
269 |
+
cb(written_data)
|
270 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
271 |
+
lambda data: self._console_raw_callback("stderr", data),
|
272 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
273 |
+
return func(self, *args, **kwargs)
|
274 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
275 |
+
return func(self, *args, **kwargs)
|
276 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
277 |
+
self._backend.interface.publish_output_raw(name, data)
|
278 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
279 |
+
self._publish_output_raw(o)
|
280 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
281 |
+
self._publish(rec)
|
282 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
283 |
+
self._sock_client.send_record_publish(record)
|
284 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
285 |
+
self.send_server_request(server_req)
|
286 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
287 |
+
self._send_message(msg)
|
288 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
289 |
+
self._sendall_with_error_handle(header + data)
|
290 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
291 |
+
sent = self._sock.send(data)
|
292 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
293 |
+
2025-06-14 21:00:51,952 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
294 |
+
Traceback (most recent call last):
|
295 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
296 |
+
cb(written_data)
|
297 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
298 |
+
lambda data: self._console_raw_callback("stderr", data),
|
299 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
300 |
+
return func(self, *args, **kwargs)
|
301 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
302 |
+
return func(self, *args, **kwargs)
|
303 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
304 |
+
self._backend.interface.publish_output_raw(name, data)
|
305 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
306 |
+
self._publish_output_raw(o)
|
307 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
308 |
+
self._publish(rec)
|
309 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
310 |
+
self._sock_client.send_record_publish(record)
|
311 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
312 |
+
self.send_server_request(server_req)
|
313 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
314 |
+
self._send_message(msg)
|
315 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
316 |
+
self._sendall_with_error_handle(header + data)
|
317 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
318 |
+
sent = self._sock.send(data)
|
319 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
320 |
+
2025-06-14 21:00:51,952 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
321 |
+
Traceback (most recent call last):
|
322 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
323 |
+
cb(written_data)
|
324 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
325 |
+
lambda data: self._console_raw_callback("stderr", data),
|
326 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
327 |
+
return func(self, *args, **kwargs)
|
328 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
329 |
+
return func(self, *args, **kwargs)
|
330 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
331 |
+
self._backend.interface.publish_output_raw(name, data)
|
332 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
333 |
+
self._publish_output_raw(o)
|
334 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
335 |
+
self._publish(rec)
|
336 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
337 |
+
self._sock_client.send_record_publish(record)
|
338 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
339 |
+
self.send_server_request(server_req)
|
340 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
341 |
+
self._send_message(msg)
|
342 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
343 |
+
self._sendall_with_error_handle(header + data)
|
344 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
345 |
+
sent = self._sock.send(data)
|
346 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
347 |
+
2025-06-14 21:00:51,953 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
348 |
+
Traceback (most recent call last):
|
349 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
350 |
+
cb(written_data)
|
351 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
352 |
+
lambda data: self._console_raw_callback("stderr", data),
|
353 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
354 |
+
return func(self, *args, **kwargs)
|
355 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
356 |
+
return func(self, *args, **kwargs)
|
357 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
358 |
+
self._backend.interface.publish_output_raw(name, data)
|
359 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
360 |
+
self._publish_output_raw(o)
|
361 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
362 |
+
self._publish(rec)
|
363 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
364 |
+
self._sock_client.send_record_publish(record)
|
365 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
366 |
+
self.send_server_request(server_req)
|
367 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
368 |
+
self._send_message(msg)
|
369 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
370 |
+
self._sendall_with_error_handle(header + data)
|
371 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
372 |
+
sent = self._sock.send(data)
|
373 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
374 |
+
2025-06-14 21:00:51,954 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
375 |
+
Traceback (most recent call last):
|
376 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
377 |
+
cb(written_data)
|
378 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
379 |
+
lambda data: self._console_raw_callback("stderr", data),
|
380 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
381 |
+
return func(self, *args, **kwargs)
|
382 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
383 |
+
return func(self, *args, **kwargs)
|
384 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
385 |
+
self._backend.interface.publish_output_raw(name, data)
|
386 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
387 |
+
self._publish_output_raw(o)
|
388 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
389 |
+
self._publish(rec)
|
390 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
391 |
+
self._sock_client.send_record_publish(record)
|
392 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
393 |
+
self.send_server_request(server_req)
|
394 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
395 |
+
self._send_message(msg)
|
396 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
397 |
+
self._sendall_with_error_handle(header + data)
|
398 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
399 |
+
sent = self._sock.send(data)
|
400 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
401 |
+
2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
402 |
+
Traceback (most recent call last):
|
403 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
404 |
+
cb(written_data)
|
405 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
406 |
+
lambda data: self._console_raw_callback("stderr", data),
|
407 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
408 |
+
return func(self, *args, **kwargs)
|
409 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
410 |
+
return func(self, *args, **kwargs)
|
411 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
412 |
+
self._backend.interface.publish_output_raw(name, data)
|
413 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
414 |
+
self._publish_output_raw(o)
|
415 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
416 |
+
self._publish(rec)
|
417 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
418 |
+
self._sock_client.send_record_publish(record)
|
419 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
420 |
+
self.send_server_request(server_req)
|
421 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
422 |
+
self._send_message(msg)
|
423 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
424 |
+
self._sendall_with_error_handle(header + data)
|
425 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
426 |
+
sent = self._sock.send(data)
|
427 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
428 |
+
2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
429 |
+
Traceback (most recent call last):
|
430 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
431 |
+
cb(written_data)
|
432 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
433 |
+
lambda data: self._console_raw_callback("stderr", data),
|
434 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
435 |
+
return func(self, *args, **kwargs)
|
436 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
437 |
+
return func(self, *args, **kwargs)
|
438 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
439 |
+
self._backend.interface.publish_output_raw(name, data)
|
440 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
441 |
+
self._publish_output_raw(o)
|
442 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
443 |
+
self._publish(rec)
|
444 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
445 |
+
self._sock_client.send_record_publish(record)
|
446 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
447 |
+
self.send_server_request(server_req)
|
448 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
449 |
+
self._send_message(msg)
|
450 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
451 |
+
self._sendall_with_error_handle(header + data)
|
452 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
453 |
+
sent = self._sock.send(data)
|
454 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
455 |
+
2025-06-14 21:00:51,955 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
456 |
+
Traceback (most recent call last):
|
457 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
458 |
+
cb(written_data)
|
459 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
460 |
+
lambda data: self._console_raw_callback("stderr", data),
|
461 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
462 |
+
return func(self, *args, **kwargs)
|
463 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
464 |
+
return func(self, *args, **kwargs)
|
465 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
466 |
+
self._backend.interface.publish_output_raw(name, data)
|
467 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
468 |
+
self._publish_output_raw(o)
|
469 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
470 |
+
self._publish(rec)
|
471 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
472 |
+
self._sock_client.send_record_publish(record)
|
473 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
474 |
+
self.send_server_request(server_req)
|
475 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
476 |
+
self._send_message(msg)
|
477 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
478 |
+
self._sendall_with_error_handle(header + data)
|
479 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
480 |
+
sent = self._sock.send(data)
|
481 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
482 |
+
2025-06-14 21:00:51,956 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
483 |
+
Traceback (most recent call last):
|
484 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
485 |
+
cb(written_data)
|
486 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
487 |
+
lambda data: self._console_raw_callback("stderr", data),
|
488 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
489 |
+
return func(self, *args, **kwargs)
|
490 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
491 |
+
return func(self, *args, **kwargs)
|
492 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
493 |
+
self._backend.interface.publish_output_raw(name, data)
|
494 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
495 |
+
self._publish_output_raw(o)
|
496 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
497 |
+
self._publish(rec)
|
498 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
499 |
+
self._sock_client.send_record_publish(record)
|
500 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
501 |
+
self.send_server_request(server_req)
|
502 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
503 |
+
self._send_message(msg)
|
504 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
505 |
+
self._sendall_with_error_handle(header + data)
|
506 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
507 |
+
sent = self._sock.send(data)
|
508 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
509 |
+
2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
510 |
+
Traceback (most recent call last):
|
511 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
512 |
+
cb(written_data)
|
513 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
514 |
+
lambda data: self._console_raw_callback("stderr", data),
|
515 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
516 |
+
return func(self, *args, **kwargs)
|
517 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
518 |
+
return func(self, *args, **kwargs)
|
519 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
520 |
+
self._backend.interface.publish_output_raw(name, data)
|
521 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
522 |
+
self._publish_output_raw(o)
|
523 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
524 |
+
self._publish(rec)
|
525 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
526 |
+
self._sock_client.send_record_publish(record)
|
527 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
528 |
+
self.send_server_request(server_req)
|
529 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
530 |
+
self._send_message(msg)
|
531 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
532 |
+
self._sendall_with_error_handle(header + data)
|
533 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
534 |
+
sent = self._sock.send(data)
|
535 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
536 |
+
2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
537 |
+
Traceback (most recent call last):
|
538 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
539 |
+
cb(written_data)
|
540 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
541 |
+
lambda data: self._console_raw_callback("stderr", data),
|
542 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
543 |
+
return func(self, *args, **kwargs)
|
544 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
545 |
+
return func(self, *args, **kwargs)
|
546 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
547 |
+
self._backend.interface.publish_output_raw(name, data)
|
548 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
549 |
+
self._publish_output_raw(o)
|
550 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
551 |
+
self._publish(rec)
|
552 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
553 |
+
self._sock_client.send_record_publish(record)
|
554 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
555 |
+
self.send_server_request(server_req)
|
556 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
557 |
+
self._send_message(msg)
|
558 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
559 |
+
self._sendall_with_error_handle(header + data)
|
560 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
561 |
+
sent = self._sock.send(data)
|
562 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
563 |
+
2025-06-14 21:00:51,957 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
564 |
+
Traceback (most recent call last):
|
565 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
566 |
+
cb(written_data)
|
567 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
568 |
+
lambda data: self._console_raw_callback("stderr", data),
|
569 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
570 |
+
return func(self, *args, **kwargs)
|
571 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
572 |
+
return func(self, *args, **kwargs)
|
573 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
574 |
+
self._backend.interface.publish_output_raw(name, data)
|
575 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
576 |
+
self._publish_output_raw(o)
|
577 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
578 |
+
self._publish(rec)
|
579 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
580 |
+
self._sock_client.send_record_publish(record)
|
581 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
582 |
+
self.send_server_request(server_req)
|
583 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
584 |
+
self._send_message(msg)
|
585 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
586 |
+
self._sendall_with_error_handle(header + data)
|
587 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
588 |
+
sent = self._sock.send(data)
|
589 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
590 |
+
2025-06-14 21:00:51,958 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
591 |
+
Traceback (most recent call last):
|
592 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
593 |
+
cb(written_data)
|
594 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
595 |
+
lambda data: self._console_raw_callback("stderr", data),
|
596 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
597 |
+
return func(self, *args, **kwargs)
|
598 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
599 |
+
return func(self, *args, **kwargs)
|
600 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
601 |
+
self._backend.interface.publish_output_raw(name, data)
|
602 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
603 |
+
self._publish_output_raw(o)
|
604 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
605 |
+
self._publish(rec)
|
606 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
607 |
+
self._sock_client.send_record_publish(record)
|
608 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
609 |
+
self.send_server_request(server_req)
|
610 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
611 |
+
self._send_message(msg)
|
612 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
613 |
+
self._sendall_with_error_handle(header + data)
|
614 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
615 |
+
sent = self._sock.send(data)
|
616 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
617 |
+
2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
618 |
+
Traceback (most recent call last):
|
619 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
620 |
+
cb(written_data)
|
621 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
622 |
+
lambda data: self._console_raw_callback("stderr", data),
|
623 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
624 |
+
return func(self, *args, **kwargs)
|
625 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
626 |
+
return func(self, *args, **kwargs)
|
627 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
628 |
+
self._backend.interface.publish_output_raw(name, data)
|
629 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
630 |
+
self._publish_output_raw(o)
|
631 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
632 |
+
self._publish(rec)
|
633 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
634 |
+
self._sock_client.send_record_publish(record)
|
635 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
636 |
+
self.send_server_request(server_req)
|
637 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
638 |
+
self._send_message(msg)
|
639 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
640 |
+
self._sendall_with_error_handle(header + data)
|
641 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
642 |
+
sent = self._sock.send(data)
|
643 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
644 |
+
2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
645 |
+
Traceback (most recent call last):
|
646 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
647 |
+
cb(written_data)
|
648 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
649 |
+
lambda data: self._console_raw_callback("stderr", data),
|
650 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
651 |
+
return func(self, *args, **kwargs)
|
652 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
653 |
+
return func(self, *args, **kwargs)
|
654 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
655 |
+
self._backend.interface.publish_output_raw(name, data)
|
656 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
657 |
+
self._publish_output_raw(o)
|
658 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
659 |
+
self._publish(rec)
|
660 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
661 |
+
self._sock_client.send_record_publish(record)
|
662 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
663 |
+
self.send_server_request(server_req)
|
664 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
665 |
+
self._send_message(msg)
|
666 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
667 |
+
self._sendall_with_error_handle(header + data)
|
668 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
669 |
+
sent = self._sock.send(data)
|
670 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
671 |
+
2025-06-14 21:00:51,959 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
672 |
+
Traceback (most recent call last):
|
673 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
674 |
+
cb(written_data)
|
675 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
676 |
+
lambda data: self._console_raw_callback("stderr", data),
|
677 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
678 |
+
return func(self, *args, **kwargs)
|
679 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
680 |
+
return func(self, *args, **kwargs)
|
681 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
682 |
+
self._backend.interface.publish_output_raw(name, data)
|
683 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
684 |
+
self._publish_output_raw(o)
|
685 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
686 |
+
self._publish(rec)
|
687 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
688 |
+
self._sock_client.send_record_publish(record)
|
689 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
690 |
+
self.send_server_request(server_req)
|
691 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
692 |
+
self._send_message(msg)
|
693 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
694 |
+
self._sendall_with_error_handle(header + data)
|
695 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
696 |
+
sent = self._sock.send(data)
|
697 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
698 |
+
2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
699 |
+
Traceback (most recent call last):
|
700 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
701 |
+
cb(written_data)
|
702 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
703 |
+
lambda data: self._console_raw_callback("stderr", data),
|
704 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
705 |
+
return func(self, *args, **kwargs)
|
706 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
707 |
+
return func(self, *args, **kwargs)
|
708 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
709 |
+
self._backend.interface.publish_output_raw(name, data)
|
710 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
711 |
+
self._publish_output_raw(o)
|
712 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
713 |
+
self._publish(rec)
|
714 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
715 |
+
self._sock_client.send_record_publish(record)
|
716 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
717 |
+
self.send_server_request(server_req)
|
718 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
719 |
+
self._send_message(msg)
|
720 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
721 |
+
self._sendall_with_error_handle(header + data)
|
722 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
723 |
+
sent = self._sock.send(data)
|
724 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
725 |
+
2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
726 |
+
Traceback (most recent call last):
|
727 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
728 |
+
cb(written_data)
|
729 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
730 |
+
lambda data: self._console_raw_callback("stderr", data),
|
731 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
732 |
+
return func(self, *args, **kwargs)
|
733 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
734 |
+
return func(self, *args, **kwargs)
|
735 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
736 |
+
self._backend.interface.publish_output_raw(name, data)
|
737 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
738 |
+
self._publish_output_raw(o)
|
739 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
740 |
+
self._publish(rec)
|
741 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
742 |
+
self._sock_client.send_record_publish(record)
|
743 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
744 |
+
self.send_server_request(server_req)
|
745 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
746 |
+
self._send_message(msg)
|
747 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
748 |
+
self._sendall_with_error_handle(header + data)
|
749 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
750 |
+
sent = self._sock.send(data)
|
751 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
752 |
+
2025-06-14 21:00:51,960 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
753 |
+
Traceback (most recent call last):
|
754 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
755 |
+
cb(written_data)
|
756 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
757 |
+
lambda data: self._console_raw_callback("stderr", data),
|
758 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
759 |
+
return func(self, *args, **kwargs)
|
760 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
761 |
+
return func(self, *args, **kwargs)
|
762 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
763 |
+
self._backend.interface.publish_output_raw(name, data)
|
764 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
765 |
+
self._publish_output_raw(o)
|
766 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
767 |
+
self._publish(rec)
|
768 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
769 |
+
self._sock_client.send_record_publish(record)
|
770 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
771 |
+
self.send_server_request(server_req)
|
772 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
773 |
+
self._send_message(msg)
|
774 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
775 |
+
self._sendall_with_error_handle(header + data)
|
776 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
777 |
+
sent = self._sock.send(data)
|
778 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
779 |
+
2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
780 |
+
Traceback (most recent call last):
|
781 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
782 |
+
cb(written_data)
|
783 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
784 |
+
lambda data: self._console_raw_callback("stderr", data),
|
785 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
786 |
+
return func(self, *args, **kwargs)
|
787 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
788 |
+
return func(self, *args, **kwargs)
|
789 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
790 |
+
self._backend.interface.publish_output_raw(name, data)
|
791 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
792 |
+
self._publish_output_raw(o)
|
793 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
794 |
+
self._publish(rec)
|
795 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
796 |
+
self._sock_client.send_record_publish(record)
|
797 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
798 |
+
self.send_server_request(server_req)
|
799 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
800 |
+
self._send_message(msg)
|
801 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
802 |
+
self._sendall_with_error_handle(header + data)
|
803 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
804 |
+
sent = self._sock.send(data)
|
805 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
806 |
+
2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
807 |
+
Traceback (most recent call last):
|
808 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
809 |
+
cb(written_data)
|
810 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
811 |
+
lambda data: self._console_raw_callback("stderr", data),
|
812 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
813 |
+
return func(self, *args, **kwargs)
|
814 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
815 |
+
return func(self, *args, **kwargs)
|
816 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
817 |
+
self._backend.interface.publish_output_raw(name, data)
|
818 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
819 |
+
self._publish_output_raw(o)
|
820 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
821 |
+
self._publish(rec)
|
822 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
823 |
+
self._sock_client.send_record_publish(record)
|
824 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
825 |
+
self.send_server_request(server_req)
|
826 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
827 |
+
self._send_message(msg)
|
828 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
829 |
+
self._sendall_with_error_handle(header + data)
|
830 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
831 |
+
sent = self._sock.send(data)
|
832 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
833 |
+
2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
834 |
+
Traceback (most recent call last):
|
835 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
836 |
+
cb(written_data)
|
837 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
838 |
+
lambda data: self._console_raw_callback("stderr", data),
|
839 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
840 |
+
return func(self, *args, **kwargs)
|
841 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
842 |
+
return func(self, *args, **kwargs)
|
843 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
844 |
+
self._backend.interface.publish_output_raw(name, data)
|
845 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
846 |
+
self._publish_output_raw(o)
|
847 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
848 |
+
self._publish(rec)
|
849 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
850 |
+
self._sock_client.send_record_publish(record)
|
851 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
852 |
+
self.send_server_request(server_req)
|
853 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
854 |
+
self._send_message(msg)
|
855 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
856 |
+
self._sendall_with_error_handle(header + data)
|
857 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
858 |
+
sent = self._sock.send(data)
|
859 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
860 |
+
2025-06-14 21:00:51,961 ERROR MainThread:11388 [redirect.py:_on_write():664] [no run ID] error in stderr callback
|
861 |
+
Traceback (most recent call last):
|
862 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
|
863 |
+
cb(written_data)
|
864 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 2380, in <lambda>
|
865 |
+
lambda data: self._console_raw_callback("stderr", data),
|
866 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper
|
867 |
+
return func(self, *args, **kwargs)
|
868 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 449, in wrapper_fn
|
869 |
+
return func(self, *args, **kwargs)
|
870 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/wandb_run.py", line 1561, in _console_raw_callback
|
871 |
+
self._backend.interface.publish_output_raw(name, data)
|
872 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface.py", line 772, in publish_output_raw
|
873 |
+
self._publish_output_raw(o)
|
874 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
|
875 |
+
self._publish(rec)
|
876 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
|
877 |
+
self._sock_client.send_record_publish(record)
|
878 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
|
879 |
+
self.send_server_request(server_req)
|
880 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
|
881 |
+
self._send_message(msg)
|
882 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
|
883 |
+
self._sendall_with_error_handle(header + data)
|
884 |
+
File "/home/hshi-j-4090/miniconda3/envs/lerobot/lib/python3.10/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
|
885 |
+
sent = self._sock.send(data)
|
886 |
+
BrokenPipeError: [Errno 32] Broken pipe
|
train/act_dice_art_run1/wandb/run-20250614_200951-fhdgxxjz/run-fhdgxxjz.wandb
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:72581254d769e402eaf31adb31ea26d538f83dc36f07ad52a2b7d640ed4eec45
|
3 |
+
size 425984
|
train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/config.yaml
ADDED
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
_wandb:
|
2 |
+
value:
|
3 |
+
cli_version: 0.20.1
|
4 |
+
m: []
|
5 |
+
python_version: 3.10.13
|
6 |
+
t:
|
7 |
+
"1":
|
8 |
+
- 1
|
9 |
+
- 41
|
10 |
+
- 49
|
11 |
+
- 51
|
12 |
+
- 105
|
13 |
+
"2":
|
14 |
+
- 1
|
15 |
+
- 41
|
16 |
+
- 49
|
17 |
+
- 51
|
18 |
+
- 105
|
19 |
+
"3":
|
20 |
+
- 5
|
21 |
+
- 13
|
22 |
+
- 14
|
23 |
+
- 15
|
24 |
+
- 16
|
25 |
+
- 55
|
26 |
+
- 61
|
27 |
+
- 62
|
28 |
+
"4": 3.10.13
|
29 |
+
"5": 0.20.1
|
30 |
+
"10":
|
31 |
+
- 21
|
32 |
+
"12": 0.20.1
|
33 |
+
"13": linux-x86_64
|
34 |
+
batch_size:
|
35 |
+
value: 8
|
36 |
+
dataset:
|
37 |
+
value:
|
38 |
+
episodes: null
|
39 |
+
image_transforms:
|
40 |
+
enable: false
|
41 |
+
max_num_transforms: 3
|
42 |
+
random_order: false
|
43 |
+
tfs:
|
44 |
+
brightness:
|
45 |
+
kwargs:
|
46 |
+
brightness:
|
47 |
+
- 0.8
|
48 |
+
- 1.2
|
49 |
+
type: ColorJitter
|
50 |
+
weight: 1
|
51 |
+
contrast:
|
52 |
+
kwargs:
|
53 |
+
contrast:
|
54 |
+
- 0.8
|
55 |
+
- 1.2
|
56 |
+
type: ColorJitter
|
57 |
+
weight: 1
|
58 |
+
hue:
|
59 |
+
kwargs:
|
60 |
+
hue:
|
61 |
+
- -0.05
|
62 |
+
- 0.05
|
63 |
+
type: ColorJitter
|
64 |
+
weight: 1
|
65 |
+
saturation:
|
66 |
+
kwargs:
|
67 |
+
saturation:
|
68 |
+
- 0.5
|
69 |
+
- 1.5
|
70 |
+
type: ColorJitter
|
71 |
+
weight: 1
|
72 |
+
sharpness:
|
73 |
+
kwargs:
|
74 |
+
sharpness:
|
75 |
+
- 0.5
|
76 |
+
- 1.5
|
77 |
+
type: SharpnessJitter
|
78 |
+
weight: 1
|
79 |
+
repo_id: BoredMachinist/dice-art-run1
|
80 |
+
revision: null
|
81 |
+
root: null
|
82 |
+
use_imagenet_stats: true
|
83 |
+
video_backend: pyav
|
84 |
+
env:
|
85 |
+
value: null
|
86 |
+
eval:
|
87 |
+
value:
|
88 |
+
batch_size: 50
|
89 |
+
n_episodes: 50
|
90 |
+
use_async_envs: false
|
91 |
+
eval_freq:
|
92 |
+
value: 20000
|
93 |
+
job_name:
|
94 |
+
value: act_dice_art_run1
|
95 |
+
log_freq:
|
96 |
+
value: 200
|
97 |
+
num_workers:
|
98 |
+
value: 4
|
99 |
+
optimizer:
|
100 |
+
value:
|
101 |
+
betas:
|
102 |
+
- 0.9
|
103 |
+
- 0.999
|
104 |
+
eps: 1e-08
|
105 |
+
grad_clip_norm: 10
|
106 |
+
lr: 1e-05
|
107 |
+
type: adamw
|
108 |
+
weight_decay: 0.0001
|
109 |
+
output_dir:
|
110 |
+
value: outputs/train/act_dice_art_run1
|
111 |
+
policy:
|
112 |
+
value:
|
113 |
+
chunk_size: 100
|
114 |
+
device: cuda
|
115 |
+
dim_feedforward: 3200
|
116 |
+
dim_model: 512
|
117 |
+
dropout: 0.1
|
118 |
+
feedforward_activation: relu
|
119 |
+
input_features:
|
120 |
+
observation.images.arm:
|
121 |
+
shape:
|
122 |
+
- 3
|
123 |
+
- 480
|
124 |
+
- 640
|
125 |
+
type: VISUAL
|
126 |
+
observation.images.world:
|
127 |
+
shape:
|
128 |
+
- 3
|
129 |
+
- 480
|
130 |
+
- 640
|
131 |
+
type: VISUAL
|
132 |
+
observation.state:
|
133 |
+
shape:
|
134 |
+
- 6
|
135 |
+
type: STATE
|
136 |
+
kl_weight: 10
|
137 |
+
latent_dim: 32
|
138 |
+
n_action_steps: 100
|
139 |
+
n_decoder_layers: 1
|
140 |
+
n_encoder_layers: 4
|
141 |
+
n_heads: 8
|
142 |
+
n_obs_steps: 1
|
143 |
+
n_vae_encoder_layers: 4
|
144 |
+
normalization_mapping:
|
145 |
+
ACTION: MEAN_STD
|
146 |
+
STATE: MEAN_STD
|
147 |
+
VISUAL: MEAN_STD
|
148 |
+
optimizer_lr: 1e-05
|
149 |
+
optimizer_lr_backbone: 1e-05
|
150 |
+
optimizer_weight_decay: 0.0001
|
151 |
+
output_features:
|
152 |
+
action:
|
153 |
+
shape:
|
154 |
+
- 6
|
155 |
+
type: ACTION
|
156 |
+
pre_norm: false
|
157 |
+
pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1
|
158 |
+
replace_final_stride_with_dilation: 0
|
159 |
+
temporal_ensemble_coeff: null
|
160 |
+
type: act
|
161 |
+
use_amp: false
|
162 |
+
use_vae: true
|
163 |
+
vision_backbone: resnet18
|
164 |
+
resume:
|
165 |
+
value: true
|
166 |
+
save_checkpoint:
|
167 |
+
value: true
|
168 |
+
save_freq:
|
169 |
+
value: 20000
|
170 |
+
scheduler:
|
171 |
+
value: null
|
172 |
+
seed:
|
173 |
+
value: 1000
|
174 |
+
steps:
|
175 |
+
value: 100000
|
176 |
+
use_policy_training_preset:
|
177 |
+
value: true
|
178 |
+
wandb:
|
179 |
+
value:
|
180 |
+
disable_artifact: false
|
181 |
+
enable: true
|
182 |
+
entity: null
|
183 |
+
mode: null
|
184 |
+
notes: null
|
185 |
+
project: lerobot
|
186 |
+
run_id: fhdgxxjz
|
train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/output.log
ADDED
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[1m[34mLogs will be synced with wandb.[0m
|
2 |
+
INFO 2025-06-15 01:25:27 db_utils.py:103 Track this run --> [1m[33mhttps://wandb.ai/ametind-o/lerobot/runs/fhdgxxjz[0m
|
3 |
+
INFO 2025-06-15 01:25:27 ts/train.py:127 Creating dataset
|
4 |
+
INFO 2025-06-15 01:25:28 ts/train.py:138 Creating policy
|
5 |
+
Loading weights from local directory
|
6 |
+
INFO 2025-06-15 01:25:28 ts/train.py:144 Creating optimizer and scheduler
|
7 |
+
INFO 2025-06-15 01:25:29 ts/train.py:156 [1m[33mOutput dir:[0m outputs/train/act_dice_art_run1
|
8 |
+
INFO 2025-06-15 01:25:29 ts/train.py:159 cfg.steps=100000 (100K)
|
9 |
+
INFO 2025-06-15 01:25:29 ts/train.py:160 dataset.num_frames=2244 (2K)
|
10 |
+
INFO 2025-06-15 01:25:29 ts/train.py:161 dataset.num_episodes=9
|
11 |
+
INFO 2025-06-15 01:25:29 ts/train.py:162 num_learnable_params=51597190 (52M)
|
12 |
+
INFO 2025-06-15 01:25:29 ts/train.py:163 num_total_params=51597238 (52M)
|
13 |
+
INFO 2025-06-15 01:25:29 ts/train.py:202 Start offline training on a fixed dataset
|
14 |
+
INFO 2025-06-15 01:25:46 ts/train.py:232 step:40K smpl:322K ep:1K epch:143.32 loss:0.062 grdn:8.178 lr:1.0e-05 updt_s:0.084 data_s:0.002
|
15 |
+
INFO 2025-06-15 01:26:02 ts/train.py:232 step:40K smpl:323K ep:1K epch:144.03 loss:0.062 grdn:8.049 lr:1.0e-05 updt_s:0.077 data_s:0.002
|
16 |
+
INFO 2025-06-15 01:26:17 ts/train.py:232 step:41K smpl:325K ep:1K epch:144.74 loss:0.061 grdn:7.868 lr:1.0e-05 updt_s:0.075 data_s:0.001
|
17 |
+
INFO 2025-06-15 01:26:32 ts/train.py:232 step:41K smpl:326K ep:1K epch:145.45 loss:0.060 grdn:8.094 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
18 |
+
INFO 2025-06-15 01:26:46 ts/train.py:232 step:41K smpl:328K ep:1K epch:146.17 loss:0.063 grdn:8.501 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
19 |
+
INFO 2025-06-15 01:27:02 ts/train.py:232 step:41K smpl:330K ep:1K epch:146.88 loss:0.060 grdn:7.931 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
20 |
+
INFO 2025-06-15 01:27:16 ts/train.py:232 step:41K smpl:331K ep:1K epch:147.59 loss:0.060 grdn:8.136 lr:1.0e-05 updt_s:0.073 data_s:0.000
|
21 |
+
INFO 2025-06-15 01:27:31 ts/train.py:232 step:42K smpl:333K ep:1K epch:148.31 loss:0.060 grdn:8.360 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
22 |
+
INFO 2025-06-15 01:27:45 ts/train.py:232 step:42K smpl:334K ep:1K epch:149.02 loss:0.062 grdn:8.511 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
23 |
+
INFO 2025-06-15 01:28:01 ts/train.py:232 step:42K smpl:336K ep:1K epch:149.73 loss:0.060 grdn:8.141 lr:1.0e-05 updt_s:0.074 data_s:0.002
|
24 |
+
INFO 2025-06-15 01:28:15 ts/train.py:232 step:42K smpl:338K ep:1K epch:150.45 loss:0.059 grdn:7.811 lr:1.0e-05 updt_s:0.072 data_s:0.000
|
25 |
+
INFO 2025-06-15 01:28:30 ts/train.py:232 step:42K smpl:339K ep:1K epch:151.16 loss:0.059 grdn:7.760 lr:1.0e-05 updt_s:0.073 data_s:0.002
|
26 |
+
INFO 2025-06-15 01:28:45 ts/train.py:232 step:43K smpl:341K ep:1K epch:151.87 loss:0.061 grdn:8.595 lr:1.0e-05 updt_s:0.072 data_s:0.002
|
27 |
+
INFO 2025-06-15 01:28:59 ts/train.py:232 step:43K smpl:342K ep:1K epch:152.58 loss:0.059 grdn:8.167 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
28 |
+
INFO 2025-06-15 01:29:13 ts/train.py:232 step:43K smpl:344K ep:1K epch:153.30 loss:0.060 grdn:8.204 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
29 |
+
INFO 2025-06-15 01:29:28 ts/train.py:232 step:43K smpl:346K ep:1K epch:154.01 loss:0.058 grdn:7.866 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
30 |
+
INFO 2025-06-15 01:29:42 ts/train.py:232 step:43K smpl:347K ep:1K epch:154.72 loss:0.059 grdn:8.584 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
31 |
+
INFO 2025-06-15 01:29:56 ts/train.py:232 step:44K smpl:349K ep:1K epch:155.44 loss:0.059 grdn:7.636 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
32 |
+
INFO 2025-06-15 01:30:10 ts/train.py:232 step:44K smpl:350K ep:1K epch:156.15 loss:0.059 grdn:8.275 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
33 |
+
INFO 2025-06-15 01:30:25 ts/train.py:232 step:44K smpl:352K ep:1K epch:156.86 loss:0.058 grdn:8.258 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
34 |
+
INFO 2025-06-15 01:30:39 ts/train.py:232 step:44K smpl:354K ep:1K epch:157.58 loss:0.059 grdn:8.812 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
35 |
+
INFO 2025-06-15 01:30:53 ts/train.py:232 step:44K smpl:355K ep:1K epch:158.29 loss:0.059 grdn:8.157 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
36 |
+
INFO 2025-06-15 01:31:07 ts/train.py:232 step:45K smpl:357K ep:1K epch:159.00 loss:0.059 grdn:7.981 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
37 |
+
INFO 2025-06-15 01:31:22 ts/train.py:232 step:45K smpl:358K ep:1K epch:159.71 loss:0.058 grdn:8.005 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
38 |
+
INFO 2025-06-15 01:31:36 ts/train.py:232 step:45K smpl:360K ep:1K epch:160.43 loss:0.057 grdn:8.034 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
39 |
+
INFO 2025-06-15 01:31:50 ts/train.py:232 step:45K smpl:362K ep:1K epch:161.14 loss:0.057 grdn:7.830 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
40 |
+
INFO 2025-06-15 01:32:04 ts/train.py:232 step:45K smpl:363K ep:1K epch:161.85 loss:0.057 grdn:8.148 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
41 |
+
INFO 2025-06-15 01:32:18 ts/train.py:232 step:46K smpl:365K ep:1K epch:162.57 loss:0.057 grdn:7.569 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
42 |
+
INFO 2025-06-15 01:32:33 ts/train.py:232 step:46K smpl:366K ep:1K epch:163.28 loss:0.056 grdn:7.397 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
43 |
+
INFO 2025-06-15 01:32:47 ts/train.py:232 step:46K smpl:368K ep:1K epch:163.99 loss:0.058 grdn:8.293 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
44 |
+
INFO 2025-06-15 01:33:01 ts/train.py:232 step:46K smpl:370K ep:1K epch:164.71 loss:0.056 grdn:7.867 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
45 |
+
INFO 2025-06-15 01:33:15 ts/train.py:232 step:46K smpl:371K ep:1K epch:165.42 loss:0.056 grdn:7.686 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
46 |
+
INFO 2025-06-15 01:33:29 ts/train.py:232 step:47K smpl:373K ep:1K epch:166.13 loss:0.056 grdn:7.390 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
47 |
+
INFO 2025-06-15 01:33:44 ts/train.py:232 step:47K smpl:374K ep:2K epch:166.84 loss:0.057 grdn:7.584 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
48 |
+
INFO 2025-06-15 01:33:58 ts/train.py:232 step:47K smpl:376K ep:2K epch:167.56 loss:0.056 grdn:7.881 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
49 |
+
INFO 2025-06-15 01:34:12 ts/train.py:232 step:47K smpl:378K ep:2K epch:168.27 loss:0.055 grdn:7.482 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
50 |
+
INFO 2025-06-15 01:34:27 ts/train.py:232 step:47K smpl:379K ep:2K epch:168.98 loss:0.056 grdn:7.879 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
51 |
+
INFO 2025-06-15 01:34:41 ts/train.py:232 step:48K smpl:381K ep:2K epch:169.70 loss:0.056 grdn:8.160 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
52 |
+
INFO 2025-06-15 01:34:55 ts/train.py:232 step:48K smpl:382K ep:2K epch:170.41 loss:0.056 grdn:8.133 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
53 |
+
INFO 2025-06-15 01:35:09 ts/train.py:232 step:48K smpl:384K ep:2K epch:171.12 loss:0.055 grdn:7.320 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
54 |
+
INFO 2025-06-15 01:35:23 ts/train.py:232 step:48K smpl:386K ep:2K epch:171.84 loss:0.054 grdn:7.045 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
55 |
+
INFO 2025-06-15 01:35:37 ts/train.py:232 step:48K smpl:387K ep:2K epch:172.55 loss:0.055 grdn:7.458 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
56 |
+
INFO 2025-06-15 01:35:52 ts/train.py:232 step:49K smpl:389K ep:2K epch:173.26 loss:0.056 grdn:8.055 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
57 |
+
INFO 2025-06-15 01:36:06 ts/train.py:232 step:49K smpl:390K ep:2K epch:173.98 loss:0.054 grdn:7.239 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
58 |
+
INFO 2025-06-15 01:36:20 ts/train.py:232 step:49K smpl:392K ep:2K epch:174.69 loss:0.055 grdn:7.588 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
59 |
+
INFO 2025-06-15 01:36:34 ts/train.py:232 step:49K smpl:394K ep:2K epch:175.40 loss:0.054 grdn:7.054 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
60 |
+
INFO 2025-06-15 01:36:48 ts/train.py:232 step:49K smpl:395K ep:2K epch:176.11 loss:0.054 grdn:7.456 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
61 |
+
INFO 2025-06-15 01:37:03 ts/train.py:232 step:50K smpl:397K ep:2K epch:176.83 loss:0.053 grdn:7.103 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
62 |
+
INFO 2025-06-15 01:37:17 ts/train.py:232 step:50K smpl:398K ep:2K epch:177.54 loss:0.055 grdn:7.994 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
63 |
+
INFO 2025-06-15 01:37:31 ts/train.py:232 step:50K smpl:400K ep:2K epch:178.25 loss:0.054 grdn:7.748 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
64 |
+
INFO 2025-06-15 01:37:45 ts/train.py:232 step:50K smpl:402K ep:2K epch:178.97 loss:0.054 grdn:7.213 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
65 |
+
INFO 2025-06-15 01:37:59 ts/train.py:232 step:50K smpl:403K ep:2K epch:179.68 loss:0.052 grdn:7.467 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
66 |
+
INFO 2025-06-15 01:38:13 ts/train.py:232 step:51K smpl:405K ep:2K epch:180.39 loss:0.054 grdn:7.982 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
67 |
+
INFO 2025-06-15 01:38:28 ts/train.py:232 step:51K smpl:406K ep:2K epch:181.11 loss:0.053 grdn:7.853 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
68 |
+
INFO 2025-06-15 01:38:42 ts/train.py:232 step:51K smpl:408K ep:2K epch:181.82 loss:0.053 grdn:7.114 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
69 |
+
INFO 2025-06-15 01:38:56 ts/train.py:232 step:51K smpl:410K ep:2K epch:182.53 loss:0.054 grdn:7.753 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
70 |
+
INFO 2025-06-15 01:39:10 ts/train.py:232 step:51K smpl:411K ep:2K epch:183.24 loss:0.054 grdn:7.507 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
71 |
+
INFO 2025-06-15 01:39:24 ts/train.py:232 step:52K smpl:413K ep:2K epch:183.96 loss:0.053 grdn:7.477 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
72 |
+
INFO 2025-06-15 01:39:38 ts/train.py:232 step:52K smpl:414K ep:2K epch:184.67 loss:0.053 grdn:7.647 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
73 |
+
INFO 2025-06-15 01:39:53 ts/train.py:232 step:52K smpl:416K ep:2K epch:185.38 loss:0.052 grdn:6.969 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
74 |
+
INFO 2025-06-15 01:40:07 ts/train.py:232 step:52K smpl:418K ep:2K epch:186.10 loss:0.053 grdn:7.590 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
75 |
+
INFO 2025-06-15 01:40:21 ts/train.py:232 step:52K smpl:419K ep:2K epch:186.81 loss:0.053 grdn:7.318 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
76 |
+
INFO 2025-06-15 01:40:35 ts/train.py:232 step:53K smpl:421K ep:2K epch:187.52 loss:0.052 grdn:7.186 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
77 |
+
INFO 2025-06-15 01:40:49 ts/train.py:232 step:53K smpl:422K ep:2K epch:188.24 loss:0.053 grdn:7.403 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
78 |
+
INFO 2025-06-15 01:41:04 ts/train.py:232 step:53K smpl:424K ep:2K epch:188.95 loss:0.051 grdn:7.168 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
79 |
+
INFO 2025-06-15 01:41:18 ts/train.py:232 step:53K smpl:426K ep:2K epch:189.66 loss:0.052 grdn:7.206 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
80 |
+
INFO 2025-06-15 01:41:32 ts/train.py:232 step:53K smpl:427K ep:2K epch:190.37 loss:0.052 grdn:7.283 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
81 |
+
INFO 2025-06-15 01:41:46 ts/train.py:232 step:54K smpl:429K ep:2K epch:191.09 loss:0.052 grdn:7.463 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
82 |
+
INFO 2025-06-15 01:42:00 ts/train.py:232 step:54K smpl:430K ep:2K epch:191.80 loss:0.051 grdn:7.277 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
83 |
+
INFO 2025-06-15 01:42:14 ts/train.py:232 step:54K smpl:432K ep:2K epch:192.51 loss:0.052 grdn:7.309 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
84 |
+
INFO 2025-06-15 01:42:29 ts/train.py:232 step:54K smpl:434K ep:2K epch:193.23 loss:0.051 grdn:6.976 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
85 |
+
INFO 2025-06-15 01:42:43 ts/train.py:232 step:54K smpl:435K ep:2K epch:193.94 loss:0.051 grdn:7.092 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
86 |
+
INFO 2025-06-15 01:42:57 ts/train.py:232 step:55K smpl:437K ep:2K epch:194.65 loss:0.050 grdn:6.810 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
87 |
+
INFO 2025-06-15 01:43:11 ts/train.py:232 step:55K smpl:438K ep:2K epch:195.37 loss:0.051 grdn:7.106 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
88 |
+
INFO 2025-06-15 01:43:25 ts/train.py:232 step:55K smpl:440K ep:2K epch:196.08 loss:0.051 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
89 |
+
INFO 2025-06-15 01:43:40 ts/train.py:232 step:55K smpl:442K ep:2K epch:196.79 loss:0.050 grdn:6.835 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
90 |
+
INFO 2025-06-15 01:43:54 ts/train.py:232 step:55K smpl:443K ep:2K epch:197.50 loss:0.050 grdn:6.746 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
91 |
+
INFO 2025-06-15 01:44:08 ts/train.py:232 step:56K smpl:445K ep:2K epch:198.22 loss:0.051 grdn:7.479 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
92 |
+
INFO 2025-06-15 01:44:22 ts/train.py:232 step:56K smpl:446K ep:2K epch:198.93 loss:0.051 grdn:7.710 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
93 |
+
INFO 2025-06-15 01:44:36 ts/train.py:232 step:56K smpl:448K ep:2K epch:199.64 loss:0.049 grdn:6.914 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
94 |
+
INFO 2025-06-15 01:44:50 ts/train.py:232 step:56K smpl:450K ep:2K epch:200.36 loss:0.051 grdn:7.331 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
95 |
+
INFO 2025-06-15 01:45:05 ts/train.py:232 step:56K smpl:451K ep:2K epch:201.07 loss:0.050 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
96 |
+
INFO 2025-06-15 01:45:19 ts/train.py:232 step:57K smpl:453K ep:2K epch:201.78 loss:0.050 grdn:6.781 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
97 |
+
INFO 2025-06-15 01:45:33 ts/train.py:232 step:57K smpl:454K ep:2K epch:202.50 loss:0.049 grdn:6.989 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
98 |
+
INFO 2025-06-15 01:45:47 ts/train.py:232 step:57K smpl:456K ep:2K epch:203.21 loss:0.050 grdn:7.546 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
99 |
+
INFO 2025-06-15 01:46:01 ts/train.py:232 step:57K smpl:458K ep:2K epch:203.92 loss:0.050 grdn:7.045 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
100 |
+
INFO 2025-06-15 01:46:15 ts/train.py:232 step:57K smpl:459K ep:2K epch:204.63 loss:0.050 grdn:7.347 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
101 |
+
INFO 2025-06-15 01:46:30 ts/train.py:232 step:58K smpl:461K ep:2K epch:205.35 loss:0.050 grdn:7.318 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
102 |
+
INFO 2025-06-15 01:46:44 ts/train.py:232 step:58K smpl:462K ep:2K epch:206.06 loss:0.049 grdn:7.083 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
103 |
+
INFO 2025-06-15 01:46:58 ts/train.py:232 step:58K smpl:464K ep:2K epch:206.77 loss:0.049 grdn:6.753 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
104 |
+
INFO 2025-06-15 01:47:12 ts/train.py:232 step:58K smpl:466K ep:2K epch:207.49 loss:0.049 grdn:7.162 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
105 |
+
INFO 2025-06-15 01:47:26 ts/train.py:232 step:58K smpl:467K ep:2K epch:208.20 loss:0.049 grdn:7.053 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
106 |
+
INFO 2025-06-15 01:47:41 ts/train.py:232 step:59K smpl:469K ep:2K epch:208.91 loss:0.049 grdn:6.821 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
107 |
+
INFO 2025-06-15 01:47:55 ts/train.py:232 step:59K smpl:470K ep:2K epch:209.63 loss:0.048 grdn:6.729 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
108 |
+
INFO 2025-06-15 01:48:09 ts/train.py:232 step:59K smpl:472K ep:2K epch:210.34 loss:0.048 grdn:7.147 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
109 |
+
INFO 2025-06-15 01:48:23 ts/train.py:232 step:59K smpl:474K ep:2K epch:211.05 loss:0.050 grdn:6.870 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
110 |
+
INFO 2025-06-15 01:48:38 ts/train.py:232 step:59K smpl:475K ep:2K epch:211.76 loss:0.047 grdn:6.563 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
111 |
+
INFO 2025-06-15 01:48:51 ts/train.py:232 step:60K smpl:477K ep:2K epch:212.48 loss:0.048 grdn:6.476 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
112 |
+
INFO 2025-06-15 01:49:06 ts/train.py:232 step:60K smpl:478K ep:2K epch:213.19 loss:0.047 grdn:6.635 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
113 |
+
INFO 2025-06-15 01:49:20 ts/train.py:232 step:60K smpl:480K ep:2K epch:213.90 loss:0.047 grdn:6.311 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
114 |
+
INFO 2025-06-15 01:49:20 ts/train.py:241 Checkpoint policy after step 60000
|
115 |
+
INFO 2025-06-15 01:49:35 ts/train.py:232 step:60K smpl:482K ep:2K epch:214.62 loss:0.048 grdn:6.895 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
116 |
+
INFO 2025-06-15 01:49:49 ts/train.py:232 step:60K smpl:483K ep:2K epch:215.33 loss:0.048 grdn:6.920 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
117 |
+
INFO 2025-06-15 01:50:04 ts/train.py:232 step:61K smpl:485K ep:2K epch:216.04 loss:0.047 grdn:6.346 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
118 |
+
INFO 2025-06-15 01:50:18 ts/train.py:232 step:61K smpl:486K ep:2K epch:216.76 loss:0.048 grdn:6.918 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
119 |
+
INFO 2025-06-15 01:50:32 ts/train.py:232 step:61K smpl:488K ep:2K epch:217.47 loss:0.048 grdn:6.664 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
120 |
+
INFO 2025-06-15 01:50:46 ts/train.py:232 step:61K smpl:490K ep:2K epch:218.18 loss:0.046 grdn:6.951 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
121 |
+
INFO 2025-06-15 01:51:01 ts/train.py:232 step:61K smpl:491K ep:2K epch:218.89 loss:0.048 grdn:7.243 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
122 |
+
INFO 2025-06-15 01:51:15 ts/train.py:232 step:62K smpl:493K ep:2K epch:219.61 loss:0.048 grdn:6.737 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
123 |
+
INFO 2025-06-15 01:51:29 ts/train.py:232 step:62K smpl:494K ep:2K epch:220.32 loss:0.046 grdn:6.675 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
124 |
+
INFO 2025-06-15 01:51:43 ts/train.py:232 step:62K smpl:496K ep:2K epch:221.03 loss:0.047 grdn:6.878 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
125 |
+
INFO 2025-06-15 01:51:57 ts/train.py:232 step:62K smpl:498K ep:2K epch:221.75 loss:0.047 grdn:6.595 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
126 |
+
INFO 2025-06-15 01:52:11 ts/train.py:232 step:62K smpl:499K ep:2K epch:222.46 loss:0.048 grdn:6.651 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
127 |
+
INFO 2025-06-15 01:52:26 ts/train.py:232 step:63K smpl:501K ep:2K epch:223.17 loss:0.048 grdn:7.065 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
128 |
+
INFO 2025-06-15 01:52:40 ts/train.py:232 step:63K smpl:502K ep:2K epch:223.89 loss:0.047 grdn:6.398 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
129 |
+
INFO 2025-06-15 01:52:54 ts/train.py:232 step:63K smpl:504K ep:2K epch:224.60 loss:0.046 grdn:6.428 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
130 |
+
INFO 2025-06-15 01:53:08 ts/train.py:232 step:63K smpl:506K ep:2K epch:225.31 loss:0.048 grdn:6.853 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
131 |
+
INFO 2025-06-15 01:53:22 ts/train.py:232 step:63K smpl:507K ep:2K epch:226.02 loss:0.045 grdn:6.491 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
132 |
+
INFO 2025-06-15 01:53:36 ts/train.py:232 step:64K smpl:509K ep:2K epch:226.74 loss:0.046 grdn:6.330 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
133 |
+
INFO 2025-06-15 01:53:51 ts/train.py:232 step:64K smpl:510K ep:2K epch:227.45 loss:0.047 grdn:7.094 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
134 |
+
INFO 2025-06-15 01:54:05 ts/train.py:232 step:64K smpl:512K ep:2K epch:228.16 loss:0.046 grdn:6.896 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
135 |
+
INFO 2025-06-15 01:54:19 ts/train.py:232 step:64K smpl:514K ep:2K epch:228.88 loss:0.046 grdn:6.510 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
136 |
+
INFO 2025-06-15 01:54:33 ts/train.py:232 step:64K smpl:515K ep:2K epch:229.59 loss:0.047 grdn:6.431 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
137 |
+
INFO 2025-06-15 01:54:47 ts/train.py:232 step:65K smpl:517K ep:2K epch:230.30 loss:0.046 grdn:6.902 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
138 |
+
INFO 2025-06-15 01:55:02 ts/train.py:232 step:65K smpl:518K ep:2K epch:231.02 loss:0.046 grdn:7.001 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
139 |
+
INFO 2025-06-15 01:55:16 ts/train.py:232 step:65K smpl:520K ep:2K epch:231.73 loss:0.045 grdn:6.760 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
140 |
+
INFO 2025-06-15 01:55:30 ts/train.py:232 step:65K smpl:522K ep:2K epch:232.44 loss:0.046 grdn:6.784 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
141 |
+
INFO 2025-06-15 01:55:44 ts/train.py:232 step:65K smpl:523K ep:2K epch:233.16 loss:0.046 grdn:6.602 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
142 |
+
INFO 2025-06-15 01:55:59 ts/train.py:232 step:66K smpl:525K ep:2K epch:233.87 loss:0.045 grdn:6.246 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
143 |
+
INFO 2025-06-15 01:56:13 ts/train.py:232 step:66K smpl:526K ep:2K epch:234.58 loss:0.045 grdn:6.372 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
144 |
+
INFO 2025-06-15 01:56:27 ts/train.py:232 step:66K smpl:528K ep:2K epch:235.29 loss:0.045 grdn:6.747 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
145 |
+
INFO 2025-06-15 01:56:41 ts/train.py:232 step:66K smpl:530K ep:2K epch:236.01 loss:0.046 grdn:6.872 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
146 |
+
INFO 2025-06-15 01:56:55 ts/train.py:232 step:66K smpl:531K ep:2K epch:236.72 loss:0.045 grdn:6.459 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
147 |
+
INFO 2025-06-15 01:57:09 ts/train.py:232 step:67K smpl:533K ep:2K epch:237.43 loss:0.044 grdn:6.369 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
148 |
+
INFO 2025-06-15 01:57:24 ts/train.py:232 step:67K smpl:534K ep:2K epch:238.15 loss:0.046 grdn:7.069 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
149 |
+
INFO 2025-06-15 01:57:38 ts/train.py:232 step:67K smpl:536K ep:2K epch:238.86 loss:0.046 grdn:6.989 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
150 |
+
INFO 2025-06-15 01:57:52 ts/train.py:232 step:67K smpl:538K ep:2K epch:239.57 loss:0.044 grdn:6.302 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
151 |
+
INFO 2025-06-15 01:58:06 ts/train.py:232 step:67K smpl:539K ep:2K epch:240.29 loss:0.044 grdn:6.276 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
152 |
+
INFO 2025-06-15 01:58:20 ts/train.py:232 step:68K smpl:541K ep:2K epch:241.00 loss:0.044 grdn:6.225 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
153 |
+
INFO 2025-06-15 01:58:34 ts/train.py:232 step:68K smpl:542K ep:2K epch:241.71 loss:0.044 grdn:6.253 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
154 |
+
INFO 2025-06-15 01:58:49 ts/train.py:232 step:68K smpl:544K ep:2K epch:242.42 loss:0.044 grdn:6.556 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
155 |
+
INFO 2025-06-15 01:59:03 ts/train.py:232 step:68K smpl:546K ep:2K epch:243.14 loss:0.045 grdn:6.488 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
156 |
+
INFO 2025-06-15 01:59:17 ts/train.py:232 step:68K smpl:547K ep:2K epch:243.85 loss:0.044 grdn:6.332 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
157 |
+
INFO 2025-06-15 01:59:31 ts/train.py:232 step:69K smpl:549K ep:2K epch:244.56 loss:0.044 grdn:6.291 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
158 |
+
INFO 2025-06-15 01:59:45 ts/train.py:232 step:69K smpl:550K ep:2K epch:245.28 loss:0.044 grdn:6.523 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
159 |
+
INFO 2025-06-15 02:00:00 ts/train.py:232 step:69K smpl:552K ep:2K epch:245.99 loss:0.044 grdn:6.346 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
160 |
+
INFO 2025-06-15 02:00:14 ts/train.py:232 step:69K smpl:554K ep:2K epch:246.70 loss:0.045 grdn:6.513 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
161 |
+
INFO 2025-06-15 02:00:28 ts/train.py:232 step:69K smpl:555K ep:2K epch:247.42 loss:0.043 grdn:6.302 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
162 |
+
INFO 2025-06-15 02:00:42 ts/train.py:232 step:70K smpl:557K ep:2K epch:248.13 loss:0.044 grdn:6.421 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
163 |
+
INFO 2025-06-15 02:00:56 ts/train.py:232 step:70K smpl:558K ep:2K epch:248.84 loss:0.044 grdn:6.630 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
164 |
+
INFO 2025-06-15 02:01:10 ts/train.py:232 step:70K smpl:560K ep:2K epch:249.55 loss:0.043 grdn:6.248 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
165 |
+
INFO 2025-06-15 02:01:25 ts/train.py:232 step:70K smpl:562K ep:2K epch:250.27 loss:0.043 grdn:6.276 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
166 |
+
INFO 2025-06-15 02:01:39 ts/train.py:232 step:70K smpl:563K ep:2K epch:250.98 loss:0.043 grdn:6.101 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
167 |
+
INFO 2025-06-15 02:01:53 ts/train.py:232 step:71K smpl:565K ep:2K epch:251.69 loss:0.043 grdn:6.456 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
168 |
+
INFO 2025-06-15 02:02:07 ts/train.py:232 step:71K smpl:566K ep:2K epch:252.41 loss:0.042 grdn:6.254 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
169 |
+
INFO 2025-06-15 02:02:22 ts/train.py:232 step:71K smpl:568K ep:2K epch:253.12 loss:0.044 grdn:6.339 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
170 |
+
INFO 2025-06-15 02:02:36 ts/train.py:232 step:71K smpl:570K ep:2K epch:253.83 loss:0.044 grdn:6.720 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
171 |
+
INFO 2025-06-15 02:02:50 ts/train.py:232 step:71K smpl:571K ep:2K epch:254.55 loss:0.042 grdn:6.028 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
172 |
+
INFO 2025-06-15 02:03:04 ts/train.py:232 step:72K smpl:573K ep:2K epch:255.26 loss:0.044 grdn:6.236 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
173 |
+
INFO 2025-06-15 02:03:18 ts/train.py:232 step:72K smpl:574K ep:2K epch:255.97 loss:0.042 grdn:6.354 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
174 |
+
INFO 2025-06-15 02:03:32 ts/train.py:232 step:72K smpl:576K ep:2K epch:256.68 loss:0.043 grdn:6.008 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
175 |
+
INFO 2025-06-15 02:03:47 ts/train.py:232 step:72K smpl:578K ep:2K epch:257.40 loss:0.043 grdn:6.608 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
176 |
+
INFO 2025-06-15 02:04:01 ts/train.py:232 step:72K smpl:579K ep:2K epch:258.11 loss:0.042 grdn:5.767 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
177 |
+
INFO 2025-06-15 02:04:15 ts/train.py:232 step:73K smpl:581K ep:2K epch:258.82 loss:0.042 grdn:6.058 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
178 |
+
INFO 2025-06-15 02:04:29 ts/train.py:232 step:73K smpl:582K ep:2K epch:259.54 loss:0.042 grdn:5.928 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
179 |
+
INFO 2025-06-15 02:04:43 ts/train.py:232 step:73K smpl:584K ep:2K epch:260.25 loss:0.042 grdn:6.008 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
180 |
+
INFO 2025-06-15 02:04:58 ts/train.py:232 step:73K smpl:586K ep:2K epch:260.96 loss:0.042 grdn:6.359 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
181 |
+
INFO 2025-06-15 02:05:12 ts/train.py:232 step:73K smpl:587K ep:2K epch:261.68 loss:0.042 grdn:6.003 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
182 |
+
INFO 2025-06-15 02:05:26 ts/train.py:232 step:74K smpl:589K ep:2K epch:262.39 loss:0.042 grdn:5.966 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
183 |
+
INFO 2025-06-15 02:05:40 ts/train.py:232 step:74K smpl:590K ep:2K epch:263.10 loss:0.042 grdn:6.431 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
184 |
+
INFO 2025-06-15 02:05:54 ts/train.py:232 step:74K smpl:592K ep:2K epch:263.81 loss:0.043 grdn:6.376 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
185 |
+
INFO 2025-06-15 02:06:08 ts/train.py:232 step:74K smpl:594K ep:2K epch:264.53 loss:0.043 grdn:6.170 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
186 |
+
INFO 2025-06-15 02:06:23 ts/train.py:232 step:74K smpl:595K ep:2K epch:265.24 loss:0.042 grdn:6.038 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
187 |
+
INFO 2025-06-15 02:06:37 ts/train.py:232 step:75K smpl:597K ep:2K epch:265.95 loss:0.041 grdn:5.885 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
188 |
+
INFO 2025-06-15 02:06:51 ts/train.py:232 step:75K smpl:598K ep:2K epch:266.67 loss:0.042 grdn:5.893 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
189 |
+
INFO 2025-06-15 02:07:05 ts/train.py:232 step:75K smpl:600K ep:2K epch:267.38 loss:0.042 grdn:6.183 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
190 |
+
INFO 2025-06-15 02:07:20 ts/train.py:232 step:75K smpl:602K ep:2K epch:268.09 loss:0.042 grdn:6.227 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
191 |
+
INFO 2025-06-15 02:07:34 ts/train.py:232 step:75K smpl:603K ep:2K epch:268.81 loss:0.042 grdn:6.243 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
192 |
+
INFO 2025-06-15 02:07:48 ts/train.py:232 step:76K smpl:605K ep:2K epch:269.52 loss:0.042 grdn:6.316 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
193 |
+
INFO 2025-06-15 02:08:02 ts/train.py:232 step:76K smpl:606K ep:2K epch:270.23 loss:0.041 grdn:6.225 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
194 |
+
INFO 2025-06-15 02:08:17 ts/train.py:232 step:76K smpl:608K ep:2K epch:270.94 loss:0.042 grdn:6.199 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
195 |
+
INFO 2025-06-15 02:08:30 ts/train.py:232 step:76K smpl:610K ep:2K epch:271.66 loss:0.041 grdn:6.032 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
196 |
+
INFO 2025-06-15 02:08:45 ts/train.py:232 step:76K smpl:611K ep:2K epch:272.37 loss:0.041 grdn:5.946 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
197 |
+
INFO 2025-06-15 02:08:59 ts/train.py:232 step:77K smpl:613K ep:2K epch:273.08 loss:0.041 grdn:5.773 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
198 |
+
INFO 2025-06-15 02:09:13 ts/train.py:232 step:77K smpl:614K ep:2K epch:273.80 loss:0.042 grdn:6.100 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
199 |
+
INFO 2025-06-15 02:09:27 ts/train.py:232 step:77K smpl:616K ep:2K epch:274.51 loss:0.041 grdn:6.014 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
200 |
+
INFO 2025-06-15 02:09:42 ts/train.py:232 step:77K smpl:618K ep:2K epch:275.22 loss:0.041 grdn:5.869 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
201 |
+
INFO 2025-06-15 02:09:56 ts/train.py:232 step:77K smpl:619K ep:2K epch:275.94 loss:0.040 grdn:6.026 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
202 |
+
INFO 2025-06-15 02:10:10 ts/train.py:232 step:78K smpl:621K ep:2K epch:276.65 loss:0.041 grdn:6.137 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
203 |
+
INFO 2025-06-15 02:10:24 ts/train.py:232 step:78K smpl:622K ep:2K epch:277.36 loss:0.041 grdn:6.259 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
204 |
+
INFO 2025-06-15 02:10:38 ts/train.py:232 step:78K smpl:624K ep:3K epch:278.07 loss:0.040 grdn:6.067 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
205 |
+
INFO 2025-06-15 02:10:52 ts/train.py:232 step:78K smpl:626K ep:3K epch:278.79 loss:0.040 grdn:5.739 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
206 |
+
INFO 2025-06-15 02:11:07 ts/train.py:232 step:78K smpl:627K ep:3K epch:279.50 loss:0.040 grdn:5.994 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
207 |
+
INFO 2025-06-15 02:11:21 ts/train.py:232 step:79K smpl:629K ep:3K epch:280.21 loss:0.042 grdn:6.285 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
208 |
+
INFO 2025-06-15 02:11:35 ts/train.py:232 step:79K smpl:630K ep:3K epch:280.93 loss:0.041 grdn:6.044 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
209 |
+
INFO 2025-06-15 02:11:49 ts/train.py:232 step:79K smpl:632K ep:3K epch:281.64 loss:0.040 grdn:6.118 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
210 |
+
INFO 2025-06-15 02:12:04 ts/train.py:232 step:79K smpl:634K ep:3K epch:282.35 loss:0.040 grdn:5.683 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
211 |
+
INFO 2025-06-15 02:12:18 ts/train.py:232 step:79K smpl:635K ep:3K epch:283.07 loss:0.039 grdn:5.616 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
212 |
+
INFO 2025-06-15 02:12:32 ts/train.py:232 step:80K smpl:637K ep:3K epch:283.78 loss:0.040 grdn:6.087 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
213 |
+
INFO 2025-06-15 02:12:46 ts/train.py:232 step:80K smpl:638K ep:3K epch:284.49 loss:0.040 grdn:6.091 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
214 |
+
INFO 2025-06-15 02:13:00 ts/train.py:232 step:80K smpl:640K ep:3K epch:285.20 loss:0.040 grdn:6.038 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
215 |
+
INFO 2025-06-15 02:13:00 ts/train.py:241 Checkpoint policy after step 80000
|
216 |
+
INFO 2025-06-15 02:13:16 ts/train.py:232 step:80K smpl:642K ep:3K epch:285.92 loss:0.040 grdn:5.929 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
217 |
+
INFO 2025-06-15 02:13:30 ts/train.py:232 step:80K smpl:643K ep:3K epch:286.63 loss:0.041 grdn:6.205 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
218 |
+
INFO 2025-06-15 02:13:44 ts/train.py:232 step:81K smpl:645K ep:3K epch:287.34 loss:0.039 grdn:5.740 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
219 |
+
INFO 2025-06-15 02:13:58 ts/train.py:232 step:81K smpl:646K ep:3K epch:288.06 loss:0.040 grdn:6.000 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
220 |
+
INFO 2025-06-15 02:14:12 ts/train.py:232 step:81K smpl:648K ep:3K epch:288.77 loss:0.041 grdn:5.993 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
221 |
+
INFO 2025-06-15 02:14:26 ts/train.py:232 step:81K smpl:650K ep:3K epch:289.48 loss:0.040 grdn:6.078 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
222 |
+
INFO 2025-06-15 02:14:41 ts/train.py:232 step:81K smpl:651K ep:3K epch:290.20 loss:0.041 grdn:6.579 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
223 |
+
INFO 2025-06-15 02:14:55 ts/train.py:232 step:82K smpl:653K ep:3K epch:290.91 loss:0.040 grdn:5.781 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
224 |
+
INFO 2025-06-15 02:15:09 ts/train.py:232 step:82K smpl:654K ep:3K epch:291.62 loss:0.040 grdn:6.159 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
225 |
+
INFO 2025-06-15 02:15:23 ts/train.py:232 step:82K smpl:656K ep:3K epch:292.34 loss:0.040 grdn:5.931 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
226 |
+
INFO 2025-06-15 02:15:37 ts/train.py:232 step:82K smpl:658K ep:3K epch:293.05 loss:0.040 grdn:6.215 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
227 |
+
INFO 2025-06-15 02:15:51 ts/train.py:232 step:82K smpl:659K ep:3K epch:293.76 loss:0.039 grdn:5.577 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
228 |
+
INFO 2025-06-15 02:16:06 ts/train.py:232 step:83K smpl:661K ep:3K epch:294.47 loss:0.040 grdn:5.723 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
229 |
+
INFO 2025-06-15 02:16:20 ts/train.py:232 step:83K smpl:662K ep:3K epch:295.19 loss:0.039 grdn:6.670 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
230 |
+
INFO 2025-06-15 02:16:34 ts/train.py:232 step:83K smpl:664K ep:3K epch:295.90 loss:0.040 grdn:5.918 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
231 |
+
INFO 2025-06-15 02:16:48 ts/train.py:232 step:83K smpl:666K ep:3K epch:296.61 loss:0.039 grdn:6.026 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
232 |
+
INFO 2025-06-15 02:17:02 ts/train.py:232 step:83K smpl:667K ep:3K epch:297.33 loss:0.039 grdn:5.951 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
233 |
+
INFO 2025-06-15 02:17:17 ts/train.py:232 step:84K smpl:669K ep:3K epch:298.04 loss:0.039 grdn:5.951 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
234 |
+
INFO 2025-06-15 02:17:31 ts/train.py:232 step:84K smpl:670K ep:3K epch:298.75 loss:0.039 grdn:5.813 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
235 |
+
INFO 2025-06-15 02:17:45 ts/train.py:232 step:84K smpl:672K ep:3K epch:299.47 loss:0.039 grdn:6.088 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
236 |
+
INFO 2025-06-15 02:17:59 ts/train.py:232 step:84K smpl:674K ep:3K epch:300.18 loss:0.040 grdn:5.943 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
237 |
+
INFO 2025-06-15 02:18:13 ts/train.py:232 step:84K smpl:675K ep:3K epch:300.89 loss:0.037 grdn:5.501 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
238 |
+
INFO 2025-06-15 02:18:28 ts/train.py:232 step:85K smpl:677K ep:3K epch:301.60 loss:0.039 grdn:5.522 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
239 |
+
INFO 2025-06-15 02:18:42 ts/train.py:232 step:85K smpl:678K ep:3K epch:302.32 loss:0.038 grdn:5.490 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
240 |
+
INFO 2025-06-15 02:18:56 ts/train.py:232 step:85K smpl:680K ep:3K epch:303.03 loss:0.039 grdn:5.950 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
241 |
+
INFO 2025-06-15 02:19:10 ts/train.py:232 step:85K smpl:682K ep:3K epch:303.74 loss:0.039 grdn:5.688 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
242 |
+
INFO 2025-06-15 02:19:24 ts/train.py:232 step:85K smpl:683K ep:3K epch:304.46 loss:0.038 grdn:5.593 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
243 |
+
INFO 2025-06-15 02:19:39 ts/train.py:232 step:86K smpl:685K ep:3K epch:305.17 loss:0.039 grdn:5.771 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
244 |
+
INFO 2025-06-15 02:19:53 ts/train.py:232 step:86K smpl:686K ep:3K epch:305.88 loss:0.038 grdn:5.819 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
245 |
+
INFO 2025-06-15 02:20:07 ts/train.py:232 step:86K smpl:688K ep:3K epch:306.60 loss:0.040 grdn:6.220 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
246 |
+
INFO 2025-06-15 02:20:21 ts/train.py:232 step:86K smpl:690K ep:3K epch:307.31 loss:0.038 grdn:5.573 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
247 |
+
INFO 2025-06-15 02:20:35 ts/train.py:232 step:86K smpl:691K ep:3K epch:308.02 loss:0.038 grdn:5.470 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
248 |
+
INFO 2025-06-15 02:20:49 ts/train.py:232 step:87K smpl:693K ep:3K epch:308.73 loss:0.038 grdn:5.576 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
249 |
+
INFO 2025-06-15 02:21:04 ts/train.py:232 step:87K smpl:694K ep:3K epch:309.45 loss:0.038 grdn:5.870 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
250 |
+
INFO 2025-06-15 02:21:18 ts/train.py:232 step:87K smpl:696K ep:3K epch:310.16 loss:0.039 grdn:5.699 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
251 |
+
INFO 2025-06-15 02:21:32 ts/train.py:232 step:87K smpl:698K ep:3K epch:310.87 loss:0.038 grdn:5.654 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
252 |
+
INFO 2025-06-15 02:21:46 ts/train.py:232 step:87K smpl:699K ep:3K epch:311.59 loss:0.038 grdn:5.526 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
253 |
+
INFO 2025-06-15 02:22:00 ts/train.py:232 step:88K smpl:701K ep:3K epch:312.30 loss:0.038 grdn:5.874 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
254 |
+
INFO 2025-06-15 02:22:15 ts/train.py:232 step:88K smpl:702K ep:3K epch:313.01 loss:0.038 grdn:5.877 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
255 |
+
INFO 2025-06-15 02:22:29 ts/train.py:232 step:88K smpl:704K ep:3K epch:313.73 loss:0.038 grdn:5.649 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
256 |
+
INFO 2025-06-15 02:22:43 ts/train.py:232 step:88K smpl:706K ep:3K epch:314.44 loss:0.038 grdn:5.432 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
257 |
+
INFO 2025-06-15 02:22:57 ts/train.py:232 step:88K smpl:707K ep:3K epch:315.15 loss:0.038 grdn:5.860 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
258 |
+
INFO 2025-06-15 02:23:11 ts/train.py:232 step:89K smpl:709K ep:3K epch:315.86 loss:0.038 grdn:5.839 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
259 |
+
INFO 2025-06-15 02:23:25 ts/train.py:232 step:89K smpl:710K ep:3K epch:316.58 loss:0.037 grdn:5.675 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
260 |
+
INFO 2025-06-15 02:23:40 ts/train.py:232 step:89K smpl:712K ep:3K epch:317.29 loss:0.038 grdn:5.895 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
261 |
+
INFO 2025-06-15 02:23:54 ts/train.py:232 step:89K smpl:714K ep:3K epch:318.00 loss:0.037 grdn:5.757 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
262 |
+
INFO 2025-06-15 02:24:08 ts/train.py:232 step:89K smpl:715K ep:3K epch:318.72 loss:0.037 grdn:5.328 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
263 |
+
INFO 2025-06-15 02:24:22 ts/train.py:232 step:90K smpl:717K ep:3K epch:319.43 loss:0.037 grdn:5.424 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
264 |
+
INFO 2025-06-15 02:24:36 ts/train.py:232 step:90K smpl:718K ep:3K epch:320.14 loss:0.037 grdn:5.791 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
265 |
+
INFO 2025-06-15 02:24:50 ts/train.py:232 step:90K smpl:720K ep:3K epch:320.86 loss:0.037 grdn:5.890 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
266 |
+
INFO 2025-06-15 02:25:05 ts/train.py:232 step:90K smpl:722K ep:3K epch:321.57 loss:0.037 grdn:5.301 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
267 |
+
INFO 2025-06-15 02:25:19 ts/train.py:232 step:90K smpl:723K ep:3K epch:322.28 loss:0.037 grdn:5.402 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
268 |
+
INFO 2025-06-15 02:25:33 ts/train.py:232 step:91K smpl:725K ep:3K epch:322.99 loss:0.037 grdn:5.723 lr:1.0e-05 updt_s:0.069 data_s:0.001
|
269 |
+
INFO 2025-06-15 02:25:47 ts/train.py:232 step:91K smpl:726K ep:3K epch:323.71 loss:0.037 grdn:5.590 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
270 |
+
INFO 2025-06-15 02:26:01 ts/train.py:232 step:91K smpl:728K ep:3K epch:324.42 loss:0.036 grdn:5.477 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
271 |
+
INFO 2025-06-15 02:26:16 ts/train.py:232 step:91K smpl:730K ep:3K epch:325.13 loss:0.037 grdn:5.779 lr:1.0e-05 updt_s:0.070 data_s:0.001
|
272 |
+
INFO 2025-06-15 02:26:30 ts/train.py:232 step:91K smpl:731K ep:3K epch:325.85 loss:0.037 grdn:5.516 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
273 |
+
INFO 2025-06-15 02:26:44 ts/train.py:232 step:92K smpl:733K ep:3K epch:326.56 loss:0.037 grdn:5.410 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
274 |
+
INFO 2025-06-15 02:26:58 ts/train.py:232 step:92K smpl:734K ep:3K epch:327.27 loss:0.036 grdn:5.618 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
275 |
+
INFO 2025-06-15 02:27:13 ts/train.py:232 step:92K smpl:736K ep:3K epch:327.99 loss:0.037 grdn:5.627 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
276 |
+
INFO 2025-06-15 02:27:27 ts/train.py:232 step:92K smpl:738K ep:3K epch:328.70 loss:0.037 grdn:5.244 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
277 |
+
INFO 2025-06-15 02:27:41 ts/train.py:232 step:92K smpl:739K ep:3K epch:329.41 loss:0.036 grdn:5.326 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
278 |
+
INFO 2025-06-15 02:27:55 ts/train.py:232 step:93K smpl:741K ep:3K epch:330.12 loss:0.036 grdn:5.493 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
279 |
+
INFO 2025-06-15 02:28:09 ts/train.py:232 step:93K smpl:742K ep:3K epch:330.84 loss:0.037 grdn:5.769 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
280 |
+
INFO 2025-06-15 02:28:23 ts/train.py:232 step:93K smpl:744K ep:3K epch:331.55 loss:0.036 grdn:5.544 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
281 |
+
INFO 2025-06-15 02:28:38 ts/train.py:232 step:93K smpl:746K ep:3K epch:332.26 loss:0.036 grdn:5.526 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
282 |
+
INFO 2025-06-15 02:28:52 ts/train.py:232 step:93K smpl:747K ep:3K epch:332.98 loss:0.037 grdn:5.474 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
283 |
+
INFO 2025-06-15 02:29:06 ts/train.py:232 step:94K smpl:749K ep:3K epch:333.69 loss:0.037 grdn:5.501 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
284 |
+
INFO 2025-06-15 02:29:20 ts/train.py:232 step:94K smpl:750K ep:3K epch:334.40 loss:0.037 grdn:5.910 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
285 |
+
INFO 2025-06-15 02:29:35 ts/train.py:232 step:94K smpl:752K ep:3K epch:335.12 loss:0.036 grdn:5.649 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
286 |
+
INFO 2025-06-15 02:29:49 ts/train.py:232 step:94K smpl:754K ep:3K epch:335.83 loss:0.036 grdn:5.402 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
287 |
+
INFO 2025-06-15 02:30:03 ts/train.py:232 step:94K smpl:755K ep:3K epch:336.54 loss:0.036 grdn:5.492 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
288 |
+
INFO 2025-06-15 02:30:17 ts/train.py:232 step:95K smpl:757K ep:3K epch:337.25 loss:0.036 grdn:5.247 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
289 |
+
INFO 2025-06-15 02:30:31 ts/train.py:232 step:95K smpl:758K ep:3K epch:337.97 loss:0.036 grdn:5.389 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
290 |
+
INFO 2025-06-15 02:30:45 ts/train.py:232 step:95K smpl:760K ep:3K epch:338.68 loss:0.036 grdn:5.655 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
291 |
+
INFO 2025-06-15 02:31:00 ts/train.py:232 step:95K smpl:762K ep:3K epch:339.39 loss:0.036 grdn:5.814 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
292 |
+
INFO 2025-06-15 02:31:14 ts/train.py:232 step:95K smpl:763K ep:3K epch:340.11 loss:0.036 grdn:5.092 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
293 |
+
INFO 2025-06-15 02:31:28 ts/train.py:232 step:96K smpl:765K ep:3K epch:340.82 loss:0.035 grdn:5.137 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
294 |
+
INFO 2025-06-15 02:31:42 ts/train.py:232 step:96K smpl:766K ep:3K epch:341.53 loss:0.036 grdn:5.573 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
295 |
+
INFO 2025-06-15 02:31:56 ts/train.py:232 step:96K smpl:768K ep:3K epch:342.25 loss:0.036 grdn:5.464 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
296 |
+
INFO 2025-06-15 02:32:10 ts/train.py:232 step:96K smpl:770K ep:3K epch:342.96 loss:0.036 grdn:5.253 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
297 |
+
INFO 2025-06-15 02:32:25 ts/train.py:232 step:96K smpl:771K ep:3K epch:343.67 loss:0.036 grdn:5.516 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
298 |
+
INFO 2025-06-15 02:32:39 ts/train.py:232 step:97K smpl:773K ep:3K epch:344.39 loss:0.036 grdn:5.734 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
299 |
+
INFO 2025-06-15 02:32:53 ts/train.py:232 step:97K smpl:774K ep:3K epch:345.10 loss:0.035 grdn:5.430 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
300 |
+
INFO 2025-06-15 02:33:07 ts/train.py:232 step:97K smpl:776K ep:3K epch:345.81 loss:0.035 grdn:5.375 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
301 |
+
INFO 2025-06-15 02:33:21 ts/train.py:232 step:97K smpl:778K ep:3K epch:346.52 loss:0.036 grdn:5.426 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
302 |
+
INFO 2025-06-15 02:33:36 ts/train.py:232 step:97K smpl:779K ep:3K epch:347.24 loss:0.035 grdn:5.084 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
303 |
+
INFO 2025-06-15 02:33:50 ts/train.py:232 step:98K smpl:781K ep:3K epch:347.95 loss:0.035 grdn:5.293 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
304 |
+
INFO 2025-06-15 02:34:04 ts/train.py:232 step:98K smpl:782K ep:3K epch:348.66 loss:0.036 grdn:5.580 lr:1.0e-05 updt_s:0.070 data_s:0.002
|
305 |
+
INFO 2025-06-15 02:34:18 ts/train.py:232 step:98K smpl:784K ep:3K epch:349.38 loss:0.036 grdn:5.605 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
306 |
+
INFO 2025-06-15 02:34:33 ts/train.py:232 step:98K smpl:786K ep:3K epch:350.09 loss:0.036 grdn:5.631 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
307 |
+
INFO 2025-06-15 02:34:47 ts/train.py:232 step:98K smpl:787K ep:3K epch:350.80 loss:0.036 grdn:5.750 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
308 |
+
INFO 2025-06-15 02:35:01 ts/train.py:232 step:99K smpl:789K ep:3K epch:351.52 loss:0.036 grdn:5.558 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
309 |
+
INFO 2025-06-15 02:35:15 ts/train.py:232 step:99K smpl:790K ep:3K epch:352.23 loss:0.035 grdn:5.340 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
310 |
+
INFO 2025-06-15 02:35:29 ts/train.py:232 step:99K smpl:792K ep:3K epch:352.94 loss:0.034 grdn:5.233 lr:1.0e-05 updt_s:0.069 data_s:0.000
|
311 |
+
INFO 2025-06-15 02:35:43 ts/train.py:232 step:99K smpl:794K ep:3K epch:353.65 loss:0.035 grdn:5.347 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
312 |
+
INFO 2025-06-15 02:35:58 ts/train.py:232 step:99K smpl:795K ep:3K epch:354.37 loss:0.035 grdn:5.340 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
313 |
+
INFO 2025-06-15 02:36:12 ts/train.py:232 step:100K smpl:797K ep:3K epch:355.08 loss:0.035 grdn:5.558 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
314 |
+
INFO 2025-06-15 02:36:26 ts/train.py:232 step:100K smpl:798K ep:3K epch:355.79 loss:0.035 grdn:5.351 lr:1.0e-05 updt_s:0.070 data_s:0.000
|
315 |
+
INFO 2025-06-15 02:36:40 ts/train.py:232 step:100K smpl:800K ep:3K epch:356.51 loss:0.035 grdn:5.366 lr:1.0e-05 updt_s:0.069 data_s:0.002
|
316 |
+
INFO 2025-06-15 02:36:40 ts/train.py:241 Checkpoint policy after step 100000
|
317 |
+
INFO 2025-06-15 02:36:41 ts/train.py:283 End of training
|
train/act_dice_art_run1/wandb/run-20250615_012525-fhdgxxjz/files/requirements.txt
ADDED
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
starlette==0.46.2
|
2 |
+
httpx-sse==0.4.0
|
3 |
+
regex==2024.11.6
|
4 |
+
markdown-it-py==3.0.0
|
5 |
+
mdurl==0.1.2
|
6 |
+
groovy==0.1.2
|
7 |
+
tqdm==4.67.1
|
8 |
+
joblib==1.5.1
|
9 |
+
httpcore==1.0.9
|
10 |
+
typer==0.16.0
|
11 |
+
pydantic_core==2.33.2
|
12 |
+
typing_extensions==4.14.0
|
13 |
+
mcp==1.9.0
|
14 |
+
aiofiles==24.1.0
|
15 |
+
ruff==0.11.13
|
16 |
+
packaging==25.0
|
17 |
+
pydantic-settings==2.9.1
|
18 |
+
exceptiongroup==1.3.0
|
19 |
+
pandas==2.3.0
|
20 |
+
Jinja2==3.1.6
|
21 |
+
nltk==3.9.1
|
22 |
+
websockets==15.0.1
|
23 |
+
tzdata==2025.2
|
24 |
+
semantic-version==2.10.0
|
25 |
+
fastapi==0.115.12
|
26 |
+
orjson==3.10.18
|
27 |
+
textblob==0.19.0
|
28 |
+
h11==0.16.0
|
29 |
+
python-dateutil==2.9.0.post0
|
30 |
+
pydub==0.25.1
|
31 |
+
sse-starlette==2.3.6
|
32 |
+
tomlkit==0.13.3
|
33 |
+
Pygments==2.19.1
|
34 |
+
anyio==4.9.0
|
35 |
+
shellingham==1.5.4
|
36 |
+
typing-inspection==0.4.1
|
37 |
+
pydantic==2.11.5
|
38 |
+
rich==14.0.0
|
39 |
+
numpy==2.2.6
|
40 |
+
uvicorn==0.34.3
|
41 |
+
huggingface-hub==0.32.4
|
42 |
+
annotated-types==0.7.0
|
43 |
+
python-dotenv==1.1.0
|
44 |
+
safehttpx==0.1.6
|
45 |
+
ffmpy==0.6.0
|
46 |
+
httpx==0.28.1
|
47 |
+
gradio_client==1.10.2
|
48 |
+
python-multipart==0.0.20
|
49 |
+
gradio==5.33.0
|
50 |
+
filelock==3.18.0
|
51 |
+
hf-xet==1.1.3
|
52 |
+
sniffio==1.3.1
|
53 |
+
itsdangerous==2.2.0
|
54 |
+
toml==0.10.2
|
55 |
+
multidict==6.4.4
|
56 |
+
mpmath==1.3.0
|
57 |
+
protobuf==6.31.1
|
58 |
+
cloudpickle==3.1.1
|
59 |
+
torchvision==0.22.1
|
60 |
+
prompt_toolkit==3.0.51
|
61 |
+
blinker==1.9.0
|
62 |
+
PySocks==1.7.1
|
63 |
+
beautifulsoup4==4.13.4
|
64 |
+
nvidia-cuda-runtime-cu12==12.6.77
|
65 |
+
sentry-sdk==2.30.0
|
66 |
+
nvidia-cuda-cupti-cu12==12.6.80
|
67 |
+
h5py==3.14.0
|
68 |
+
aiohttp==3.12.13
|
69 |
+
gitdb==4.0.12
|
70 |
+
setproctitle==1.3.6
|
71 |
+
GitPython==3.1.44
|
72 |
+
six==1.17.0
|
73 |
+
einops==0.8.1
|
74 |
+
Farama-Notifications==0.0.4
|
75 |
+
diffusers==0.33.1
|
76 |
+
cffi==1.17.1
|
77 |
+
pyarrow==20.0.0
|
78 |
+
nvidia-cuda-nvrtc-cu12==12.6.77
|
79 |
+
zipp==3.23.0
|
80 |
+
pycparser==2.22
|
81 |
+
nvidia-nccl-cu12==2.26.2
|
82 |
+
zarr==2.18.3
|
83 |
+
antlr4-python3-runtime==4.9.3
|
84 |
+
numcodecs==0.13.1
|
85 |
+
MarkupSafe==3.0.2
|
86 |
+
pip==25.1
|
87 |
+
mergedeep==1.3.4
|
88 |
+
certifi==2025.6.15
|
89 |
+
soupsieve==2.7
|
90 |
+
numba==0.61.2
|
91 |
+
safetensors==0.5.3
|
92 |
+
gymnasium==0.29.1
|
93 |
+
multiprocess==0.70.16
|
94 |
+
idna==3.10
|
95 |
+
orderly-set==5.4.1
|
96 |
+
networkx==3.4.2
|
97 |
+
sympy==1.14.0
|
98 |
+
xxhash==3.5.0
|
99 |
+
pillow==11.2.1
|
100 |
+
draccus==0.10.0
|
101 |
+
platformdirs==4.3.8
|
102 |
+
pyserial==3.5
|
103 |
+
requests==2.32.4
|
104 |
+
fasteners==0.19
|
105 |
+
pyzmq==27.0.0
|
106 |
+
nvidia-cudnn-cu12==9.5.1.17
|
107 |
+
torch==2.7.1
|
108 |
+
nvidia-cusparse-cu12==12.5.4.2
|
109 |
+
psutil==7.0.0
|
110 |
+
scipy==1.15.3
|
111 |
+
async-timeout==5.0.1
|
112 |
+
urllib3==2.4.0
|
113 |
+
nvidia-curand-cu12==10.3.7.77
|
114 |
+
av==14.4.0
|
115 |
+
attrs==25.3.0
|
116 |
+
smmap==5.0.2
|
117 |
+
wandb==0.20.1
|
118 |
+
hf_transfer==0.1.9
|
119 |
+
propcache==0.3.2
|
120 |
+
setuptools==78.1.1
|
121 |
+
imageio-ffmpeg==0.6.0
|
122 |
+
pynput==1.8.1
|
123 |
+
pytz==2025.2
|
124 |
+
pfzy==0.3.4
|
125 |
+
torchcodec==0.4.0
|
126 |
+
imageio==2.37.0
|
127 |
+
nvidia-cublas-cu12==12.6.4.1
|
128 |
+
triton==3.3.1
|
129 |
+
pymunk==6.11.1
|
130 |
+
wcwidth==0.2.13
|
131 |
+
jsonlines==4.0.0
|
132 |
+
python-xlib==0.33
|
133 |
+
mypy_extensions==1.1.0
|
134 |
+
cmake==4.0.3
|
135 |
+
evdev==1.9.2
|
136 |
+
omegaconf==2.3.0
|
137 |
+
asciitree==0.3.3
|
138 |
+
nvidia-cufft-cu12==11.3.0.4
|
139 |
+
charset-normalizer==3.4.2
|
140 |
+
deepdiff==8.5.0
|
141 |
+
click==8.2.1
|
142 |
+
termcolor==3.1.0
|
143 |
+
rerun-sdk==0.23.3
|
144 |
+
aiosignal==1.3.2
|
145 |
+
llvmlite==0.44.0
|
146 |
+
PyYAML==6.0.2
|
147 |
+
importlib_metadata==8.7.0
|
148 |
+
lerobot==0.1.0
|
149 |
+
frozenlist==1.7.0
|
150 |
+
Werkzeug==3.1.3
|
151 |
+
nvidia-nvjitlink-cu12==12.6.85
|
152 |
+
typing-inspect==0.9.0
|
153 |
+
nvidia-cusolver-cu12==11.7.1.2
|
154 |
+
dill==0.3.8
|
155 |
+
pyyaml-include==1.4.1
|
156 |
+
gdown==5.2.0
|
157 |
+
datasets==3.6.0
|
158 |
+
fsspec==2025.3.0
|
159 |
+
yarl==1.20.1
|
160 |
+
inquirerpy==0.3.4
|
161 |
+
nvidia-nvtx-cu12==12.6.77
|
162 |
+
Flask==3.1.1
|
163 |
+
aiohappyeyeballs==2.6.1
|
164 |
+
opencv-python-headless==4.11.0.86
|
165 |
+
wheel==0.45.1
|
166 |
+
nvidia-cufile-cu12==1.11.1.6
|
167 |
+
nvidia-cusparselt-cu12==0.6.3
|