Hugging Face's logo

datablations
/
lm5-2b8-55b-c4

TensorBoard
Model card Files Metrics Community
Muennighoff commited on
Commit
b757584
·
1 Parent(s): 79391b8

Add

Browse files
Files changed (6) hide show
  1. eval_all_gen.sh +357 -0
  2. launch.sh +53 -0
  3. rank_eval_all.sh +253 -0
  4. rank_eval_all_prefix_ul2.sh +258 -0
  5. run_eval_lumi_gen.sh +73 -0
  6. run_eval_lumi_xp3eval.sh +162 -0
eval_all_gen.sh ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid005159
3
+ #SBATCH --nodes=1
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=4
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p small-g
8
+ #SBATCH -t 2-0:00:00
9
+ #SBATCH --gpus-per-node=mi250:0
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ # if run without sbatch, invoke here
17
+ if [ -z $SLURM_JOB_ID ]; then
18
+ mkdir -p logs
19
+ sbatch "$0"
20
+ exit
21
+ fi
22
+
23
+ set -euo pipefail
24
+
25
+ # symlink logs/latest_eval.out and logs/latest_eval.err
26
+ ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
27
+ ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
28
+
29
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
30
+
31
+ echo "START TIME: $(date)"
32
+
33
+ # defining the right environment variables
34
+ export HF_DATASETS_OFFLINE=1
35
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
36
+
37
+ # Converted transformer checkpoint
38
+ # cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
39
+
40
+ # Data
41
+ CONFIGS=(
42
+ copa,"best_option",validation
43
+ copa,"choose",validation
44
+ copa,"i_am_hesitating",validation
45
+ copa,"cause_effect",validation
46
+ copa,"plausible_alternatives",validation
47
+ superglue_rte,"MNLI crowdsource",validation
48
+ superglue_rte,"GPT-3 style",validation
49
+ superglue_rte,"does it follow that",validation
50
+ superglue_rte,"should assume",validation
51
+ superglue_rte,"guaranteed true",validation
52
+ anli_r1,"guaranteed/possible/impossible",dev_r1
53
+ anli_r1,"MNLI crowdsource",dev_r1
54
+ anli_r1,"GPT-3 style",dev_r1
55
+ anli_r1,"justified in saying",dev_r1
56
+ anli_r1,"can we infer",dev_r1
57
+ anli_r2,"guaranteed/possible/impossible",dev_r2
58
+ anli_r2,"MNLI crowdsource",dev_r2
59
+ anli_r2,"GPT-3 style",dev_r2
60
+ anli_r2,"justified in saying",dev_r2
61
+ anli_r2,"can we infer",dev_r2
62
+ anli_r3,"guaranteed/possible/impossible",dev_r3
63
+ anli_r3,"MNLI crowdsource",dev_r3
64
+ anli_r3,"GPT-3 style",dev_r3
65
+ anli_r3,"justified in saying",dev_r3
66
+ anli_r3,"can we infer",dev_r3
67
+ cb,"guaranteed/possible/impossible",validation
68
+ cb,"MNLI crowdsource",validation
69
+ cb,"GPT-3 style",validation
70
+ cb,"justified in saying",validation
71
+ cb,"can we infer",validation
72
+ winogrande,"underscore refer to",validation
73
+ winogrande,"Replace",validation
74
+ winogrande,"stand for",validation
75
+ winogrande,"does underscore refer to",validation
76
+ winogrande,"True or False",validation
77
+ story_cloze_2016,"Story Continuation and Options",validation
78
+ story_cloze_2016,"Answer Given options",validation
79
+ story_cloze_2016,"Novel Correct Ending",validation
80
+ story_cloze_2016,"Generate Ending",validation
81
+ story_cloze_2016,"Choose Story Ending",validation
82
+ boolq,"after_reading",validation
83
+ boolq,"GPT-3 Style",validation
84
+ boolq,"yes_no_question",validation
85
+ boolq,"exercise",validation
86
+ boolq,"valid_binary",validation
87
+ arc_easy,"pick_the_most_correct_option",test
88
+ arc_easy,"qa_options",test
89
+ arc_easy,"i_am_hesitating",test
90
+ arc_easy,"multiple_choice",test
91
+ arc_easy,"heres_a_problem",test
92
+ arc_challenge,"pick_the_most_correct_option",test
93
+ arc_challenge,"qa_options",test
94
+ arc_challenge,"i_am_hesitating",test
95
+ arc_challenge,"multiple_choice",test
96
+ arc_challenge,"heres_a_problem",test
97
+ sciq,"Direct Question (Closed Book)",test
98
+ sciq,"Multiple Choice (Closed Book)",test
99
+ sciq,"Multiple Choice Question First",test
100
+ sciq,"Multiple Choice",test
101
+ sciq,"Direct Question",test
102
+ piqa,"what_is_the_correct_ending"
103
+ piqa,"pick_correct_choice_index"
104
+ piqa,"Correct the solution"
105
+ piqa,"choose the most appropriate solution"
106
+ piqa,"no prompt needed"
107
+ GEM/wiki_lingua_en,"tldr_en"
108
+ GEM/wiki_lingua_en,"article_summary_en"
109
+ GEM/wiki_lingua_en,"summarize_above_en"
110
+ GEM/wiki_lingua_en,"rephrase_en"
111
+ GEM/wiki_lingua_en,"write_abstract_en"
112
+ gem_xsum,"article_DOC_summary"
113
+ gem_xsum,"summarize_DOC"
114
+ gem_xsum,"summarize_this_DOC_summary"
115
+ gem_xsum,"DOC_tldr"
116
+ gem_xsum,"DOC_boils_down_to_simple_idea_that"
117
+ GEM/web_nlg_en,"PALM_prompt"
118
+ GEM/web_nlg_en,"explicit-graph-description2"
119
+ GEM/web_nlg_en,"non-explicit-description"
120
+ GEM/web_nlg_en,"very-explicit-description"
121
+ GEM/web_nlg_en,"implicit-graph-description"
122
+ e2e_nlg_cleaned,"generate_text_restaurant"
123
+ e2e_nlg_cleaned,"coherent_text"
124
+ e2e_nlg_cleaned,"create_text_for_me"
125
+ e2e_nlg_cleaned,"generate_gramatically_correct_text"
126
+ e2e_nlg_cleaned,"text"
127
+ )
128
+
129
+ CONFIGS=(
130
+ GEM/wiki_lingua_en,"tldr_en"
131
+ gem_xsum,"article_DOC_summary"
132
+ GEM/web_nlg_en,"PALM_prompt"
133
+ e2e_nlg_cleaned,"generate_text_restaurant"
134
+ )
135
+
136
+ CKPTS=(
137
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/transformers
138
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
139
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
140
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
141
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
142
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
143
+ )
144
+
145
+ CKPTS=(
146
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
147
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
148
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
149
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
150
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
151
+ )
152
+
153
+ CKPTS=(
154
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
155
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
156
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
157
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
158
+ )
159
+
160
+ CKPTS=(
161
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/transformers
162
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
163
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
164
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
165
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
166
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
167
+ )
168
+
169
+ CKPTS=(
170
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
171
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
172
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
173
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
174
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
175
+ )
176
+
177
+ CKPTS=(
178
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
179
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
180
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
181
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
182
+ )
183
+
184
+ CKPTS=(
185
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
186
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
187
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
188
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
189
+ )
190
+
191
+ CKPTS=(
192
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
193
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
194
+ )
195
+
196
+ CKPTS=(
197
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/transformers
198
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/transformers
199
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/transformers
200
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/transformers
201
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/transformers
202
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/transformers
203
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/transformers
204
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/transformers
205
+ )
206
+
207
+ CKPTS=(
208
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
209
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
210
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
211
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
212
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
213
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
214
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
215
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
216
+ )
217
+
218
+ CKPTSX=(
219
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/transformers
220
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/transformers
221
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/transformers
222
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/transformers
223
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/transformers
224
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/transformers
225
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/transformers
226
+ )
227
+
228
+
229
+ CKPTSX=(
230
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
231
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
232
+ )
233
+
234
+ CKPTSX=(
235
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
236
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
237
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
238
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
239
+ )
240
+
241
+ CKPTSX=(
242
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/transformers
243
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/transformers
244
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/transformers
245
+ )
246
+ CKPTS=(
247
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
248
+ )
249
+
250
+
251
+ CKPTS=(
252
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
253
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
254
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
255
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
256
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
257
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
258
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
259
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
260
+ )
261
+
262
+ CKPTS=(
263
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/transformers
264
+ )
265
+
266
+ FEWSHOT_CONFIGS=(
267
+ 0
268
+ 1
269
+ 2
270
+ 3
271
+ 4
272
+ 5
273
+ )
274
+
275
+ TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/gpt2
276
+
277
+
278
+ # Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
279
+ for ((i=0; i<${#CKPTS[@]}; i++)); do
280
+ for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
281
+ for ((k=0; k<${#CONFIGS[@]}; k++)); do
282
+ #echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
283
+
284
+ DATA_CONFIG=${CONFIGS[$k]}
285
+ IFS=',' read dataset_name template_name x <<< "${DATA_CONFIG}"
286
+ MODEL_CKPT=${CKPTS[$i]}
287
+ MODEL_CKPT_NO_TRF=${MODEL_CKPT%/*}
288
+ MODEL_NAME=${MODEL_CKPT_NO_TRF##*/}
289
+ OUTPUT_PATH=$MODEL_CKPT_NO_TRF/evaluation/generation_sss_denoiser
290
+ mkdir -p $OUTPUT_PATH
291
+ OUTPUT_NAME=$MODEL_NAME\_$dataset_name\_$template_name\_${FEWSHOT_CONFIGS[$j]}
292
+
293
+ eval_script="./eval_$i-$j-$k.slurm"
294
+ cat <<EOT > $eval_script
295
+ #!/bin/bash
296
+ #SBATCH --exclude=nid005159
297
+ #SBATCH --nodes=1
298
+ #SBATCH --ntasks-per-node=1
299
+ #SBATCH --cpus-per-task=8
300
+ #SBATCH --mem=256G
301
+ #SBATCH -p small-g
302
+ #SBATCH -t 2-0:00:00
303
+ #SBATCH --gpus-per-node=mi250:1
304
+ #SBATCH --exclusive=user
305
+ #SBATCH --hint=nomultithread
306
+ #SBATCH --account=project_462000119
307
+ #SBATCH -o logs/%j.out
308
+ #SBATCH -e logs/%j.err
309
+ set -euo pipefail
310
+ # symlink logs/latest_eval.out and logs/latest_eval.err
311
+ ln -f -s "\$SLURM_JOB_ID.out" logs/latest_eval.out
312
+ ln -f -s "\$SLURM_JOB_ID.err" logs/latest_eval.err
313
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
314
+ echo "START TIME: $(date)"
315
+ # defining the right environment variables
316
+ export HF_DATASETS_OFFLINE=1
317
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
318
+ # Converted transformer checkpoint
319
+ cd /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/bigscience/lm-evaluation-harness
320
+ python main.py \
321
+ --model_api_name 'hf-causal' \
322
+ --model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
323
+ --device cuda \
324
+ --batch_size 16 \
325
+ --no_tracking \
326
+ --task_name $dataset_name \
327
+ --template_names "$template_name" \
328
+ --bootstrap_iters 10 \
329
+ --limit 3000 \
330
+ --num_fewshot ${FEWSHOT_CONFIGS[$j]} \
331
+ --output_dir $OUTPUT_PATH \
332
+ --output_path "$OUTPUT_NAME"
333
+ python main.py \
334
+ --model_api_name 'hf-causal' \
335
+ --model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
336
+ --device cuda \
337
+ --batch_size 8 \
338
+ --no_tracking \
339
+ --task_name $dataset_name \
340
+ --template_names "$template_name" \
341
+ --bootstrap_iters 10 \
342
+ --limit 3000 \
343
+ --num_fewshot ${FEWSHOT_CONFIGS[$j]} \
344
+ --output_dir $OUTPUT_PATH \
345
+ --output_path "$OUTPUT_NAME"
346
+ echo "END TIME: $(date)"
347
+ EOT
348
+ # Submit the job
349
+ sbatch $eval_script
350
+ # Sleep for a bit to avoid hitting the job submission limit
351
+ sleep 0.1
352
+ done
353
+ done
354
+ done
355
+
356
+
357
+ echo "END TIME: $(date)"
launch.sh ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # Launch script using torch.distributed.run(). Used by slurm
4
+ # scripts, don't invoke directly.
5
+
6
+ # Samuel's fix for apparent error in SLURM initialization
7
+ if [ $SLURM_LOCALID -eq 0 ]; then
8
+ rm -rf /dev/shm/*
9
+ rocm-smi || true
10
+ else
11
+ sleep 2
12
+ fi
13
+
14
+ export NCCL_SOCKET_IFNAME=hsn0,hsn1,hsn2,hsn3
15
+ export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
16
+ export FI_CXI_DEFAULT_CQ_SIZE=131072
17
+
18
+ # debugging (noisy)
19
+ #export NCCL_DEBUG=INFO
20
+ #export RCCL_KERNEL_COLL_TRACE_ENABLE=1
21
+ #export NCCL_DEBUG_SUBSYS=INIT,COLL
22
+
23
+ module --quiet purge
24
+ module load cray-python
25
+
26
+ module load CrayEnv
27
+ module load PrgEnv-cray/8.3.3
28
+ module load craype-accel-amd-gfx90a
29
+ module load cray-python
30
+
31
+ module use /pfs/lustrep2/projappl/project_462000125/samantao-public/mymodules
32
+ module load suse-repo-deps/sam-default
33
+ module load rocm/sam-5.2.3.lua
34
+ module load rccl/sam-develop.lua
35
+ module load aws-ofi-rccl/sam-default.lua
36
+
37
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
38
+
39
+ MASTER_NODE=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
40
+ MASTER_PORT=9999
41
+
42
+ echo "Launching on $SLURMD_NODENAME ($SLURM_PROCID/$SLURM_JOB_NUM_NODES)," \
43
+ "master $MASTER_NODE port $MASTER_PORT," \
44
+ "GPUs $SLURM_GPUS_ON_NODE," \
45
+ "CUDA: $(python -c 'import torch; print(torch.cuda.is_available())')"
46
+
47
+ python -u -m torch.distributed.run \
48
+ --nnodes $SLURM_JOB_NUM_NODES \
49
+ --nproc_per_node $SLURM_GPUS_ON_NODE \
50
+ --node_rank=$SLURM_PROCID \
51
+ --master_addr $MASTER_NODE \
52
+ --master_port $MASTER_PORT \
53
+ "$@"
rank_eval_all.sh ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid005159
3
+ #SBATCH --nodes=1
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=4
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p small-g
8
+ #SBATCH -t 2-0:00:00
9
+ #SBATCH --gpus-per-node=mi250:0
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ # if run without sbatch, invoke here
17
+ if [ -z $SLURM_JOB_ID ]; then
18
+ mkdir -p logs
19
+ sbatch "$0"
20
+ exit
21
+ fi
22
+
23
+ set -euo pipefail
24
+
25
+ # symlink logs/latest_eval.out and logs/latest_eval.err
26
+ ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
27
+ ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
28
+
29
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
30
+
31
+ echo "START TIME: $(date)"
32
+
33
+ # defining the right environment variables
34
+ export HF_DATASETS_OFFLINE=1
35
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
36
+
37
+ # Converted transformer checkpoint
38
+ # cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
39
+
40
+
41
+ CKPTS=(
42
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/global_step80108
43
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/global_step80108
44
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/global_step80108
45
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/global_step80108
46
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/global_step80108
47
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/global_step80108
48
+ )
49
+
50
+ CKPTS=(
51
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/global_step80108
52
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/global_step80108
53
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/global_step80108
54
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/global_step80108
55
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/global_step80108
56
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/global_step80108
57
+ )
58
+
59
+
60
+
61
+ CKPTS=(
62
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/global_step84877
63
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/global_step84877
64
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/global_step84877
65
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/global_step84877
66
+ )
67
+
68
+ CKPTS=(
69
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/global_step84877
70
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/global_step84877
71
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/global_step84877
72
+ )
73
+
74
+ CKPTSX=(
75
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/global_step80108
76
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/global_step80108
77
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/global_step80108
78
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/global_step80108
79
+ )
80
+
81
+ CKPTS=(
82
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
83
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
84
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
85
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
86
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
87
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
88
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
89
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
90
+ )
91
+
92
+ CKPTSX=(
93
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/global_step52452
94
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/global_step52452
95
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/global_step52452
96
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/global_step52452
97
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/global_step52452
98
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/global_step52452
99
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/global_step52452
100
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/global_step52452
101
+ )
102
+
103
+ CKPTSX=(
104
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/global_step52452
105
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/global_step52452
106
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/global_step52452
107
+ )
108
+
109
+ CKPTS=(
110
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
111
+ )
112
+
113
+ CKPTS=(
114
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
115
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
116
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
117
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
118
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
119
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
120
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
121
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
122
+ )
123
+
124
+
125
+ CKPTS=(
126
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/global_step52452
127
+ )
128
+
129
+ FEWSHOT_CONFIGS=(
130
+ 0
131
+ 1
132
+ 2
133
+ 3
134
+ 4
135
+ 5
136
+ )
137
+
138
+ TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2
139
+
140
+ # Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
141
+ for ((i=0; i<${#CKPTS[@]}; i++)); do
142
+ for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
143
+ #echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
144
+ MODEL_CKPT=${CKPTS[$i]}
145
+ MODEL_CKPT_NO_STEP=${MODEL_CKPT%/*}
146
+ MODEL_NAME=${MODEL_CKPT_NO_STEP##*/}
147
+ mkdir -p $MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser
148
+ #mv $MODEL_CKPT_NO_STEP/evaluation/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.* $MODEL_CKPT_NO_STEP/evaluation/rankeval/
149
+ OUTPUT_PATH=$MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.json
150
+ eval_script="./eval_$i-$j.slurm"
151
+ cat <<EOT > $eval_script
152
+ #!/bin/bash
153
+ #SBATCH --exclude=nid005159
154
+ #SBATCH --nodes=1
155
+ #SBATCH --ntasks-per-node=1
156
+ #SBATCH --cpus-per-task=8
157
+ #SBATCH --mem=256G
158
+ #SBATCH -p small-g
159
+ #SBATCH -t 2-0:00:00
160
+ #SBATCH --gpus-per-node=mi250:1
161
+ #SBATCH --exclusive=user
162
+ #SBATCH --hint=nomultithread
163
+ #SBATCH --account=project_462000119
164
+ #SBATCH -o logs/%j.out
165
+ #SBATCH -e logs/%j.err
166
+
167
+ export HF_DATASETS_OFFLINE=1
168
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
169
+
170
+ VOCAB_FILE="gpt2/vocab.json"
171
+ MERGE_FILE="gpt2/merges.txt"
172
+
173
+ PP_SIZE=1
174
+ TP_SIZE=1
175
+ # different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
176
+ # make as big as it can fit into gpu w/o OOM, but not too close to 100%
177
+ EVAL_MICRO_BATCH_SIZE=1
178
+ MICRO_BS_MULTIPLIER=1
179
+
180
+ # Model parameters
181
+ SEQ_LEN=2048
182
+
183
+ # Dummy arguments
184
+ MEGATRON_REQUIRED_ARGS=" \
185
+ --num-layers -1 \
186
+ --hidden-size -1 \
187
+ --num-attention-heads -1 \
188
+ --seq-length -1 \
189
+ --max-position-embeddings -1 \
190
+ "
191
+
192
+ ZERO_STAGE=0
193
+
194
+ mkdir -p ds_configs
195
+ DS_CONFIG_PATH="ds_configs/\$SLURM_JOB_ID.json"
196
+
197
+ cat <<EOF > "\$DS_CONFIG_PATH"
198
+ {
199
+ "train_micro_batch_size_per_gpu": 1,
200
+ "train_batch_size": 1,
201
+ "gradient_clipping": 1.0,
202
+ "zero_optimization": {
203
+ "stage": \$ZERO_STAGE
204
+ },
205
+ "bf16": {
206
+ "enabled": true
207
+ },
208
+ "steps_per_print": 2000,
209
+ "wall_clock_breakdown": false
210
+ }
211
+ EOF
212
+
213
+ DEEPSPEED_ARGS=" \
214
+ --deepspeed \
215
+ --deepspeed_config \$DS_CONFIG_PATH \
216
+ --zero-stage \$ZERO_STAGE \
217
+ "
218
+
219
+ CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate.py \
220
+ --load $MODEL_CKPT \
221
+ --results_path $OUTPUT_PATH \
222
+ --tensor-model-parallel-size \$TP_SIZE \
223
+ --pipeline-model-parallel-size \$PP_SIZE \
224
+ --vocab-file \$VOCAB_FILE \
225
+ --merge-file \$MERGE_FILE \
226
+ --micro-batch-size \$EVAL_MICRO_BATCH_SIZE \
227
+ --no-load-optim \
228
+ --no-load-rng \
229
+ --bf16 \
230
+ --inference \
231
+ --seq-length \$SEQ_LEN \
232
+ --task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
233
+ --intermed_results \
234
+ --adaptive_seq_len \
235
+ --micro_bs_multiplier \$MICRO_BS_MULTIPLIER \
236
+ --fewshots ${FEWSHOT_CONFIGS[$j]} \
237
+ \$MEGATRON_REQUIRED_ARGS \
238
+ \$DEEPSPEED_ARGS \
239
+ "
240
+
241
+ echo "\$CMD"
242
+
243
+ echo "START \$SLURM_JOBID: $(date)"
244
+
245
+ srun --label launch.sh \$CMD
246
+
247
+ echo "END \$SLURM_JOBID: $(date)"
248
+ EOT
249
+ sbatch $eval_script
250
+ # Sleep for a bit to avoid hitting the job submission limit
251
+ sleep 0.1
252
+ done
253
+ done
rank_eval_all_prefix_ul2.sh ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid005159
3
+ #SBATCH --nodes=1
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=4
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p small-g
8
+ #SBATCH -t 2-0:00:00
9
+ #SBATCH --gpus-per-node=mi250:0
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ # if run without sbatch, invoke here
17
+ if [ -z $SLURM_JOB_ID ]; then
18
+ mkdir -p logs
19
+ sbatch "$0"
20
+ exit
21
+ fi
22
+
23
+ set -euo pipefail
24
+
25
+ # symlink logs/latest_eval.out and logs/latest_eval.err
26
+ ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
27
+ ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
28
+
29
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
30
+
31
+ echo "START TIME: $(date)"
32
+
33
+ # defining the right environment variables
34
+ export HF_DATASETS_OFFLINE=1
35
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
36
+
37
+ # Converted transformer checkpoint
38
+ # cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
39
+
40
+
41
+ CKPTS=(
42
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/global_step80108
43
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/global_step80108
44
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/global_step80108
45
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/global_step80108
46
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/global_step80108
47
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/global_step80108
48
+ )
49
+
50
+ CKPTS=(
51
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/global_step80108
52
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/global_step80108
53
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/global_step80108
54
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/global_step80108
55
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/global_step80108
56
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/global_step80108
57
+ )
58
+
59
+
60
+
61
+ CKPTS=(
62
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/global_step84877
63
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/global_step84877
64
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/global_step84877
65
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/global_step84877
66
+ )
67
+
68
+ CKPTS=(
69
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/global_step84877
70
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/global_step84877
71
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/global_step84877
72
+ )
73
+
74
+ CKPTSX=(
75
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/global_step80108
76
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/global_step80108
77
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/global_step80108
78
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/global_step80108
79
+ )
80
+
81
+ CKPTS=(
82
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
83
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
84
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
85
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
86
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
87
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
88
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
89
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
90
+ )
91
+
92
+ CKPTSX=(
93
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/global_step52452
94
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/global_step52452
95
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/global_step52452
96
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/global_step52452
97
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/global_step52452
98
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/global_step52452
99
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/global_step52452
100
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/global_step52452
101
+ )
102
+
103
+ CKPTSX=(
104
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/global_step52452
105
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/global_step52452
106
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/global_step52452
107
+ )
108
+
109
+ CKPTS=(
110
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
111
+ )
112
+
113
+ CKPTS=(
114
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
115
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
116
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
117
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
118
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
119
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
120
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
121
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
122
+ )
123
+
124
+
125
+ CKPTS=(
126
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm4-2b8-55b-c4/global_step52452
127
+ )
128
+
129
+ CKPTS=(
130
+ /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/checkpoints_2b855b55bc4ul2ndfixnew/global_step52452
131
+ )
132
+
133
+ FEWSHOT_CONFIGS=(
134
+ 0
135
+ 1
136
+ 2
137
+ 3
138
+ 4
139
+ 5
140
+ )
141
+
142
+ TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2
143
+
144
+ # Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
145
+ for ((i=0; i<${#CKPTS[@]}; i++)); do
146
+ for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
147
+ #echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
148
+ MODEL_CKPT=${CKPTS[$i]}
149
+ MODEL_CKPT_NO_STEP=${MODEL_CKPT%/*}
150
+ MODEL_NAME=${MODEL_CKPT_NO_STEP##*/}
151
+ mkdir -p $MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser
152
+ #mv $MODEL_CKPT_NO_STEP/evaluation/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.* $MODEL_CKPT_NO_STEP/evaluation/rankeval/
153
+ OUTPUT_PATH=$MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.json
154
+ eval_script="./eval_$i-$j.slurm"
155
+ cat <<EOT > $eval_script
156
+ #!/bin/bash
157
+ #SBATCH --exclude=nid005159
158
+ #SBATCH --nodes=1
159
+ #SBATCH --ntasks-per-node=1
160
+ #SBATCH --cpus-per-task=8
161
+ #SBATCH --mem=256G
162
+ #SBATCH -p small-g
163
+ #SBATCH -t 2-0:00:00
164
+ #SBATCH --gpus-per-node=mi250:1
165
+ #SBATCH --exclusive=user
166
+ #SBATCH --hint=nomultithread
167
+ #SBATCH --account=project_462000119
168
+ #SBATCH -o logs/%j.out
169
+ #SBATCH -e logs/%j.err
170
+
171
+ export HF_DATASETS_OFFLINE=1
172
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
173
+
174
+ VOCAB_FILE="gpt2/vocab.json"
175
+ MERGE_FILE="gpt2/merges.txt"
176
+
177
+ PP_SIZE=1
178
+ TP_SIZE=1
179
+ # different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
180
+ # make as big as it can fit into gpu w/o OOM, but not too close to 100%
181
+ EVAL_MICRO_BATCH_SIZE=1
182
+ MICRO_BS_MULTIPLIER=1
183
+
184
+ # Model parameters
185
+ SEQ_LEN=2048
186
+
187
+ # Dummy arguments
188
+ MEGATRON_REQUIRED_ARGS=" \
189
+ --num-layers -1 \
190
+ --hidden-size -1 \
191
+ --num-attention-heads -1 \
192
+ --seq-length -1 \
193
+ --max-position-embeddings -1 \
194
+ "
195
+
196
+ ZERO_STAGE=0
197
+
198
+ mkdir -p ds_configs
199
+ DS_CONFIG_PATH="ds_configs/\$SLURM_JOB_ID.json"
200
+
201
+ cat <<EOF > "\$DS_CONFIG_PATH"
202
+ {
203
+ "train_micro_batch_size_per_gpu": 1,
204
+ "train_batch_size": 1,
205
+ "gradient_clipping": 1.0,
206
+ "zero_optimization": {
207
+ "stage": \$ZERO_STAGE
208
+ },
209
+ "bf16": {
210
+ "enabled": true
211
+ },
212
+ "steps_per_print": 2000,
213
+ "wall_clock_breakdown": false
214
+ }
215
+ EOF
216
+
217
+ DEEPSPEED_ARGS=" \
218
+ --deepspeed \
219
+ --deepspeed_config \$DS_CONFIG_PATH \
220
+ --zero-stage \$ZERO_STAGE \
221
+ "
222
+
223
+ CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate_prefix_ul2.py \
224
+ --load $MODEL_CKPT \
225
+ --results_path $OUTPUT_PATH \
226
+ --tensor-model-parallel-size \$TP_SIZE \
227
+ --pipeline-model-parallel-size \$PP_SIZE \
228
+ --vocab-file \$VOCAB_FILE \
229
+ --merge-file \$MERGE_FILE \
230
+ --micro-batch-size \$EVAL_MICRO_BATCH_SIZE \
231
+ --no-load-optim \
232
+ --no-load-rng \
233
+ --bf16 \
234
+ --inference \
235
+ --seq-length \$SEQ_LEN \
236
+ --task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
237
+ --intermed_results \
238
+ --adaptive_seq_len \
239
+ --micro_bs_multiplier \$MICRO_BS_MULTIPLIER \
240
+ --fewshots ${FEWSHOT_CONFIGS[$j]} \
241
+ --prefix \
242
+ \$MEGATRON_REQUIRED_ARGS \
243
+ \$DEEPSPEED_ARGS \
244
+ "
245
+
246
+ echo "\$CMD"
247
+
248
+ echo "START \$SLURM_JOBID: $(date)"
249
+
250
+ srun --label launch.sh \$CMD
251
+
252
+ echo "END \$SLURM_JOBID: $(date)"
253
+ EOT
254
+ sbatch $eval_script
255
+ # Sleep for a bit to avoid hitting the job submission limit
256
+ sleep 0.1
257
+ done
258
+ done
run_eval_lumi_gen.sh ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid005159
3
+ #SBATCH --nodes=1
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p small-g
8
+ #SBATCH -t 2-0:00:00
9
+ #SBATCH --gpus-per-node=mi250:1
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ # if run without sbatch, invoke here
17
+ if [ -z $SLURM_JOB_ID ]; then
18
+ mkdir -p logs
19
+ sbatch "$0"
20
+ exit
21
+ fi
22
+
23
+ set -euo pipefail
24
+
25
+ # symlink logs/latest_eval.out and logs/latest_eval.err
26
+ ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
27
+ ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
28
+
29
+ source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
30
+
31
+ echo "START TIME: $(date)"
32
+
33
+ # defining the right environment variables
34
+ export HF_DATASETS_OFFLINE=1
35
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
36
+
37
+ # Converted transformer checkpoint
38
+ MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18bc4/transformers
39
+ MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/transformers
40
+ MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4py/2b855b90c4py/transformers
41
+ #MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-realtasky/transformers
42
+ MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/transformers
43
+ TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/gpt2
44
+
45
+ cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
46
+
47
+ # WMT19 ZH-EN does not work
48
+ DATASETS_AND_CONFIGS=(
49
+ GEM/wiki_lingua_en,"tldr_en"
50
+ gem_xsum,"article_DOC_summary"
51
+ GEM/web_nlg_en,"PALM_prompt"
52
+ e2e_nlg_cleaned,"generate_text_restaurant"
53
+ )
54
+
55
+ DATASET_AND_CONFIG=${DATASETS_AND_CONFIGS[$SLURM_ARRAY_TASK_ID]}
56
+ #echo $ARGUMENT
57
+
58
+ IFS=',' read dataset_name template_name <<< "${DATASET_AND_CONFIG}"
59
+
60
+ # Use this fork of lm-eval: https://github.com/bigscience-workshop/lm-evaluation-harness/pull/109
61
+ python main.py \
62
+ --model_api_name 'hf-causal' \
63
+ --model_args pretrained=$MODEL_CKPT,use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
64
+ --device cuda \
65
+ --batch_size 16 \
66
+ --no_tracking \
67
+ --task_name $dataset_name \
68
+ --template_names $template_name \
69
+ --bootstrap_iters 10 \
70
+ --limit 3000 \
71
+ --num_fewshot 1
72
+
73
+ echo "END TIME: $(date)"
run_eval_lumi_xp3eval.sh ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ #SBATCH --exclude=nid005159
3
+ #SBATCH --nodes=1
4
+ #SBATCH --ntasks-per-node=1
5
+ #SBATCH --cpus-per-task=32
6
+ #SBATCH --mem=256G
7
+ #SBATCH -p small-g
8
+ #SBATCH -t 2-0:00:00
9
+ #SBATCH --gpus-per-node=mi250:1
10
+ #SBATCH --exclusive=user
11
+ #SBATCH --hint=nomultithread
12
+ #SBATCH --account=project_462000119
13
+ #SBATCH -o logs/%j.out
14
+ #SBATCH -e logs/%j.err
15
+
16
+ # if run without sbatch, invoke here
17
+ if [ -z $SLURM_JOB_ID ]; then
18
+ mkdir -p logs
19
+ sbatch "$0"
20
+ exit
21
+ fi
22
+
23
+ set -euo pipefail
24
+
25
+ # symlink logs/latest_eval.out and logs/latest_eval.err
26
+ ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
27
+ ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
28
+
29
+ # Data
30
+ #CHECKPOINT_PATH=/scratch/project_462000119/muennighoff/nov-2022-optimization/checkpoints/global_step10
31
+ #VARIANT=global_step10
32
+
33
+ CHECKPOINT_PATH=lm1-220m/global_step14324
34
+ VARIANT=lm1-220m
35
+ CHECKPOINT_PATH=lm1-220m-7b5-oscar/global_step14324
36
+ VARIANT=lm1-220m-7b5-oscar
37
+
38
+ #CHECKPOINT_PATH=lm1-280m/global_step11269
39
+ #VARIANT=lm1-280m-5b9
40
+ #CHECKPOINT_PATH=lm1-280m-5b9-oscar/global_step11269
41
+ #VARIANT=lm1-280m-5b9-oscar
42
+
43
+ CHECKPOINT_PATH=lm1-1b1-21b-oscar/global_step39672
44
+ VARIANT=lm1-1b1-21b-oscar
45
+ #CHECKPOINT_PATH=lm1-1b1-21b/global_step39672
46
+ #VARIANT=lm1-1b1-21b
47
+
48
+ #CHECKPOINT_PATH=lm1-2b8-55b-oscar/global_step52452
49
+ #VARIANT=lm1-2b8-55b-oscar
50
+ #CHECKPOINT_PATH=lm1-2b8-55b/global_step52452
51
+ #VARIANT=lm1-2b8-55b
52
+
53
+ #CHECKPOINT_PATH=lm1-2b8-55b-oscar/global_step52452
54
+ #VARIANT=lm1-2b8-55b-oscar
55
+ #CHECKPOINT_PATH=lm1-3b9-77b/global_step73814
56
+ #VARIANT=lm1-3b9-77b
57
+
58
+ #CHECKPOINT_PATH=lm1-1b1-21b-c4/global_step39672
59
+ #VARIANT=lm1-1b1-21b-c4
60
+
61
+
62
+ # tensorboard_2b855b11bc4 tensorboard_2b855b14bc4 tensorboard_2b855b18bc4 tensorboard_2b855b28bc4 tensorboard_2b855b9bc4
63
+ #2b855b50c4py 2b855b60c4py 2b855b70c4py 2b855b80c4py 2b855b90c4py
64
+ VARIANT=2b855b70c4py
65
+ CHECKPOINT_PATH=lm1-2b8-55b-c4py/$VARIANT/global_step52452
66
+ #2b855b11bc4 2b855b14bc4 2b855b18bc4 2b855b28bc4 2b855b9bc4
67
+ #VARIANT=2b855b9boscar
68
+ #CHECKPOINT_PATH=lm1-2b8-55b-oscar-repetitions/$VARIANT/global_step52452
69
+
70
+ #VARIANT=realtasky
71
+ #CHECKPOINT_PATH=checkpoints_2b855brealtasky/global_step52452
72
+ #CHECKPOINT_PATH=lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
73
+
74
+
75
+ CHECKPOINT_PATH=checkpoints_2b855b55bc4ul2valfast/global_step52452
76
+ VARIANT=ul2valfast
77
+
78
+ CHECKPOINT_PATH=lm2-2b8-55b-c4-new/global_step52452
79
+ VARIANT=ul2new
80
+
81
+ export HF_DATASETS_OFFLINE=1
82
+ export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
83
+
84
+ VOCAB_FILE="gpt2/vocab.json"
85
+ MERGE_FILE="gpt2/merges.txt"
86
+
87
+ PP_SIZE=1
88
+ TP_SIZE=1
89
+ # different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
90
+ # make as big as it can fit into gpu w/o OOM, but not too close to 100%
91
+ EVAL_MICRO_BATCH_SIZE=1
92
+ MICRO_BS_MULTIPLIER=1
93
+
94
+ # Model parameters
95
+ SEQ_LEN=2048
96
+
97
+ # Dummy arguments
98
+ MEGATRON_REQUIRED_ARGS=" \
99
+ --num-layers -1 \
100
+ --hidden-size -1 \
101
+ --num-attention-heads -1 \
102
+ --seq-length -1 \
103
+ --max-position-embeddings -1 \
104
+ "
105
+
106
+ ZERO_STAGE=0
107
+
108
+ mkdir -p ds_configs
109
+ DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
110
+
111
+ cat <<EOF > $DS_CONFIG_PATH
112
+ {
113
+ "train_micro_batch_size_per_gpu": 1,
114
+ "train_batch_size": 1,
115
+ "gradient_clipping": 1.0,
116
+ "zero_optimization": {
117
+ "stage": $ZERO_STAGE
118
+ },
119
+ "bf16": {
120
+ "enabled": true
121
+ },
122
+ "steps_per_print": 2000,
123
+ "wall_clock_breakdown": false
124
+ }
125
+ EOF
126
+
127
+ DEEPSPEED_ARGS=" \
128
+ --deepspeed \
129
+ --deepspeed_config $DS_CONFIG_PATH \
130
+ --zero-stage $ZERO_STAGE \
131
+ "
132
+
133
+ CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate.py \
134
+ --load $CHECKPOINT_PATH \
135
+ --results_path $VARIANT-results.json \
136
+ --tensor-model-parallel-size $TP_SIZE \
137
+ --pipeline-model-parallel-size $PP_SIZE \
138
+ --vocab-file $VOCAB_FILE \
139
+ --merge-file $MERGE_FILE \
140
+ --micro-batch-size $EVAL_MICRO_BATCH_SIZE \
141
+ --no-load-optim \
142
+ --no-load-rng \
143
+ --bf16 \
144
+ --inference \
145
+ --seq-length $SEQ_LEN \
146
+ --task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
147
+ --intermed_results \
148
+ --adaptive_seq_len \
149
+ --add_denoiser \
150
+ --micro_bs_multiplier $MICRO_BS_MULTIPLIER \
151
+ $MEGATRON_REQUIRED_ARGS \
152
+ $DEEPSPEED_ARGS \
153
+ "
154
+
155
+ echo $CMD
156
+
157
+ echo "START $SLURM_JOBID: $(date)"
158
+
159
+ srun --label launch.sh $CMD
160
+
161
+ echo "END $SLURM_JOBID: $(date)"
162
+