Add files

.gitattributes

@@ -30,3 +30,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step1992/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step2114/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step249/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step747/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step996/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step1245/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step1494/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixnewcodelonglossseq_global_step1743/tokenizer.json filter=lfs diff=lfs merge=lfs -text

xp3capmixnewcodelonglossseq_global_step1245/Context_Section_Type/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "pubmed_qa",
+  "dataset_config_name": "pqa_labeled",
+  "template_name": "Context Section Type",
+  "evaluation": {
+    "accuracy": 0.607
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pqa_labeled', dataset_name='pubmed_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='Context Section Type', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/config.json

@@ -0,0 +1,25 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "attention_dropout": 0.0,
+  "architectures": [
+    "BloomModel"
+  ],
+  "attention_softmax_in_fp32": true,
+  "seq_length": 2048,
+  "pad_token_id": 3,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 14336,
+  "n_layer": 70,
+  "num_attention_heads": 112,
+  "pretraining_tp": 4,
+  "slow_but_exact": false,
+  "transformers_version": "4.21.0",
+  "use_cache": true,
+  "vocab_size": 250880
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/en/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.572
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/en/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.528
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/en/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4044
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/es/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4612
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/es/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4356
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/es/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2894
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/fr/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4738
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/fr/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.458
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/fr/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3222
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/zh/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4434
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/zh/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4262
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/amazon_reviews_multi/zh/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2988
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/aqua_rat/raw/Answer_questions_from_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "Answer questions from options",
+  "evaluation": {
+    "accuracy": 0.2125984251968504
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/aqua_rat/raw/answer_quiz/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "answer_quiz",
+  "evaluation": {
+    "accuracy": 0.22440944881889763
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/aqua_rat/raw/select_the_best_option/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "select_the_best_option",
+  "evaluation": {
+    "accuracy": 0.2204724409448819
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/art/choose_hypothesis/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis",
+  "evaluation": {
+    "accuracy": 0.7003916449086162
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/art/choose_hypothesis_believable/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_believable",
+  "evaluation": {
+    "accuracy": 0.70822454308094
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/art/choose_hypothesis_desc/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_desc",
+  "evaluation": {
+    "accuracy": 0.5802872062663186
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/art/choose_hypothesis_likely/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_likely",
+  "evaluation": {
+    "accuracy": 0.5783289817232375
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/art/choose_hypothesis_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_options",
+  "evaluation": {
+    "accuracy": 0.7075718015665796
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/banking77/direct_to_which_department/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "direct_to_which_department",
+  "evaluation": {
+    "accuracy": 0.19805194805194806
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/banking77/help_page_topic/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "help_page_topic",
+  "evaluation": {
+    "accuracy": 0.32045454545454544
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/banking77/rephrase_as_banking_term/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "rephrase_as_banking_term",
+  "evaluation": {
+    "accuracy": 0.26266233766233765
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=1, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blbooksgenre/title_genre_classifiction/classify/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "classify",
+  "evaluation": {
+    "accuracy": 0.33064516129032256
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blbooksgenre/title_genre_classifiction/multi-choice/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "multi-choice",
+  "evaluation": {
+    "accuracy": 0.8467741935483871
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blbooksgenre/title_genre_classifiction/premise_context_first/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "premise_context_first",
+  "evaluation": {
+    "accuracy": 0.8225806451612904
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blimp/adjunct_island/grammatical_between_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_1_2",
+  "evaluation": {
+    "accuracy": 0.512
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blimp/adjunct_island/grammatical_between_A_B/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_A_B",
+  "evaluation": {
+    "accuracy": 0.475
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blimp/adjunct_island/grammatical_which_one_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_which_one_1_2",
+  "evaluation": {
+    "accuracy": 0.528
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blimp/adjunct_island/single_sentence_bad_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_bad_yes_no",
+  "evaluation": {
+    "accuracy": 0.514
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/blimp/adjunct_island/single_sentence_good_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_good_yes_no",
+  "evaluation": {
+    "accuracy": 0.488
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/climate_fever/claim_and_all_supporting_evidences/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "claim_and_all_supporting_evidences",
+  "evaluation": {
+    "accuracy": 0.35309446254071664
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=2, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/climate_fever/fifth_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "fifth_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.650814332247557
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/climate_fever/first_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "first_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.5576547231270358
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/climate_fever/second_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "second_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.5785016286644951
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/climate_fever/third_evidence_claim_pair/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "third_evidence_claim_pair",
+  "evaluation": {
+    "accuracy": 0.6006514657980456
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/codah/codah/affirmative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.8393371757925072
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/codah/codah/affirmative_instruction_before_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_before_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.8353746397694525
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/codah/codah/interrogative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "interrogative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.840778097982709
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/commonsense_qa/answer_given_question_without_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "answer_given_question_without_options",
+  "evaluation": {
+    "accuracy": 0.7207207207207207
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/commonsense_qa/most_suitable_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "most_suitable_answer",
+  "evaluation": {
+    "accuracy": 0.9205569205569205
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/commonsense_qa/question_answering/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "question_answering",
+  "evaluation": {
+    "accuracy": 0.9123669123669124
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/conv_ai_3/ambiguous/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "ambiguous",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/conv_ai_3/clarification_needed/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "clarification_needed",
+  "evaluation": {
+    "accuracy": 0.40077821011673154
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/conv_ai_3/directly_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "directly_answer",
+  "evaluation": {
+    "accuracy": 0.5940337224383917
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/conv_ai_3/score_give_number/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_give_number",
+  "evaluation": {
+    "accuracy": 0.395157803718115
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixnewcodelonglossseq_global_step1245/evaluation/conv_ai_3/score_how_much/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_how_much",
+  "evaluation": {
+    "accuracy": 0.3584089926502378
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='bfloat16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/176bt0/xp3capmixnewcodelonglossseq_global_step1245/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)"
+}