Training in progress, step 4000, checkpoint

checkpoint-4000/config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "gpt2-medium",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 0,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 1,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 512,
+  "n_embd": 1024,
+  "n_head": 16,
+  "n_inner": null,
+  "n_layer": 24,
+  "n_positions": 512,
+  "n_special": 0,
+  "pad_token_id": 2,
+  "predict_special_tokens": true,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 256
+}

checkpoint-4000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "transformers_version": "4.44.2"
+}

checkpoint-4000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab9bb7f46c5c05ba7fb8e1c8a896b200063c18fa8d87a705e79b40a084c27c32
+size 1212421632

checkpoint-4000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:282ed15dff0c2629a4e2c0f3580189313097fc22e85d90b5bb33bfc2f8e869b1
+size 2425026746

checkpoint-4000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a07d587999af5f9ed0ea595e5eeec35bd15753664df82830fd06a37d2f0706f
+size 14244

checkpoint-4000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d841f56ef62ec4f681758a6a1d49d89b5cab4470bdd255e1a07506424283626e
+size 1064

checkpoint-4000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-4000/tokenizer.json

@@ -0,0 +1,553 @@
+{
+  "version": "1.0",
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<bos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<eos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<bos>": 0,
+      "<eos>": 1,
+      "<pad>": 2,
+      "#": 3,
+      "+": 4,
+      "-": 5,
+      "1": 6,
+      "2": 7,
+      "3": 8,
+      "4": 9,
+      "5": 10,
+      "6": 11,
+      "7": 12,
+      "8": 13,
+      "=": 14,
+      "B": 15,
+      "K": 16,
+      "N": 17,
+      "O": 18,
+      "Q": 19,
+      "R": 20,
+      "a": 21,
+      "b": 22,
+      "c": 23,
+      "d": 24,
+      "e": 25,
+      "f": 26,
+      "g": 27,
+      "h": 28,
+      "x": 29,
+      "Ġ": 30,
+      "ĠN": 31,
+      "ĠR": 32,
+      "ĠB": 33,
+      "ĠQ": 34,
+      "ĠK": 35,
+      "xd": 36,
+      "xe": 37,
+      "Ġe": 38,
+      "Ġd": 39,
+      "Ġc": 40,
+      "xf": 41,
+      "ĠNf": 42,
+      "xc": 43,
+      "Ġh": 44,
+      "Ġg": 45,
+      "ĠNc": 46,
+      "Ġf": 47,
+      "Ġb": 48,
+      "Ġa": 49,
+      "ĠO": 50,
+      "ĠBe": 51,
+      "xg": 52,
+      "ĠBg": 53,
+      "ĠKf": 54,
+      "ĠNd": 55,
+      "ĠNe": 56,
+      "xb": 57,
+      "ĠBd": 58,
+      "ĠKg": 59,
+      "ĠRe": 60,
+      "ĠRa": 61,
+      "ĠRd": 62,
+      "ĠRc": 63,
+      "ĠRf": 64,
+      "ĠKe": 65,
+      "ĠQd": 66,
+      "ĠQe": 67,
+      "ĠBf": 68,
+      "ĠKh": 69,
+      "ĠBb": 70,
+      "ĠQc": 71,
+      "ĠNb": 72,
+      "ĠKd": 73,
+      "ĠRb": 74,
+      "xa": 75,
+      "ĠBc": 76,
+      "xh": 77,
+      "ĠQf": 78,
+      "ĠNg": 79,
+      "Ġcxd": 80,
+      "ĠNxe": 81,
+      "ĠQb": 82,
+      "ĠNxd": 83,
+      "ĠKc": 84,
+      "ĠBxf": 85,
+      "ĠQxd": 86,
+      "ĠQg": 87,
+      "ĠRxd": 88,
+      "ĠRh": 89,
+      "ĠKb": 90,
+      "Ġexd": 91,
+      "ĠRg": 92,
+      "ĠBxe": 93,
+      "ĠBxd": 94,
+      "Ġdxe": 95,
+      "ĠRxe": 96,
+      "ĠBxc": 97,
+      "ĠQh": 98,
+      "ĠQa": 99,
+      "ĠQxe": 100,
+      "ĠRxc": 101,
+      "Ġbxc": 102,
+      "ĠRxf": 103,
+      "ĠNbd": 104,
+      "ĠNxc": 105,
+      "ĠQxf": 106,
+      "ĠBh": 107,
+      "Ġhxg": 108,
+      "Ġfxe": 109,
+      "ĠNxf": 110,
+      "Ġaxb": 111,
+      "Ġdxc": 112,
+      "ĠNh": 113,
+      "ĠQxc": 114,
+      "Ġgxf": 115,
+      "Ġexf": 116,
+      "ĠRxa": 117,
+      "ĠRxb": 118,
+      "ĠNa": 119,
+      "ĠBxg": 120,
+      "ĠBa": 121,
+      "ĠKa": 122,
+      "ĠQxb": 123,
+      "ĠRad": 124,
+      "ĠBxb": 125,
+      "ĠQxg": 126,
+      "ĠKxg": 127,
+      "ĠRxg": 128,
+      "Ġfxg": 129,
+      "ĠRxh": 130,
+      "ĠKxf": 131,
+      "ĠQxa": 132,
+      "ĠRfd": 133,
+      "ĠNxg": 134,
+      "ĠRac": 135,
+      "ĠRfe": 136,
+      "ĠNxb": 137,
+      "ĠQxh": 138,
+      "ĠBxh": 139,
+      "ĠBxa": 140,
+      "Ġcxb": 141,
+      "Ġgxh": 142,
+      "Nf": 143,
+      "ĠRae": 144,
+      "ĠKxd": 145,
+      "ĠKxe": 146,
+      "Ġbxa": 147,
+      "ĠNge": 148,
+      "ĠKxh": 149,
+      "ĠNxa": 150,
+      "ĠNxh": 151,
+      "ĠRab": 152,
+      "ĠRfc": 153,
+      "ĠNgf": 154,
+      "ĠKxc": 155,
+      "ĠNfd": 156,
+      "ĠKxb": 157,
+      "ĠRhe": 158,
+      "ĠKxa": 159,
+      "ĠRed": 160,
+      "ĠNce": 161,
+      "ĠRde": 162,
+      "ĠRcd": 163,
+      "ĠRhd": 164,
+      "ĠRaf": 165,
+      "ĠNdf": 166,
+      "ĠNbc": 167,
+      "ĠRhg": 168,
+      "ĠRfb": 169,
+      "ĠRhf": 170,
+      "ĠRdd": 171,
+      "ĠRbc": 172,
+      "ĠRee": 173,
+      "ĠRec": 174,
+      "ĠNde": 175,
+      "ĠRbd": 176,
+      "ĠRdc": 177,
+      "ĠRcc": 178,
+      "ĠRdf": 179,
+      "ĠRce": 180,
+      "ĠRdg": 181,
+      "ĠNed": 182,
+      "ĠNcd": 183,
+      "ĠNdb": 184,
+      "ĠNfe": 185,
+      "ĠRbe": 186,
+      "ĠRhc": 187,
+      "ĠNdxe": 188,
+      "ĠRbb": 189,
+      "ĠRff": 190,
+      "ĠRef": 191,
+      "Nc": 192,
+      "ĠRaa": 193,
+      "ĠNbxd": 194,
+      "ĠNhf": 195,
+      "ĠNcxe": 196,
+      "ĠRag": 197,
+      "ĠNeg": 198,
+      "ĠNgxe": 199,
+      "ĠNec": 200,
+      "ĠReb": 201,
+      "ĠNef": 202,
+      "ĠRaxd": 203,
+      "ĠRfa": 204,
+      "ĠRgf": 205,
+      "ĠRfxd": 206,
+      "ĠNdc": 207,
+      "ĠNac": 208,
+      "ĠRcf": 209,
+      "ĠNfxd": 210,
+      "ĠRcb": 211,
+      "ĠNfg": 212,
+      "ĠRdh": 213,
+      "ĠRdb": 214,
+      "ĠRaxc": 215,
+      "ĠRge": 216,
+      "ĠRbf": 217,
+      "ĠNcb": 218,
+      "ĠNfxe": 219,
+      "ĠRhb": 220,
+      "ĠNexd": 221,
+      "ĠRfg": 222,
+      "ĠRea": 223,
+      "ĠRgg": 224,
+      "ĠRhh": 225,
+      "ĠNcxd": 226,
+      "ĠRgd": 227,
+      "ĠRfxf": 228,
+      "ĠRah": 229,
+      "ĠQff": 230,
+      "ĠQgg": 231,
+      "ĠRcg": 232,
+      "ĠReg": 233,
+      "ĠQee": 234,
+      "ĠQbb": 235,
+      "ĠRca": 236,
+      "ĠRdxd": 237,
+      "ĠRexe": 238,
+      "ĠQcc": 239,
+      "ĠNab": 240,
+      "ĠQdd": 241,
+      "ĠRba": 242,
+      "ĠRcxc": 243,
+      "ĠRda": 244,
+      "ĠRbg": 245,
+      "ĠRexd": 246,
+      "ĠQca": 247,
+      "ĠRgc": 248,
+      "ĠRbxb": 249,
+      "ĠRgh": 250,
+      "ĠRcxd": 251,
+      "ĠQhh": 252,
+      "ĠNdxc": 253,
+      "ĠQdb": 254,
+      "ĠQaa": 255
+    },
+    "merges": [
+      "Ġ N",
+      "Ġ R",
+      "Ġ B",
+      "Ġ Q",
+      "Ġ K",
+      "x d",
+      "x e",
+      "Ġ e",
+      "Ġ d",
+      "Ġ c",
+      "x f",
+      "ĠN f",
+      "x c",
+      "Ġ h",
+      "Ġ g",
+      "ĠN c",
+      "Ġ f",
+      "Ġ b",
+      "Ġ a",
+      "Ġ O",
+      "ĠB e",
+      "x g",
+      "ĠB g",
+      "ĠK f",
+      "ĠN d",
+      "ĠN e",
+      "x b",
+      "ĠB d",
+      "ĠK g",
+      "ĠR e",
+      "ĠR a",
+      "ĠR d",
+      "ĠR c",
+      "ĠR f",
+      "ĠK e",
+      "ĠQ d",
+      "ĠQ e",
+      "ĠB f",
+      "ĠK h",
+      "ĠB b",
+      "ĠQ c",
+      "ĠN b",
+      "ĠK d",
+      "ĠR b",
+      "x a",
+      "ĠB c",
+      "x h",
+      "ĠQ f",
+      "ĠN g",
+      "Ġc xd",
+      "ĠN xe",
+      "ĠQ b",
+      "ĠN xd",
+      "ĠK c",
+      "ĠB xf",
+      "ĠQ xd",
+      "ĠQ g",
+      "ĠR xd",
+      "ĠR h",
+      "ĠK b",
+      "Ġe xd",
+      "ĠR g",
+      "ĠB xe",
+      "ĠB xd",
+      "Ġd xe",
+      "ĠR xe",
+      "ĠB xc",
+      "ĠQ h",
+      "ĠQ a",
+      "ĠQ xe",
+      "ĠR xc",
+      "Ġb xc",
+      "ĠR xf",
+      "ĠNb d",
+      "ĠN xc",
+      "ĠQ xf",
+      "ĠB h",
+      "Ġh xg",
+      "Ġf xe",
+      "ĠN xf",
+      "Ġa xb",
+      "Ġd xc",
+      "ĠN h",
+      "ĠQ xc",
+      "Ġg xf",
+      "Ġe xf",
+      "ĠR xa",
+      "ĠR xb",
+      "ĠN a",
+      "ĠB xg",
+      "ĠB a",
+      "ĠK a",
+      "ĠQ xb",
+      "ĠRa d",
+      "ĠB xb",
+      "ĠQ xg",
+      "ĠK xg",
+      "ĠR xg",
+      "Ġf xg",
+      "ĠR xh",
+      "ĠK xf",
+      "ĠQ xa",
+      "ĠRf d",
+      "ĠN xg",
+      "ĠRa c",
+      "ĠRf e",
+      "ĠN xb",
+      "ĠQ xh",
+      "ĠB xh",
+      "ĠB xa",
+      "Ġc xb",
+      "Ġg xh",
+      "N f",
+      "ĠRa e",
+      "ĠK xd",
+      "ĠK xe",
+      "Ġb xa",
+      "ĠNg e",
+      "ĠK xh",
+      "ĠN xa",
+      "ĠN xh",
+      "ĠRa b",
+      "ĠRf c",
+      "ĠNg f",
+      "ĠK xc",
+      "ĠNf d",
+      "ĠK xb",
+      "ĠRh e",
+      "ĠK xa",
+      "ĠRe d",
+      "ĠNc e",
+      "ĠRd e",
+      "ĠRc d",
+      "ĠRh d",
+      "ĠRa f",
+      "ĠNd f",
+      "ĠNb c",
+      "ĠRh g",
+      "ĠRf b",
+      "ĠRh f",
+      "ĠRd d",
+      "ĠRb c",
+      "ĠRe e",
+      "ĠRe c",
+      "ĠNd e",
+      "ĠRb d",
+      "ĠRd c",
+      "ĠRc c",
+      "ĠRd f",
+      "ĠRc e",
+      "ĠRd g",
+      "ĠNe d",
+      "ĠNc d",
+      "ĠNd b",
+      "ĠNf e",
+      "ĠRb e",
+      "ĠRh c",
+      "ĠNd xe",
+      "ĠRb b",
+      "ĠRf f",
+      "ĠRe f",
+      "N c",
+      "ĠRa a",
+      "ĠNb xd",
+      "ĠNh f",
+      "ĠNc xe",
+      "ĠRa g",
+      "ĠNe g",
+      "ĠNg xe",
+      "ĠNe c",
+      "ĠRe b",
+      "ĠNe f",
+      "ĠRa xd",
+      "ĠRf a",
+      "ĠRg f",
+      "ĠRf xd",
+      "ĠNd c",
+      "ĠNa c",
+      "ĠRc f",
+      "ĠNf xd",
+      "ĠRc b",
+      "ĠNf g",
+      "ĠRd h",
+      "ĠRd b",
+      "ĠRa xc",
+      "ĠRg e",
+      "ĠRb f",
+      "ĠNc b",
+      "ĠNf xe",
+      "ĠRh b",
+      "ĠNe xd",
+      "ĠRf g",
+      "ĠRe a",
+      "ĠRg g",
+      "ĠRh h",
+      "ĠNc xd",
+      "ĠRg d",
+      "ĠRf xf",
+      "ĠRa h",
+      "ĠQf f",
+      "ĠQg g",
+      "ĠRc g",
+      "ĠRe g",
+      "ĠQe e",
+      "ĠQb b",
+      "ĠRc a",
+      "ĠRd xd",
+      "ĠRe xe",
+      "ĠQc c",
+      "ĠNa b",
+      "ĠQd d",
+      "ĠRb a",
+      "ĠRc xc",
+      "ĠRd a",
+      "ĠRb g",
+      "ĠRe xd",
+      "ĠQc a",
+      "ĠRg c",
+      "ĠRb xb",
+      "ĠRg h",
+      "ĠRc xd",
+      "ĠQh h",
+      "ĠNd xc",
+      "ĠQd b",
+      "ĠQa a"
+    ]
+  }
+}

checkpoint-4000/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<eos>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-4000/trainer_state.json

@@ -0,0 +1,102 @@
+{
+  "best_metric": 1.2697532176971436,
+  "best_model_checkpoint": "chessgpt2-medium-m/checkpoint-4000",
+  "epoch": 0.5119017148707448,
+  "eval_steps": 1000,
+  "global_step": 4000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.1279754287176862,
+      "grad_norm": 1.6428743600845337,
+      "learning_rate": 4.9959725443266765e-05,
+      "loss": 2.4607,
+      "step": 1000
+    },
+    {
+      "epoch": 0.1279754287176862,
+      "eval_loss": 1.8105345964431763,
+      "eval_runtime": 376.5583,
+      "eval_samples_per_second": 73.781,
+      "eval_steps_per_second": 9.223,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2559508574353724,
+      "grad_norm": 1.0423442125320435,
+      "learning_rate": 4.952992634500169e-05,
+      "loss": 1.6726,
+      "step": 2000
+    },
+    {
+      "epoch": 0.2559508574353724,
+      "eval_loss": 1.4938699007034302,
+      "eval_runtime": 376.7393,
+      "eval_samples_per_second": 73.746,
+      "eval_steps_per_second": 9.219,
+      "step": 2000
+    },
+    {
+      "epoch": 0.3839262861530586,
+      "grad_norm": 1.234433889389038,
+      "learning_rate": 4.863737385256343e-05,
+      "loss": 1.4686,
+      "step": 3000
+    },
+    {
+      "epoch": 0.3839262861530586,
+      "eval_loss": 1.3617429733276367,
+      "eval_runtime": 379.1693,
+      "eval_samples_per_second": 73.273,
+      "eval_steps_per_second": 9.159,
+      "step": 3000
+    },
+    {
+      "epoch": 0.5119017148707448,
+      "grad_norm": 1.1198837757110596,
+      "learning_rate": 4.729890583531792e-05,
+      "loss": 1.3601,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5119017148707448,
+      "eval_loss": 1.2697532176971436,
+      "eval_runtime": 376.7562,
+      "eval_samples_per_second": 73.743,
+      "eval_steps_per_second": 9.218,
+      "step": 4000
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 23442,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.575962706644828e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-4000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77066658613413ca5a1b75c4af73f99c524573bd7e4e50b485269c488fae3e35
+size 5240