Upload folder using huggingface_hub

config.json

@@ -0,0 +1,53 @@
+{
+  "_name_or_path": "/net/nfs.cirrascale/allennlp/hamishi/checkpoints/olmo_7b_finetune",
+  "activation_type": "swiglu",
+  "alibi": false,
+  "alibi_bias_max": 8.0,
+  "architectures": [
+    "OLMoForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_layer_norm": false,
+  "attention_layer_norm_with_affine": false,
+  "auto_map": {
+    "AutoConfig": "configuration_olmo.OLMoConfig",
+    "AutoModelForCausalLM": "modeling_olmo.OLMoForCausalLM",
+    "AutoTokenizer": [
+      "tokenization_olmo_fast.OLMoTokenizerFast",
+      "tokenization_olmo_fast.OLMoTokenizerFast"
+    ]
+  },
+  "bias_for_layer_norm": false,
+  "block_group_size": 1,
+  "block_type": "sequential",
+  "d_model": 4096,
+  "embedding_dropout": 0.0,
+  "embedding_size": 50304,
+  "eos_token_id": 50279,
+  "flash_attention": true,
+  "include_bias": false,
+  "init_cutoff_factor": null,
+  "init_device": "meta",
+  "init_fn": "mitchell",
+  "init_std": 0.02,
+  "layer_norm_type": "default",
+  "layer_norm_with_affine": false,
+  "max_sequence_length": 2048,
+  "mlp_hidden_size": 22016,
+  "mlp_ratio": 4,
+  "model_type": "olmo",
+  "multi_query_attention": false,
+  "n_heads": 32,
+  "n_layers": 32,
+  "pad_token_id": 1,
+  "precision": "amp_bf16",
+  "residual_dropout": 0.0,
+  "rope": true,
+  "rope_full_precision": true,
+  "scale_logits": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50280,
+  "weight_tying": false
+}

configuration_olmo.py

@@ -0,0 +1,3 @@
+from hf_olmo import OLMoConfig
+
+

modeling_olmo.py

@@ -0,0 +1,3 @@
+from hf_olmo import OLMoForCausalLM
+
+

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c01e8e5db3174103ebbf4435dfb9c746e6fb2eb890966429e4a9e182ee3c277
+size 9945777322

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3ae639bb44aca919f0c8f0966824d2f4dfe2eca5719eba0d5d70a02ed8cfbec
+size 3830460110

pytorch_model.bin.index.json

@@ -0,0 +1,137 @@
+{
+  "metadata": {
+    "total_size": 13776191488
+  },
+  "weight_map": {
+    "model.transformer.blocks.0.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.0.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.0.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.0.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.1.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.1.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.1.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.1.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.10.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.10.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.10.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.10.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.11.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.11.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.11.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.11.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.12.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.12.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.12.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.12.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.13.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.13.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.13.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.13.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.14.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.14.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.14.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.14.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.15.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.15.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.15.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.15.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.16.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.16.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.16.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.16.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.17.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.17.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.17.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.17.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.18.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.18.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.18.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.18.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.19.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.19.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.19.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.19.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.2.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.2.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.2.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.2.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.20.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.20.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.20.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.20.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.21.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.21.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.21.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.21.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.22.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.22.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.22.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.22.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.23.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.23.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.23.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.23.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.24.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.24.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.24.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.24.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.25.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.25.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.25.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.25.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.26.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.26.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.26.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.26.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.27.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.27.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.27.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.27.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.28.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.28.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.28.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.28.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.29.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.29.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.29.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.29.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.3.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.3.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.3.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.3.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.30.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.30.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.30.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.30.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.31.att_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.31.attn_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.31.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.31.ff_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.blocks.4.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.4.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.4.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.4.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.5.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.5.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.5.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.5.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.6.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.6.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.6.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.6.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.7.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.7.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.7.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.7.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.8.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.8.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.8.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.8.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.9.att_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.9.attn_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.9.ff_out.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.blocks.9.ff_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.transformer.ff_out.weight": "pytorch_model-00002-of-00002.bin",
+    "model.transformer.wte.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|padding|>"
+}
+

tokenization_olmo_fast.py

@@ -0,0 +1,2 @@
+from hf_olmo.tokenization_olmo_fast import OLMoTokenizerFast
+

tokenizer_config.json

@@ -0,0 +1,242 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "max_length": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "OLMoTokenizer",
+  "truncation": "right",
+  "auto_map": {
+    "AutoConfig": "configuration_olmo.OLMoConfig",
+    "AutoTokenizer": [
+      "tokenization_olmo_fast.OLMoTokenizerFast",
+      "tokenization_olmo_fast.OLMoTokenizerFast"
+    ]
+  }
+}