Training in progress, step 82

config.json

@@ -1,34 +1,31 @@
 {
-  "_name_or_path": "microsoft/phi-2",
+  "_name_or_path": "stabilityai/stable-code-3b",
   "architectures": [
-    "PhiForCausalLM"
+    "StableLMEpochForCausalLM"
   ],
-  "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "microsoft/phi-2--configuration_phi.PhiConfig",
-    "AutoModelForCausalLM": "microsoft/phi-2--modeling_phi.PhiForCausalLM"
+    "AutoConfig": "configuration_stablelm_epoch.StableLMEpochConfig",
+    "AutoModelForCausalLM": "modeling_stablelm_epoch.StableLMEpochForCausalLM"
   },
-  "bos_token_id": null,
-  "embd_pdrop": 0.0,
-  "eos_token_id": null,
-  "hidden_act": "gelu_new",
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
   "hidden_size": 2560,
   "initializer_range": 0.02,
-  "intermediate_size": 10240,
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 2048,
-  "model_type": "phi",
+  "intermediate_size": 6912,
+  "max_position_embeddings": 16384,
+  "model_type": "stablelm_epoch",
+  "norm_eps": 1e-05,
   "num_attention_heads": 32,
+  "num_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 32,
-  "partial_rotary_factor": 0.4,
-  "qk_layernorm": false,
-  "resid_pdrop": 0.1,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
+  "rope_pct": 0.25,
+  "rope_theta": 1000000,
+  "rotary_scaling_factor": 1.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.37.0.dev0",
   "use_cache": false,
-  "vocab_size": 51200
+  "vocab_size": 50304
 }

configuration_stablelm_epoch.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright 2023 Stability and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" StableLM Epoch model configuration"""
+from transformers import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class StableLMEpochConfig(PretrainedConfig):
+    r"""
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 50_304):
+            Vocabulary size of the StableLM model. Defines the number of different tokens that
+            can be represented by the `inputs_ids` passed when calling [`StableLMEpochModel`].
+        intermediate_size (`int`, *optional*, defaults to 6912):
+            Dimension of the MLP representations.
+        hidden_size (`int`, *optional*, defaults to 2560):
+            Dimension of the decoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string).
+        rope_pct (`float`, *optional*, defaults to 1.0):
+            Percentage of hidden dimensions to allocate to rotary embeddings.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+            Typically set this to something large just in case (e.g., 512 or 1024 or 2048).
+        initializer_range (`float`, *optional*, defaults to 1e-5):
+            The standard deviation of the truncated_normal_initializer for initializing
+             all weight matrices.
+        norm_eps (`float`, *optional*, defaults to 1e-8):
+            The epsilon used by the normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions
+            (not used by all models). Only relevant if `config.is_decoder=True`.
+        tie_word_embeddings(`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+    """
+    model_type = "stablelm_epoch"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=50_304,
+        intermediate_size=6912,
+        hidden_size=2560,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        hidden_act="silu",
+        rope_pct=0.25,
+        rope_theta=10_000,
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        norm_eps=1.0e-5,
+        use_cache=True,
+        bos_token_id=0,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.intermediate_size = intermediate_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.rope_pct = rope_pct
+        self.rope_theta = rope_theta
+        self.initializer_range = initializer_range
+        self.norm_eps = norm_eps
+        self.use_cache = use_cache
+        self.tie_word_embeddings = tie_word_embeddings
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+            )

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
-    "50256": {
+    "0": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -9,8 +9,40 @@
       "single_word": false,
       "special": true
     },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
     "50257": {
-      "content": "                               ",
+      "content": "                     ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -18,7 +50,7 @@
       "special": false
     },
     "50258": {
-      "content": "                              ",
+      "content": "                    ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -26,7 +58,7 @@
       "special": false
     },
     "50259": {
-      "content": "                             ",
+      "content": "                   ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -34,7 +66,7 @@
       "special": false
     },
     "50260": {
-      "content": "                            ",
+      "content": "                  ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -42,7 +74,7 @@
       "special": false
     },
     "50261": {
-      "content": "                           ",
+      "content": "                 ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -50,7 +82,7 @@
       "special": false
     },
     "50262": {
-      "content": "                          ",
+      "content": "                ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -58,7 +90,7 @@
       "special": false
     },
     "50263": {
-      "content": "                         ",
+      "content": "               ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -66,7 +98,7 @@
       "special": false
     },
     "50264": {
-      "content": "                        ",
+      "content": "              ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -74,7 +106,7 @@
       "special": false
     },
     "50265": {
-      "content": "                       ",
+      "content": "             ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -82,7 +114,7 @@
       "special": false
     },
     "50266": {
-      "content": "                      ",
+      "content": "            ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -90,7 +122,7 @@
       "special": false
     },
     "50267": {
-      "content": "                     ",
+      "content": "           ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -98,7 +130,7 @@
       "special": false
     },
     "50268": {
-      "content": "                    ",
+      "content": "          ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -106,7 +138,7 @@
       "special": false
     },
     "50269": {
-      "content": "                   ",
+      "content": "         ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -114,7 +146,7 @@
       "special": false
     },
     "50270": {
-      "content": "                  ",
+      "content": "        ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -122,7 +154,7 @@
       "special": false
     },
     "50271": {
-      "content": "                 ",
+      "content": "       ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -130,7 +162,7 @@
       "special": false
     },
     "50272": {
-      "content": "                ",
+      "content": "      ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -138,7 +170,7 @@
       "special": false
     },
     "50273": {
-      "content": "               ",
+      "content": "     ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -146,7 +178,7 @@
       "special": false
     },
     "50274": {
-      "content": "              ",
+      "content": "    ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -154,7 +186,7 @@
       "special": false
     },
     "50275": {
-      "content": "             ",
+      "content": "   ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -162,7 +194,7 @@
       "special": false
     },
     "50276": {
-      "content": "            ",
+      "content": "  ",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -170,151 +202,151 @@
       "special": false
     },
     "50277": {
-      "content": "           ",
+      "content": "<fim_prefix>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50278": {
-      "content": "          ",
+      "content": "<fim_middle>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50279": {
-      "content": "         ",
+      "content": "<fim_suffix>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50280": {
-      "content": "        ",
+      "content": "<fim_pad>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50281": {
-      "content": "       ",
+      "content": "<filename>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50282": {
-      "content": "      ",
+      "content": "<gh_stars>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50283": {
-      "content": "     ",
+      "content": "<issue_start>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50284": {
-      "content": "    ",
+      "content": "<issue_comment>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50285": {
-      "content": "   ",
+      "content": "<issue_closed>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50286": {
-      "content": "  ",
+      "content": "<jupyter_start>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50287": {
-      "content": "\t\t\t\t\t\t\t\t\t",
+      "content": "<jupyter_text>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50288": {
-      "content": "\t\t\t\t\t\t\t\t",
+      "content": "<jupyter_code>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50289": {
-      "content": "\t\t\t\t\t\t\t",
+      "content": "<jupyter_output>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50290": {
-      "content": "\t\t\t\t\t\t",
+      "content": "<empty_output>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50291": {
-      "content": "\t\t\t\t\t",
+      "content": "<commit_before>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50292": {
-      "content": "\t\t\t\t",
+      "content": "<commit_msg>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50293": {
-      "content": "\t\t\t",
+      "content": "<commit_after>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50294": {
-      "content": "\t\t",
+      "content": "<reponame>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "50295": {
-      "content": "<|im_end|>",
+      "content": "<repo_continuation>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -322,6 +354,22 @@
       "special": true
     },
     "50296": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50297": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50298": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -333,8 +381,8 @@
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|im_end|>",
-  "model_max_length": 2048,
+  "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
-  "tokenizer_class": "CodeGenTokenizer",
+  "tokenizer_class": "GPTNeoXTokenizer",
   "unk_token": "<|endoftext|>"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6befbe8ca7b7438cadbde16c64dabbee9d4de4cbe980bab4bcd735484fa37f0b
+oid sha256:965952c63f3d85ba9b126050b91a41ef727cd00c59a89e0f6e549d31ead26108
 size 4859