Upload folder using huggingface_hub

beagle.json

@@ -0,0 +1,128 @@
+{
+  "dataset.debug": false,
+  "dataset.eval_path": "./output/datasets/self_ds_Llama-2-7b-chat-hf/",
+  "dataset.git_diff": "",
+  "dataset.git_sha1": "unknown",
+  "dataset.manual_sample_ids": [],
+  "dataset.max_read_items": null,
+  "dataset.output_dir": "output",
+  "dataset.path": "output/datasets/ds_Llama-2-7b-chat-hf",
+  "dataset.read_eagle_format": false,
+  "dataset.run_name": "temp_run",
+  "dataset.seed": 42,
+  "dataset_generation.batch_size": 1,
+  "dataset_generation.debug": false,
+  "dataset_generation.debug_target": null,
+  "dataset_generation.ds_prefix": "ds_",
+  "dataset_generation.git_diff": "",
+  "dataset_generation.git_sha1": "unknown",
+  "dataset_generation.max_length": 4096,
+  "dataset_generation.output_dir": "output",
+  "dataset_generation.run_name": "temp_run",
+  "dataset_generation.save_every": 1000,
+  "dataset_generation.seed": 42,
+  "dataset_generation.sharegpt_path": "Aeala/ShareGPT_Vicuna_unfiltered",
+  "device_names": [
+    "NVIDIA L40S"
+  ],
+  "inference.alpha_stats": false,
+  "inference.debug": false,
+  "inference.draft_growing": false,
+  "inference.draft_tree_shape": "mc_sim_7b_64",
+  "inference.dynamic_draft": true,
+  "inference.dynamic_draft_all_top_k": 59,
+  "inference.dynamic_draft_max_depth": 5,
+  "inference.dynamic_draft_top_k": 10,
+  "inference.git_diff": "",
+  "inference.git_sha1": "unknown",
+  "inference.interactive": false,
+  "inference.max_draft_growing_depth": 100,
+  "inference.max_new_tokens": 512,
+  "inference.mode": "speculative",
+  "inference.output_dir": "output",
+  "inference.run_name": "temp_run",
+  "inference.seed": 42,
+  "inference.timer": false,
+  "modeling.add_noise": true,
+  "modeling.attention_wind": "3",
+  "modeling.ckpt_path": null,
+  "modeling.debug": false,
+  "modeling.decoder_key_remap": {},
+  "modeling.dtype": "torch.float32",
+  "modeling.frozen_targets": [],
+  "modeling.git_diff": "",
+  "modeling.git_sha1": "unknown",
+  "modeling.layer_path": "model.layers",
+  "modeling.lmhead_path": "lm_head",
+  "modeling.load_config_from_model_path": false,
+  "modeling.mask_token": "\u2581blank",
+  "modeling.mask_token_rate": 0,
+  "modeling.model_path": "beagle/models/llama/Llama-2-7b-chat-hf/",
+  "modeling.norm_path": "model.norm",
+  "modeling.only_first_state_distill": false,
+  "modeling.output_dir": "output",
+  "modeling.reuse_layer": null,
+  "modeling.rotary_path": "model.rotary_emb",
+  "modeling.run_name": "temp_run",
+  "modeling.save_loading": true,
+  "modeling.seed": 42,
+  "modeling.strictly_follow_eagle_decoder": false,
+  "modeling.tokenizer_path": "meta-llama/Llama-2-7b-chat-hf",
+  "modeling.use_dyt": false,
+  "modeling.use_fc_eagle": false,
+  "modeling.use_lower_layers": 0,
+  "modeling.use_moe": false,
+  "modeling.use_state_distill": true,
+  "training.adam_beta1": 0.9,
+  "training.adam_beta2": 0.95,
+  "training.average_tokens_across_devices": false,
+  "training.bf16": true,
+  "training.ddp_find_unused_parameters": false,
+  "training.debug": false,
+  "training.disable_sampled_print": false,
+  "training.eval_max_tti_wind": null,
+  "training.eval_steps": 100,
+  "training.eval_strategy": "steps",
+  "training.fast_dry_run": false,
+  "training.filter_out_shorts": true,
+  "training.force_model_parallel": false,
+  "training.git_diff": "diff --git a/beagle/configs.ini b/beagle/configs.ini\nindex 7e119f9..6c1545a 100644\n--- a/beagle/configs.ini\n+++ b/beagle/configs.ini\n@@ -157,6 +157,11 @@ per_device_train_batch_size = 1\n gradient_accumulation_steps = 8\n max_length = 4096\n \n+[training.@l40s_bs8_ctx2048_sm]\n+per_device_train_batch_size = 2\n+gradient_accumulation_steps = 4\n+max_length = 2048\n+\n [dataset]\n #manual_sample_ids = ['v4PzAY8_0', 'oM7QCY2_0', 'efVCaLN_0']\n manual_sample_ids = []",
+  "training.git_sha1": "0fd5ab514678b181c0602502811571cb9ffb5944",
+  "training.gradient_accumulation_steps": 4,
+  "training.learning_rate": 3e-05,
+  "training.logging_first_step": true,
+  "training.logging_steps": 20,
+  "training.loss_pivot_only": false,
+  "training.loss_reweighting": false,
+  "training.loss_reweighting_use_surrogate": false,
+  "training.lr_scheduler_type": "constant_with_warmup",
+  "training.max_grad_norm": 0.5,
+  "training.max_length": 2048,
+  "training.max_steps": -1,
+  "training.max_tti_wind": 1,
+  "training.model_init_ckpt": null,
+  "training.num_train_epochs": 10,
+  "training.number_sampled_print": "index",
+  "training.optim": "adamw_torch_fused",
+  "training.output_dir": "output/old-force-560",
+  "training.overwrite_output_dir": true,
+  "training.per_device_eval_batch_size": 1,
+  "training.per_device_train_batch_size": 2,
+  "training.project": "beagle",
+  "training.ranking_distill_topk": null,
+  "training.report_to": "wandb",
+  "training.resume_from_checkpoint": false,
+  "training.resume_wandb_runid": null,
+  "training.run_name": "temp_run",
+  "training.save_steps": 500,
+  "training.save_strategy": "steps",
+  "training.save_total_limit": 2,
+  "training.save_vram": true,
+  "training.seed": 42,
+  "training.slow_dry_run": false,
+  "training.tf32": false,
+  "training.topk_w": 1.0,
+  "training.use_eagle_pipeline": false,
+  "training.warmup_steps": 0,
+  "training.world_size": 1
+}

config.json

@@ -0,0 +1,64 @@
+{
+  "_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "architectures": [
+    "LlamaForSpeculativeCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForSpeculativeCausalLM": "modeling_speculative_llama.LlamaForSpeculativeCausalLM"
+  },
+  "beagle_add_noise": true,
+  "beagle_attention_wind": "3",
+  "beagle_ckpt_path": null,
+  "beagle_debug": false,
+  "beagle_decoder_key_remap": {},
+  "beagle_dtype": "torch.float32",
+  "beagle_frozen_targets": [],
+  "beagle_git_diff": "",
+  "beagle_git_sha1": "unknown",
+  "beagle_layer_path": "model.layers",
+  "beagle_lmhead_path": "lm_head",
+  "beagle_load_config_from_model_path": false,
+  "beagle_mask_token": "\u2581blank",
+  "beagle_mask_token_rate": 0,
+  "beagle_model_path": "beagle/models/llama/Llama-2-7b-chat-hf/",
+  "beagle_norm_path": "model.norm",
+  "beagle_only_first_state_distill": false,
+  "beagle_output_dir": "output",
+  "beagle_reuse_layer": null,
+  "beagle_rotary_path": "model.rotary_emb",
+  "beagle_run_name": "temp_run",
+  "beagle_save_loading": true,
+  "beagle_seed": 42,
+  "beagle_strictly_follow_eagle_decoder": false,
+  "beagle_tokenizer_path": "meta-llama/Llama-2-7b-chat-hf",
+  "beagle_use_dyt": false,
+  "beagle_use_fc_eagle": false,
+  "beagle_use_lower_layers": 0,
+  "beagle_use_moe": false,
+  "beagle_use_state_distill": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 0,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "speculative_base_model": "meta-llama/Llama-2-7b-chat-hf",
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3ed0ad203ff0d9a2292f5413b10b491b286756b50e0e6595d50f065d081f564
+size 1858127224

modeling_speculative_llama.py

@@ -0,0 +1,113 @@
+from transformers.models.llama.modeling_llama import *
+from beagle.mixin import *
+from beagle.dyt import *
+from beagle.moe import *
+
+
+class LlamaBeagleAttention(LlamaAttention, BeagleAttentionMixin):
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+
+        bsz, q_len, _ = hidden_states.size()
+        query_states, key_states, value_states = self.qkv_transform(
+            hidden_states, past_key_value, use_cache, position_embeddings, **kwargs)
+
+        ################################################
+        ### everything kept original starting from here
+        ################################################
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attention_mask is not None:  # no matter the length, we just slice it
+            causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+            attn_weights = attn_weights + causal_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+
+        attn_output = attn_output.reshape(bsz, q_len, -1)
+
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class LlamaBeagleDecoderLayer(LlamaDecoderLayer):
+    def __init__(self, config, layer_id):
+        super().__init__(config, layer_id)
+
+        if not config.beagle_use_fc_eagle:
+            delattr(self, 'self_attn')
+            recycle_vram()
+            self.self_attn = LlamaBeagleAttention(
+                config=config, layer_idx=layer_id
+            )
+
+        if config.beagle_use_moe:
+            def init_exp(moe_hidden_size, moe_intermediate_size):
+                config.hidden_size = moe_hidden_size
+                config.intermediate_size = moe_intermediate_size
+                return LlamaMLP(config)
+            from transformers import AutoConfig
+            moe_config = AutoConfig.from_pretrained(
+                "beagle/models/deepseek-moe", trust_remote_code=True)
+            moe_config.hidden_size = config.hidden_size
+            moe_config.moe_intermediate_size = config.intermediate_size
+            moe_config.n_routed_experts = 3
+            moe_config.n_shared_experts = None
+            moe_config.num_experts_per_tok = 1
+            self.mlp = DeepseekMoE(moe_config, init_exp)
+
+        if config.beagle_strictly_follow_eagle_decoder:
+            delattr(self, 'input_layernorm')
+            recycle_vram()
+            self.input_layernorm = nn.Identity()
+
+        elif config.beagle_use_dyt:
+            normalized_shape = self.input_layernorm.weight.shape
+            delattr(self, 'input_layernorm')
+            delattr(self, 'post_attention_layernorm')
+            recycle_vram()
+            self.input_layernorm = DynamicTanhNorm(normalized_shape)
+            self.post_attention_layernorm = DynamicTanhNorm(normalized_shape)
+
+
+class LlamaForSpeculativeCausalLM(LlamaForCausalLM, BeagleMixin):
+    _no_split_modules = ["LlamaDecoderLayer", "LlamaBeagleDecoderLayer"]
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        BeagleMixin.__init__(self, config)
+        self.speculative_decoder = LlamaBeagleDecoderLayer(config, layer_id=0)
+
+        self.post_init()
+
+    def forward(self, *args, **kwargs) -> Union[Tuple[torch.Tensor], BaseModelOutputWithPast]:
+        return self.beagle_forward(*args, **kwargs)

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{% generation %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endgeneration %}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}