diff --git a/config.yml b/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..f58ac36ee03b47d9281e4bf332993ef24efe16d3 --- /dev/null +++ b/config.yml @@ -0,0 +1,229 @@ +{ + "version": ".unknown.", + "runner": { + "runner_type": "pdsh", + "hostsfile": null, + "hosts": null, + "master_port": 29500, + "master_addr": null, + "script": "src/scaling/transformer/train.py", + "default_gpu_count": 8, + "docker_config": { + "docker_container": null, + "docker_sudo": false, + "docker_mounts": [ + [ + "/mnt/", + "/mnt/" + ] + ] + }, + "use_determined": true + }, + "logger": { + "log_level": "info", + "log_dir": null, + "metrics_ranks": null, + "use_wandb": true, + "wandb_ranks": null, + "wandb_host": "https://api.wandb.ai", + "wandb_team": "aleph-alpha", + "wandb_project": "ng-semantic-embedding", + "wandb_group": "pharia_7B_embedding_all_source_256-2024-10-01-19-28-52", + "wandb_api_key": "d80060c84c496915fcad15283809dbde65087d73", + "use_tensorboard": false, + "tensorboard_ranks": null, + "determined_metrics_ranks": null + }, + "topology": { + "global_rank": 0, + "world_size": 256, + "local_slot": 0, + "model_parallel_size": 1, + "pipe_parallel_size": 1, + "data_parallel_size": 256, + "global_batch_size": 2048, + "micro_batch_size": 8, + "gradient_accumulation_steps": 1, + "pipe_partition_method": "balanced", + "pipe_partition_overwrite": null, + "activation_checkpointing_type": "every_layer", + "sequence_parallel": false + }, + "optimizer": { + "method": "adamw", + "beta1": 0.9, + "beta2": 0.95, + "eps": 1e-15, + "gradient_clipping": 0.0, + "allreduce_bucket_size": 500000000, + "loss_scaler": { + "enable": false, + "initial_scale": 4294967296.0, + "window": 1000, + "hysteresis": 2.0, + "consecutive_hysteresis": false, + "min_scale": 1.0, + "factor": 2.0 + }, + "zero": true, + "zero_save_static": false, + "debug_log": false + }, + "learning_rate_scheduler": { + "learning_rate": 2e-05, + "learning_rate_minimum": 0.0, + "learning_rate_decay_style": "linear", + "learning_rate_decay_iters": 938, + "learning_rate_warmup_steps": 50 + }, + "embedding_learning_rate_scheduler": { + "learning_rate": 0.0, + "learning_rate_minimum": 0.0, + "learning_rate_decay_style": "cosine", + "learning_rate_decay_iters": 0, + "learning_rate_warmup_steps": 0 + }, + "training": { + "weight_decay": 0.0001, + "finetune": true, + "finetunable_parameters": [ + "embed" + ], + "parameters_exclude": [], + "use_separate_lr_on_embeddings": false, + "use_deterministic_torch_algorithms": false, + "loss_function_config": { + "loss_type": "contrastive_loss", + "number_of_hard_negatives": 1, + "use_instructions": true, + "query_side_only": false, + "scale": 50, + "log_verbose_metrics": true + } + }, + "trainer": { + "save_dir": "checkpoints", + "save_interval": 938, + "load_dir": "/scratch/samuel/Pharia-1-LLM-7B-control", + "train_iterations": 938, + "assert_checkpoint_loaded": true, + "load_optimizer_states": false, + "delete_past_optimizer_states": true, + "load_context": false, + "allowed_missing_keys_in_checkpoint": [ + "embed" + ], + "allowed_unexpected_keys_in_checkpoint": null, + "ignore_keys_in_checkpoint": null, + "merge_lora_after_loading_checkpoint": false, + "seed": 42, + "dataloader_num_workers": 0, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": null, + "eval_iterations": 1, + "eval_interval": null, + "separate_file_for_parameters": [ + "adapter_embed" + ] + }, + "profiler": { + "profile_steps": 0, + "profile_start_at_step": 0, + "profiler_output": null + }, + "transformer_architecture": { + "vocab_size": 128000, + "vocab_file": "/scratch/samuel/Pharia-1-LLM-7B-control/vocab.json", + "hidden_size": 4608, + "num_layers": 27, + "num_attention_heads": 36, + "num_local_attention_heads": 0, + "local_attention_window_size": null, + "rotary_embedding_base": 1000000, + "rotary_percentage": 1.0, + "sequence_length": 2048, + "norm_type": "layernorm", + "relative_position_embedding_type": "rotary_complex", + "mlp_type": "default", + "mlp_factor": 4.0, + "attention_bias": true, + "attention_qkv_in_one": false, + "attention_num_kv_heads": 4, + "attention_use_matmul": false, + "mlp_bias": true, + "key_query_norm": false, + "weight_tying": false, + "masked_softmax": { + "kernel": "torch", + "softmax_in_fp32": true, + "scale": 1.0, + "deterministic_flash_attn_bwd": false + }, + "layernorm": { + "optimization_type": "torch", + "layernorm_epsilon": 1e-05 + }, + "precision": "bfloat16", + "dropout_embedding": 0.0, + "dropout_attention_probs": 0.0, + "dropout_after_attention": 0.0, + "dropout_after_mlp": 0.0, + "bitfit_bias_config": null, + "finetunable_token_ids": [], + "image_encoder": false, + "dropout_image_encoder": 0.0, + "softprompt_config": null, + "adapter_config": { + "name": "embed", + "attention_downsampling_factor": 0.25, + "mlp_downsampling_factor": 0.25, + "init_std": 1e-05, + "version": ".unknown." + }, + "lora_config": null, + "embedding_head_config": { + "name": "pooling_only", + "proj_layers": [ + 256 + ], + "pooling": "weighted_mean" + }, + "lm_head": false, + "causal": true + }, + "data": { + "legacy_dataset": false, + "load_mmap_index_to_memory": false, + "use_mmap": false, + "load_data_item_mmap_index_to_memory": false, + "finetuning_dataset": false, + "finetuning_chat_dataset": false, + "finetuning_dataset_memory_map": false, + "embedding_dataset": true, + "embedding_dataset_memory_map": false, + "data_prefixes": [ + "/scratch/samuel/all_source_data_templated/allnli_eng.jsonl", + "/scratch/samuel/all_source_data_templated/hotpot_with_bm25_negatives.jsonl", + "/scratch/samuel/all_source_data_templated/medi_1.jsonl", + "/scratch/samuel/all_source_data_templated/msmarco.jsonl" + ], + "validation_data_prefixes": null, + "blended_dataset": { + "weight_by_num_documents": true, + "weighted_sampler_alpha": 1.0, + "weights": null, + "weight_examples_proportional": false, + "ep_maximum": null, + "ep_temperature": 1.0, + "minimum_dataset_size": 0, + "cache_directory": "/scratch/samuel/blended_dataset_cache", + "shuffle_dataset_indices": false, + "load_dataset_indices_to_memory": false + }, + "only_full_sequences": false, + "allow_incomplete_sequences_every_n": 0 + }, + "determined_experiment_id": 2965, + "determined_trial_id": 86040 +} \ No newline at end of file diff --git a/model_state_layer_0_EmbeddingInput.pt b/model_state_layer_0_EmbeddingInput.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea87cea11e9f82751cea4f720901ce79d5f6e4f5 --- /dev/null +++ b/model_state_layer_0_EmbeddingInput.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3722b0053cc854fcf13733cf9d10317d041d9aae5b65643f697974cc2882efb3 +size 1179649443 diff --git a/model_state_layer_10_TransformerLayer.pt b/model_state_layer_10_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..5042d9ed9ce2d5fab30ac492d31b87b087e07378 --- /dev/null +++ b/model_state_layer_10_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef63496e306c9f3e0112742300dd47b51c31fd2226fd3d7aa5c2adc1a5ffb265 +size 434219902 diff --git a/model_state_layer_10_TransformerLayer_adapter_embed.pt b/model_state_layer_10_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..83b6ea45b87ccd8272e1884a0552ac80f38753d4 --- /dev/null +++ b/model_state_layer_10_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3071455dec4665113216f294ea7cd1af7196380dda917a59c3efb0285b7e7c1f +size 42469872 diff --git a/model_state_layer_11_TransformerLayer.pt b/model_state_layer_11_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..0853b9b37a5ee10e0adb0fb4bfacf0becaf4fc5a --- /dev/null +++ b/model_state_layer_11_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ae21f20ea4e1492bdf2369f7ef91dda55015f81ab4eab9604a9362086d12cb +size 434219902 diff --git a/model_state_layer_11_TransformerLayer_adapter_embed.pt b/model_state_layer_11_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..0250e9cd9d6425dccb2031f731321055b9225786 --- /dev/null +++ b/model_state_layer_11_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55da45ccc291c5dd372ad890c4d050f1d3cbb7f8408a284bbc70ec8128988f11 +size 42469872 diff --git a/model_state_layer_12_TransformerLayer.pt b/model_state_layer_12_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae28d5cebcd9ffd97e46a6ac03b5d47c6e046632 --- /dev/null +++ b/model_state_layer_12_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd902b46a5a353687a2f31e754164e3db18fe72c769a69ed13284bbff224f455 +size 434219902 diff --git a/model_state_layer_12_TransformerLayer_adapter_embed.pt b/model_state_layer_12_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..551a2e252ecb8c69b82e20774163a77642a3d542 --- /dev/null +++ b/model_state_layer_12_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db75b1193e64fe400550066600e0cac3444a7ccada4a8a44c2d13a3c084c1fed +size 42469872 diff --git a/model_state_layer_13_TransformerLayer.pt b/model_state_layer_13_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..32a3d09f18baafe907f4cf45f2a7a0a888c4d521 --- /dev/null +++ b/model_state_layer_13_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664c1b719da7b9218debdc2e9794798ca9d0f22568b138996e182229d79c3444 +size 434219902 diff --git a/model_state_layer_13_TransformerLayer_adapter_embed.pt b/model_state_layer_13_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c9c9403404f97524a5d26649a568223ff350a3e --- /dev/null +++ b/model_state_layer_13_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97f60798bafe4179fda66410bc6ea4d926b4bc10be86c565187a505387ecc9ec +size 42469872 diff --git a/model_state_layer_14_TransformerLayer.pt b/model_state_layer_14_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a808ad135660a3311cc9f1daaa4d11cf09520a36 --- /dev/null +++ b/model_state_layer_14_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef77d20a502e6c8d8908cbd7cb222d04ccea69002e78042a57ac11a8a0b4362 +size 434219902 diff --git a/model_state_layer_14_TransformerLayer_adapter_embed.pt b/model_state_layer_14_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..89379c33ef9b39c9320fecd670da2a5ad71147e3 --- /dev/null +++ b/model_state_layer_14_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2416a2626b40f412a01b9177bbc4b295856663430abbf4c1f3770f2bb2c33170 +size 42469872 diff --git a/model_state_layer_15_TransformerLayer.pt b/model_state_layer_15_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..e293a0d4cc7449d6a6491b3bd0afcc5ef38e8e1b --- /dev/null +++ b/model_state_layer_15_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7de81299e97471d03af358c730a40238867eb6032c93c4cebcba714f2db5d47 +size 434219902 diff --git a/model_state_layer_15_TransformerLayer_adapter_embed.pt b/model_state_layer_15_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec2e9f8222a204a39e87e0e9dc57217c5dcd04d0 --- /dev/null +++ b/model_state_layer_15_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00c68832f2ed909e3663b9317413b6f08e37e3326553b63b4d87b5f99b25e05e +size 42469872 diff --git a/model_state_layer_16_TransformerLayer.pt b/model_state_layer_16_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d24e3747489f11c3189365e3ee2740626c95f81b --- /dev/null +++ b/model_state_layer_16_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e866357f6d06e90c4a4785e7b0cc2056b50361904168d54fb6fc314ef9f4639b +size 434219902 diff --git a/model_state_layer_16_TransformerLayer_adapter_embed.pt b/model_state_layer_16_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c35420c019277ef3b84a7230fb9cac96f3e7db5 --- /dev/null +++ b/model_state_layer_16_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00f14bd76f5e18b3d790ad6b15b49273dadad3b6682cdb39c1690505e01a8b6f +size 42469872 diff --git a/model_state_layer_17_TransformerLayer.pt b/model_state_layer_17_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..24957933c84687a8638e1f3b6f5e0ca9f67e8fa5 --- /dev/null +++ b/model_state_layer_17_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b4c380ed92cd055a067919127a7365bcd66f18fa8dfa8d38653018ac0f597c +size 434219902 diff --git a/model_state_layer_17_TransformerLayer_adapter_embed.pt b/model_state_layer_17_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cabb8a542b149b128ac2fed5685bef1675d1f38 --- /dev/null +++ b/model_state_layer_17_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c6fd7770f7910dfb57aee214b30d8be581c1621c83eabbdab0ae76421d244ac +size 42469872 diff --git a/model_state_layer_18_TransformerLayer.pt b/model_state_layer_18_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fcc5bb30f00dbdf86e871cf9608f065e26bf4bb --- /dev/null +++ b/model_state_layer_18_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c34494e568a417b9adae97acb4d7c76012a9903a3e3d596e68d0020f1af6d2b0 +size 434219902 diff --git a/model_state_layer_18_TransformerLayer_adapter_embed.pt b/model_state_layer_18_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..483fba6abc85d066e37cb7be2cdc86add4bef368 --- /dev/null +++ b/model_state_layer_18_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d8a4cdc5f694d1607a2c0824f49a5a43e36917ba31e7cbdbaecd17179fa85e9 +size 42469872 diff --git a/model_state_layer_19_TransformerLayer.pt b/model_state_layer_19_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff2ef30db8a509e360791ce18766316ec9d7b02c --- /dev/null +++ b/model_state_layer_19_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2807c758304a165e6140b8942fa9eab390814f81c26fd327628822ce004ebfe7 +size 434219902 diff --git a/model_state_layer_19_TransformerLayer_adapter_embed.pt b/model_state_layer_19_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..984a4430b28ab038526c705b991d95e39f5ac81a --- /dev/null +++ b/model_state_layer_19_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88ff78dc09729fe2a641b665b06bbe51537bf12f702269759abbc94335c66a30 +size 42469872 diff --git a/model_state_layer_1_TransformerLayer.pt b/model_state_layer_1_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce52259f29060bc7b07d273aa0af7bf36ebb170f --- /dev/null +++ b/model_state_layer_1_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2679a5ca251090f3a75c87b7add5f42e975a213ec8aaee90bde27da688a3d1 +size 434219882 diff --git a/model_state_layer_1_TransformerLayer_adapter_embed.pt b/model_state_layer_1_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ce011581cea7659439c218cd802ace631f82994 --- /dev/null +++ b/model_state_layer_1_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e21ae05b1e2d5e791b3aaeacca8f47668b2f93ff4c1c046a765be91aae26f95 +size 42469864 diff --git a/model_state_layer_20_TransformerLayer.pt b/model_state_layer_20_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..68e2526d4816fddfddcd92cca53fb5e135ccef2f --- /dev/null +++ b/model_state_layer_20_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197bec6a16feca3ac18653521b975673a56a071bf59ae3a9bfce737648bda775 +size 434219902 diff --git a/model_state_layer_20_TransformerLayer_adapter_embed.pt b/model_state_layer_20_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff46467df066877a96aea685f80242f5486b91c7 --- /dev/null +++ b/model_state_layer_20_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b5a318c6ecabdf8019452d2d74e50f9ae4ad99ef1d034a180133503afcd190 +size 42469872 diff --git a/model_state_layer_21_TransformerLayer.pt b/model_state_layer_21_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..40caf8699db267e1e337fcb42296c2b5059b9f42 --- /dev/null +++ b/model_state_layer_21_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31eb0f91b2b326f7b869a2dc5dae07be2a6831efb72d3193d1d6d457a2d6fcff +size 434219902 diff --git a/model_state_layer_21_TransformerLayer_adapter_embed.pt b/model_state_layer_21_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..affaef699a639578ed58721d2fa21d102ee045c6 --- /dev/null +++ b/model_state_layer_21_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02d4de53679a0261925cc092fa0be06ba6d663735f037cbbb8f506a61baa88c2 +size 42469872 diff --git a/model_state_layer_22_TransformerLayer.pt b/model_state_layer_22_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..e72e2a701eaaae21c5d99d8aabf8287988bdaeb8 --- /dev/null +++ b/model_state_layer_22_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8deac3ee07d0285db53ce915d9315491287708edec05a9f16e17e3b0c2395ceb +size 434219902 diff --git a/model_state_layer_22_TransformerLayer_adapter_embed.pt b/model_state_layer_22_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad11395603d99444ad154bfe84b15951a04094bd --- /dev/null +++ b/model_state_layer_22_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b85af03d8a9ed9c6bf66619563e5cd26b8894b30a0301cb8cd0e45df52a7a0f +size 42469872 diff --git a/model_state_layer_23_TransformerLayer.pt b/model_state_layer_23_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..1abddab75b6a17185738047c473cc2766584f0a0 --- /dev/null +++ b/model_state_layer_23_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a4de46c226702e34dbf4d5cd8ead1d366f7cd94b5c5a0f9481dc0f5d502214 +size 434219902 diff --git a/model_state_layer_23_TransformerLayer_adapter_embed.pt b/model_state_layer_23_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..3054e6c5246cf6ccd99b0e6c91fd707bd45d57a1 --- /dev/null +++ b/model_state_layer_23_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb864841645a7ae04dc3867d1fe26a307be28da55a787ccb1c8ab044352c13fd +size 42469872 diff --git a/model_state_layer_24_TransformerLayer.pt b/model_state_layer_24_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..04e582a9e1171fb3671932f90bfd71aad0be9e63 --- /dev/null +++ b/model_state_layer_24_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:290bde2e41ce42a45bef1ae6977cbc4f8add6ca43f1493fedeac5d17b7497248 +size 434219902 diff --git a/model_state_layer_24_TransformerLayer_adapter_embed.pt b/model_state_layer_24_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c98d76f82baff71975bcb47e5ea6b3b5c3e51fa --- /dev/null +++ b/model_state_layer_24_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a671aed521d6783c25d80f43a9530599d2a15d48f6f2d98b3eceba8294a94c77 +size 42469872 diff --git a/model_state_layer_25_TransformerLayer.pt b/model_state_layer_25_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..30346d78227e07061565357f4eb37fcf8f3bb601 --- /dev/null +++ b/model_state_layer_25_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45968daef6d99223322950b1251b22b8462b8d95fd355edb67421feaef60fc49 +size 434219902 diff --git a/model_state_layer_25_TransformerLayer_adapter_embed.pt b/model_state_layer_25_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c8b0a8c9db7cdd3a0c66947914d2ed05dcdf77b --- /dev/null +++ b/model_state_layer_25_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:487b308e165857cdbd6e52deb2df0239372dad547befb07bdd0f7ff7662d0314 +size 42469872 diff --git a/model_state_layer_26_TransformerLayer.pt b/model_state_layer_26_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..64b4fd50249d622cc92a6f478faf910956f40c6b --- /dev/null +++ b/model_state_layer_26_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47274e293886f267b6ef434f55b4087c3400132b40e33ea732762fdcf544648d +size 434219902 diff --git a/model_state_layer_26_TransformerLayer_adapter_embed.pt b/model_state_layer_26_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..7845a9541b8e276b359d6727faf44ab68678bb6a --- /dev/null +++ b/model_state_layer_26_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ac9f37f0eb3b2aa48338ae92482917cede328234aac94cb5acd4e6e0535826a +size 42469872 diff --git a/model_state_layer_27_TransformerLayer.pt b/model_state_layer_27_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0bc0cbe5ab48e1b9a2c0f113ba7dc41246de07b --- /dev/null +++ b/model_state_layer_27_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f25827475fe28d36d12002c77f4e337f01af1d38524d74163c804f387ebcad4 +size 434219902 diff --git a/model_state_layer_27_TransformerLayer_adapter_embed.pt b/model_state_layer_27_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd88caaccba80fc4407d0886a32477969cce94c --- /dev/null +++ b/model_state_layer_27_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8619d0cd488dc2f267dc84bf9a85094894e93e6067b623223442bc706ed88f52 +size 42469872 diff --git a/model_state_layer_28_LayerNormWrapper.pt b/model_state_layer_28_LayerNormWrapper.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a469bd6f3924197d4f367475959e24000e4d2b2 --- /dev/null +++ b/model_state_layer_28_LayerNormWrapper.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd1372d2ae21d66d644fb98394d0868cc9aeff84ccc20619a9bbf95eeebcd62 +size 20172 diff --git a/model_state_layer_29_TransformerEmbeddingHead.pt b/model_state_layer_29_TransformerEmbeddingHead.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1a9c8edde50a65104ed6bddb95f0a66955e4fb7 --- /dev/null +++ b/model_state_layer_29_TransformerEmbeddingHead.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84863b3c4b23eb9d18224b2c4275f2998fd48901ce5361bfe4cd5618598c2386 +size 2360794 diff --git a/model_state_layer_2_TransformerLayer.pt b/model_state_layer_2_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..71a16400809dba160effbcd92cd130583290da60 --- /dev/null +++ b/model_state_layer_2_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15decebd9fe0f57b1d69766411ab106d37f354b7dc1b6b280d48baca79626756 +size 434219882 diff --git a/model_state_layer_2_TransformerLayer_adapter_embed.pt b/model_state_layer_2_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..e014cc5e35f56484787b4d795504d203a04390ca --- /dev/null +++ b/model_state_layer_2_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cecf3037992e2b5d443fae5d312368032dc3e3191bf4c466c69cd90380b4bc0 +size 42469864 diff --git a/model_state_layer_3_TransformerLayer.pt b/model_state_layer_3_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d97793681a33ce0b588e48912bb464419cd08362 --- /dev/null +++ b/model_state_layer_3_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9465e9dcb05d29f2c9e320c53e486825e05341a6c5097fcefa86ca1384f495 +size 434219882 diff --git a/model_state_layer_3_TransformerLayer_adapter_embed.pt b/model_state_layer_3_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..23dfaaaf1a5987325b07fb87e967e1d1695e4c9f --- /dev/null +++ b/model_state_layer_3_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbe555db4a265531edfd537af40a1c62de8a0b363aeb310af820aedc8eac1a3c +size 42469864 diff --git a/model_state_layer_4_TransformerLayer.pt b/model_state_layer_4_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..6efb5abcda3da450af1b11310a96859870cf69f3 --- /dev/null +++ b/model_state_layer_4_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f8449ff2999f09a5ae6a621104c5d8929bb9f6969315cf4210bfe327dd0bc72 +size 434219882 diff --git a/model_state_layer_4_TransformerLayer_adapter_embed.pt b/model_state_layer_4_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6a8a26df89090b19d8bfe3d923d79e839fd2bb9 --- /dev/null +++ b/model_state_layer_4_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36242c30fb06de70fe7a674fbf2b8aef6291f837847e90b365d58f9314ca2345 +size 42469864 diff --git a/model_state_layer_5_TransformerLayer.pt b/model_state_layer_5_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..550ddfd240ffe479b3d7269eac12dd6a2888240c --- /dev/null +++ b/model_state_layer_5_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a4b77dab79cedd475c6335c22188eefd71bcb07cf39a11d8f80deb7b678f2ac +size 434219882 diff --git a/model_state_layer_5_TransformerLayer_adapter_embed.pt b/model_state_layer_5_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d83982bcec48fad657cdce6712d50bab7858f1d --- /dev/null +++ b/model_state_layer_5_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25e9e8c698d343cfd55e21e113ade823149883e63fe38b0c550e061c28acff54 +size 42469864 diff --git a/model_state_layer_6_TransformerLayer.pt b/model_state_layer_6_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4efe86fc81f0f0cc897838eef7437676dee5959 --- /dev/null +++ b/model_state_layer_6_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca257c7e6088e46d529b4b8be5c185d1767f3f5916db5e0e799942af25c9c1b0 +size 434219882 diff --git a/model_state_layer_6_TransformerLayer_adapter_embed.pt b/model_state_layer_6_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f5c8db0644ec8d955a61348207aaf0e7ff15907 --- /dev/null +++ b/model_state_layer_6_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4cc3ca012a2687165f521eba640d5287eee1d02dbb77743dbc3f773fcfbb82d +size 42469864 diff --git a/model_state_layer_7_TransformerLayer.pt b/model_state_layer_7_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..04bd96efc4c76d63a8bc5f31645451a0aff75d5c --- /dev/null +++ b/model_state_layer_7_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277bcf92bcb4aa993c92b366cc08130d58154b5d5b752104ee3bec9a7f07d651 +size 434219882 diff --git a/model_state_layer_7_TransformerLayer_adapter_embed.pt b/model_state_layer_7_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..03c6c15b06ecd6f56ca7dbbd7b17f20971352943 --- /dev/null +++ b/model_state_layer_7_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08e06d44fd14d0e883ed81839e077caec91cb20d03bf0bba281d2a54d6251836 +size 42469864 diff --git a/model_state_layer_8_TransformerLayer.pt b/model_state_layer_8_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..00865abb5c3e95376e0d6c2e00cb6b48f302cfb3 --- /dev/null +++ b/model_state_layer_8_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4915427440c4df2ded342564af754874194372897c4cb0d073643e77a415cf88 +size 434219882 diff --git a/model_state_layer_8_TransformerLayer_adapter_embed.pt b/model_state_layer_8_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a9d259134d40b086efe497a78c990e6721b4a2b --- /dev/null +++ b/model_state_layer_8_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a22e6be93afc0b6e30a3439c59a483f9d0942e2d1a905b05f2233ca760dc3043 +size 42469864 diff --git a/model_state_layer_9_TransformerLayer.pt b/model_state_layer_9_TransformerLayer.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0578a4de24b59348970482a5bcbc9734a8d618d --- /dev/null +++ b/model_state_layer_9_TransformerLayer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf38f1873cbf8fdbd811849c9cbe2c63707d7098c70a8c8c8b5159f0a432b23 +size 434219882 diff --git a/model_state_layer_9_TransformerLayer_adapter_embed.pt b/model_state_layer_9_TransformerLayer_adapter_embed.pt new file mode 100644 index 0000000000000000000000000000000000000000..127ce4be0e4486fb1dec9324ef8310995345de3a --- /dev/null +++ b/model_state_layer_9_TransformerLayer_adapter_embed.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d076d35bc6eb28f3f31723dcfc7f448a87f90821004c1e999d091c3b7d5b922d +size 42469864