Upload folder using huggingface_hub

beeper_rose_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee0f8495abba954d7125d2cb282d32b18465a3fb5d66550716b6e16ed52aa900
+size 111301146

beeper_rose_final.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b8b682beba4d8e3a7ad2a4977bc429c7f83120fd2707879d85cd704030c68a2
+size 111286288

config.json

@@ -0,0 +1,55 @@
+{
+  "name": "Beeper",
+  "variant": "rose",
+  "dataset_name": "roneneldan/TinyStories",
+  "dataset_split": "train[:10%]",
+  "context": 512,
+  "vocab_size": 8192,
+  "add_bos_eos": true,
+  "val_ratio": 0.01,
+  "test_ratio": 0.01,
+  "dim": 512,
+  "n_layers": 6,
+  "n_heads": 8,
+  "mlp_ratio": 4.0,
+  "dropout": 0.0,
+  "resid_dropout": 0.1,
+  "grad_checkpoint": true,
+  "compile_model": false,
+  "batch_size": 32,
+  "grad_accum_steps": 1,
+  "epochs": 3,
+  "lr": 0.0003,
+  "betas": [
+    0.9,
+    0.95
+  ],
+  "weight_decay": 0.1,
+  "warmup_steps": 500,
+  "max_steps": null,
+  "clip_grad": 1.0,
+  "min_lr": 1e-06,
+  "label_smoothing": 0.0,
+  "use_rose": true,
+  "lambda_rose": 0.1,
+  "rose_hidden_pool": "mean",
+  "use_pent": true,
+  "lambda_pent": 0.01,
+  "pent_align_weight": 0.5,
+  "pent_regular_weight": 0.5,
+  "mixed_precision": "bf16",
+  "log_dir": "./runs/beeper",
+  "log_interval": 50,
+  "ckpt_dir": "./beeper_checkpoints",
+  "export_dir": "./beeper_export",
+  "temperature": 0.9,
+  "top_k": 40,
+  "top_p": 0.9,
+  "repetition_penalty": 1.1,
+  "presence_penalty": 0.6,
+  "frequency_penalty": 0.0,
+  "upload_to_hub": true,
+  "hf_repo": "AbstractPhil/beeper-rose-tinystories-6l-512d-ctx512",
+  "tokenizer_path": "beeper.tokenizer.json",
+  "seed": 1337
+}