Upload folder using huggingface_hub

README.md

@@ -0,0 +1,88 @@
+## Experiment Configuration
+```yaml
+callbacks:
+  grad_accum:
+    _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
+    scheduling:
+      0: 2
+  grad_norm:
+    _target_: src.callbacks.grad_norm.GradNorm
+    check_clipping: false
+    group_separator: /
+    histogram_freq: null
+    log_weight_distribution: false
+    norm_type: 2
+    only_total: true
+  lr_monitor:
+    _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
+  model_checkpoint:
+    _target_: src.callbacks.model_checkpoint.ModelCheckpoint
+    dirpath: .checkpoints
+    enable_version_counter: false
+    every_n_train_steps: 2000
+    filename: '{step}'
+    save_initial_checkpoint: true
+    save_last: link
+    save_top_k: -1
+    verbose: true
+  speed_monitor:
+    _target_: src.callbacks.speed_monitor.SpeedMonitor
+data:
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+dataset: minipile
+loggers:
+  tensorboard:
+    _target_: src.loggers.TensorBoardLogger
+    name: ''
+    save_dir: ./
+    version: null
+model: smol_llama-370M-tied
+optim:
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+out_parent_folder: model_train
+pwd: .
+resume_from_checkpoint: .checkpoints/last.ckpt
+run_folder: .
+save_initial_checkpoint: true
+seed: 42
+tok_name: bpe32000minipile
+tok_path: ./outputs/tokenizers/bpe32000minipile
+torch_compile: true
+train_data_path: ./data/minipile/bpe32000minipile/train
+trainer:
+  accelerator: gpu
+  deterministic: false
+  devices: 4
+  enable_progress_bar: true
+  fast_dev_run: false
+  gradient_clip_algorithm: norm
+  gradient_clip_val: 1.0
+  limit_val_batches: 500
+  log_every_n_steps: 1
+  max_steps: 50000
+  precision: bf16-true
+  val_check_interval: 2000
+val_data_path: ./data/minipile/bpe32000minipile/validation
+```

hparams.yaml

@@ -0,0 +1,85 @@
+loggers:
+  tensorboard:
+    _target_: src.loggers.TensorBoardLogger
+    save_dir: ./
+    name: ''
+    version: null
+callbacks:
+  lr_monitor:
+    _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
+  grad_norm:
+    _target_: src.callbacks.grad_norm.GradNorm
+    norm_type: 2
+    group_separator: /
+    histogram_freq: null
+    check_clipping: false
+    log_weight_distribution: false
+    only_total: true
+  speed_monitor:
+    _target_: src.callbacks.speed_monitor.SpeedMonitor
+  grad_accum:
+    _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
+    scheduling:
+      0: 2
+  model_checkpoint:
+    _target_: src.callbacks.model_checkpoint.ModelCheckpoint
+    dirpath: .checkpoints
+    filename: '{step}'
+    enable_version_counter: false
+    every_n_train_steps: 2000
+    save_top_k: -1
+    save_last: link
+    verbose: true
+    save_initial_checkpoint: true
+tok_path: /home/pl487/rds/hpc-work/rdd/outputs/tokenizers/bpe32000minipile
+run_folder: .
+out_parent_folder: model_train
+tok_name: bpe32000minipile
+dataset: minipile
+pwd: /home/pl487/rds/hpc-work/rdd
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation
+model: smol_llama-370M-tied
+resume_from_checkpoint: .checkpoints/last.ckpt
+save_initial_checkpoint: true
+seed: 42
+torch_compile: true
+data:
+  batch_size: 16
+  eval_batch_size: 128
+  shuffle: true
+  drop_last: false
+  num_workers: 12
+  pin_memory: true
+  persistent_workers: false
+  prefetch_factor: 2
+  multiprocessing_context: null
+optim:
+  optim_name: adamw
+  lr: 0.0006
+  weight_decay: 0.1
+  optim_kwargs:
+    fused: true
+    eps: 1.0e-08
+    betas:
+    - 0.9
+    - 0.95
+  scheduler_name: warmup_stable_decay
+  num_warmup_steps: 2000
+  scheduler_kwargs:
+    num_stable_steps: 46000
+    num_decay_steps: 2000
+    min_lr_ratio: 0.01
+trainer:
+  accelerator: gpu
+  devices: 4
+  precision: bf16-true
+  deterministic: false
+  log_every_n_steps: 1
+  enable_progress_bar: true
+  fast_dev_run: false
+  gradient_clip_val: 1.0
+  gradient_clip_algorithm: norm
+  val_check_interval: 2000
+  max_steps: 50000
+  limit_val_batches: 500

tb_logs.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90d6e44b3588e2535b0dd5bf75ddc43357ba073794194f0ff36941f39b508799
+size 410705

version_0/events.out.tfevents.1740076201.gpu-q-64.450716.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b39d3646e730a9824daed4ab4b6216a3994e6ea194acfdf469bd01f36ce0ce2a
+size 6989694

version_0/hparams.yaml

@@ -0,0 +1,31 @@
+dataloader_config: !!python/object:src.datamodule.DataloaderConfig
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+eod_token_id: 0
+max_position_embeddings: 2048
+optim_config: !!python/object:src.module.OptimCofig
+  keller_kwargs: {}
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation

version_1/events.out.tfevents.1740342798.gpu-q-40.305079.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b8e68593b07d73d30da7af7aabc8d9a22e7a0cc177626b57619b9374d91ea73
+size 7063299

version_1/hparams.yaml

@@ -0,0 +1,31 @@
+dataloader_config: !!python/object:src.datamodule.DataloaderConfig
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+eod_token_id: 0
+max_position_embeddings: 2048
+optim_config: !!python/object:src.module.OptimCofig
+  keller_kwargs: {}
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation

version_2/events.out.tfevents.1740418997.gpu-q-26.301883.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d01a4b6c9ba46639cca7c60733a2e279acd5fed5724370410a604fc7df09df74
+size 6896780

version_2/hparams.yaml

@@ -0,0 +1,31 @@
+dataloader_config: !!python/object:src.datamodule.DataloaderConfig
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+eod_token_id: 0
+max_position_embeddings: 2048
+optim_config: !!python/object:src.module.OptimCofig
+  keller_kwargs: {}
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation

version_3/events.out.tfevents.1740465311.gpu-q-14.26324.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858cf32191e45d0cf0f26b6f13c070302b2684b7b7bbc481578160a0db2acd2e
+size 6961792

version_3/hparams.yaml

@@ -0,0 +1,31 @@
+dataloader_config: !!python/object:src.datamodule.DataloaderConfig
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+eod_token_id: 0
+max_position_embeddings: 2048
+optim_config: !!python/object:src.module.OptimCofig
+  keller_kwargs: {}
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation

version_4/events.out.tfevents.1740520324.gpu-q-75.43935.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34b01ed42f078ce361fcfd32424a1d3a9962d7302b2b2417a9b2e8140c51f8f7
+size 5975203

version_4/hparams.yaml

@@ -0,0 +1,31 @@
+dataloader_config: !!python/object:src.datamodule.DataloaderConfig
+  batch_size: 16
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+eod_token_id: 0
+max_position_embeddings: 2048
+optim_config: !!python/object:src.module.OptimCofig
+  keller_kwargs: {}
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 2000
+    num_stable_steps: 46000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.1
+train_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/train
+val_data_path: /home/pl487/rds/hpc-work/rdd/data/minipile/bpe32000minipile/validation