mode: pt
device: gpu
precision: bf16
eval_only: false
predict_only: false
seed: 80085
model:
  klass: local_t5
  name: pszemraj/tFINE-base-65kBPE-FLAN
  overwrite:
    dropout_rate: 0.0
  add_config:
    is_bf16: false
  checkpoint_path: ''
  random_init: true
  compile: true
tokenizer:
  name: BEE-spoke-data/slimpajama_tok-48128-BPE-forT5
data:
  input_length: 1024
  mlm_probability: 0.15
  mean_noise_span_length: 3.0
  num_workers: 8
optim:
  name: adamwscale
  base_lr: 0.008
  batch_size: 120
  total_steps: 80000
  epochs: -1
  warmup_steps: 10000
  lr_scheduler: cosine
  weight_decay: 0.0001
  grad_clip: 1.0
  grad_acc: 24
  final_cosine: 1.0e-05
eval:
  every_steps: 100000
  steps: 500
checkpoint:
  every_steps: 5000
logging:
  neptune: false
  neptune_creds:
    project: null
    api_token: null
    tags: ''
  every_steps: 50
  grad_l2: true
  weights_l2: true