config.yaml

config: /root/in-context-learning-GLM/src/glm_configs/joint_training.yaml
model:
  family: gpt2
  n_dims: 10
  n_embd: 256
  n_head: 8
  n_layer: 12
  n_positions: 40
out_dir: gs://glm_weights/a648a15d-4896-4e95-8a8b-7150ff5ac620
test_run: false
training:
  batch_size: 256
  curriculum:
    dims:
      end: 10
      inc: 2
      interval: 2000
      start: 10
    points:
      end: 40
      inc: 8
      interval: 2000
      start: 40
  data: gaussian
  keep_every_steps: 10
  learning_rate: 0.00025
  num_tasks: 10000
  num_training_examples: 10000
  resume_id: null
  save_every_steps: 10
  task: GLM
  task_kwargs:
    function_type:
    - neg_binomial
    - poisson
    scaling: 0.32
  train_steps: 20000
wandb:
  entity: in-context
  log_every_steps: 10
  name: null
  notes: ''
  project: in-context-training