togethercomputer
/

m2-bert-80M-32k-retrieval

Sentence Similarity

text-classification

Model card Files Files and versions Community

juewang commited on Nov 4, 2023

Commit

a599b20

•

1 Parent(s): 5059123

Update config.yaml

Files changed (1) hide show

config.yaml +3 -3

config.yaml CHANGED Viewed

@@ -2,7 +2,7 @@
 # Please resolve any `null` fields before launching!
 precision: amp_bf16
-max_seq_len: 8192
 # Tokenizer for dataset creation
 tokenizer_name: bert-base-uncased
@@ -16,10 +16,10 @@ model:
     num_attention_heads: 12
     num_hidden_layers: 12
     attention_probs_dropout_prob: 0.0
-    max_position_embeddings: 8192
     monarch_mixer_sequence_mixing: True
-    long_conv_l_max: 8192
     long_conv_kernel_learning_rate: 1e-3
     hyena_lr_pos_emb: 1e-5
     hyena_w: 10

 # Please resolve any `null` fields before launching!
 precision: amp_bf16
+max_seq_len: 32768
 # Tokenizer for dataset creation
 tokenizer_name: bert-base-uncased
     num_attention_heads: 12
     num_hidden_layers: 12
     attention_probs_dropout_prob: 0.0
+    max_position_embeddings: 32768
     monarch_mixer_sequence_mixing: True
+    long_conv_l_max: 32768
     long_conv_kernel_learning_rate: 1e-3
     hyena_lr_pos_emb: 1e-5
     hyena_w: 10