feat: default dim

Signed-off-by: jupyterjazz <[email protected]>

modeling_xlm_roberta.py

@@ -91,7 +91,7 @@ def create_mixer_cls(config, cross_attn=False, return_residual=False):
     rotary_kwargs = {}
     if config.position_embedding_type == "rotary":
         rotary_kwargs["rotary_emb_dim"] = getattr(
-            config, "rotary_emb_dim", config.hidden_size / 12
+            config, "rotary_emb_dim", config.hidden_size / config.num_attention_heads
         )
         rotary_kwargs["rotary_emb_base"] = getattr(config, "rotary_emb_base", 10000.0)
         rotary_kwargs["rotary_emb_scale_base"] = getattr(