{ "_sliding_window_pattern": 6, "architectures": [ "Gemma3ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "attn_logit_softcapping": null, "bos_token_id": 2, "eos_token_id": 106, "final_logit_softcapping": null, "head_dim": 256, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 640, "initializer_range": 0.02, "intermediate_size": 2048, "layer_types": [ "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "sliding_attention", "full_attention" ], "max_position_embeddings": 32768, "model_type": "gemma3_text", "num_attention_heads": 4, "num_hidden_layers": 18, "num_key_value_heads": 1, "pad_token_id": 0, "quantization_config": { "include_input_output_embeddings": false, "modules_to_not_convert": null, "quant_method": "torchao", "quant_type": { "default": { "_data": { "activation_dtype": { "_data": "float8_e4m3fn", "_type": "torch.dtype" }, "activation_value_lb": null, "activation_value_ub": null, "granularity": [ { "_data": {}, "_type": "PerRow", "_version": 1 }, { "_data": {}, "_type": "PerRow", "_version": 1 } ], "kernel_preference": { "_data": "AUTO", "_type": "KernelPreference" }, "mm_config": { "_data": { "emulate": false, "pad_inner_dim": false, "use_fast_accum": true }, "_type": "Float8MMConfig", "_version": 1 }, "set_inductor_config": true, "weight_dtype": { "_data": "float8_e4m3fn", "_type": "torch.dtype" } }, "_type": "Float8DynamicActivationFloat8WeightConfig", "_version": 2 } }, "quant_type_kwargs": {}, "untie_embedding_weights": false }, "query_pre_attn_scalar": 256, "rms_norm_eps": 1e-06, "rope_local_base_freq": 10000.0, "rope_scaling": null, "rope_theta": 1000000.0, "sliding_window": 512, "torch_dtype": "bfloat16", "transformers_version": "4.55.2", "unsloth_fixed": true, "use_bidirectional_attention": false, "use_cache": true, "vocab_size": 262144 }