Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.009882118,
+  "intermediate_size": 14336,
+  "layer_norm_eps": null,
+  "max_position_embeddings": 32768,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "resid_pdrop": 0.1,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.2",
+  "use_cache": true,
+  "vocab_size": 155136
+}

cruise_cli.yaml

@@ -0,0 +1,444 @@
+trainer:
+  default_root_dir: null
+  default_hdfs_dir: hdfs://haruna/home/byte_data_seed/hdd_hldy/user/yuyu.zhang/seekpath/P61_D6_8B_8M_tp2_stage2_H800_code
+  logger:
+  - tracking
+  - console
+  log_every_n_steps: 50
+  benchmark: false
+  enable_speedmonitor: true
+  stats_speedmonitor: false
+  enable_versions: false
+  detect_anomaly: false
+  deterministic: false
+  accelerator: gpu
+  accelerator_kwargs:
+    mega_config: null
+  precision: bf16
+  max_epochs: 1
+  max_steps: -1
+  limit_train_batches: null
+  limit_val_batches: null
+  limit_test_batches: null
+  static_sync_limit_val: false
+  sync_batchnorm: false
+  sync_fit_metrics: null
+  val_check_interval:
+  - 20000000
+  save_before_val: false
+  accumulate_grad_batches: null
+  gradient_clip_val: 1.0
+  max_grad_clip: 0.0
+  seed: null
+  summarize_model_depth: 0
+  resume_ckpt_path: auto
+  frozen_ckpt_path: null
+  resume_strict: true
+  resume_optimizer: true
+  resume_metadata: true
+  resume_loader_state: false
+  callbacks: null
+  enable_checkpoint:
+  - 1
+  - 10000
+  checkpoint_monitor: step
+  checkpoint_mode: max
+  dataloader_timeout: -1
+  dataloader_retry_limit: 100
+  dataloader_retry_persistent_limit: 5
+  find_unused_parameters: false
+  project_name: seekpath_v3
+  experiment_name: P61_D6_8B_npu_8M_tp2_H800stage1_code
+  enable_trace: false
+  reload_dataloaders_every_n_epochs: -1
+  strategy: megatron
+  enable_qat: false
+  no_quant_module: []
+  enable_ptq: true
+  qat_kwargs: {}
+  optimizer_kwargs:
+    optimizer:
+      type: adam
+      params:
+        lr: 3.0e-05
+        betas:
+        - 0.9
+        - 0.95
+        eps: 1.0e-08
+        weight_decay: 0.1
+        bias_correction: true
+        adam_w_mode: true
+        momentum: 0.9
+        lr_mult_keys: []
+        no_weight_decay_keys: []
+        weight_decay_keys: []
+        lr_mult_start_epoch: 0
+        lr_mult: 1.0
+    scheduler:
+      type: megatron.optimizer_param_schedule.OptimizerParamScheduler
+      total_steps_param_name: num_training_steps
+      warmup_steps_param_name: num_warmup_steps
+      interval: step
+      params:
+        warmup_step_rate: 0.002
+        lr_end: 0.1
+        lr_decay_style: constant
+        lr_decay_rate: 1.0
+  grad_norm_layers: []
+  checkpoint_kwargs:
+    verbose: false
+    save_last: false
+    save_weights_only: false
+    every_n_train_steps: -1
+    every_n_seconds: -1
+    save_best: false
+    storage:
+      enable_shm_download: false
+      enable_shm_upload: false
+      download_thread_num: 16
+      upload_thread_num: 1
+  enable_save_checkpoint_async: true
+  enable_profiler: false
+  profiler_schedule_kwargs:
+    wait: 50
+    warmup: 3
+    active: 3
+    repeat: 1
+  profile_all_ranks: false
+  enable_bsdp: false
+  bsdp_num_prefetch: 64
+  keep_frozen_weights: true
+  val_reduce_fn: {}
+  experiment_id: null
+  enable_omnistore: false
+model:
+  network:
+    hidden_size: 4096
+    n_embed: 4096
+    n_inner: 14336
+    n_head: 32
+    n_layer: 32
+    vocab_size: 155136
+    max_position_embeddings: 32768
+    cross_entropy_spilt_num: 1
+    layer_norm_epsilon: 1.0e-05
+    activation_function: gelu_new
+    resid_pdrop: 0.1
+    embd_pdrop: 0.0
+    attn_pdrop: 0.1
+    scale_attn_weights: true
+    scale_attn_by_inverse_layer_idx: false
+    reorder_and_upcast_attn: false
+    initializer_range: 0.009882118
+    gradient_checkpointing: false
+    gradient_checkpointing_ln: false
+    gradient_checkpointing_mlp: false
+    gradient_checkpointing_start_layers: 0
+    tie_weight: false
+    pad_idx: 1
+    use_ft_flash_attn: false
+    use_ft_linear: false
+    use_ft_layernorm: false
+    use_xperf_rotary: false
+    use_rmpad: true
+    fuse_gelu_gemm: false
+    pad_output: false
+    position_embeddings_type: rope
+    skip_n_iters: -1
+    n_shared_qhead: 4
+    num_q_heads: -1
+    num_kv_heads: -1
+    head_dim: -1
+    kv_mirror_layers: []
+    kv_mirror_imitated_layers: []
+    residual_post_ln_layers: []
+    hyperconnection_rate: -1
+    repeat_kv_heads: true
+    sparse_attention_window_size:
+    - -1
+    use_query_swiglu: false
+    query_swiglu_inner_dim: 8192
+    force_mem_efficient_layers:
+    - -1
+    noop_transformer_layers: []
+    dense_ffn_layers: []
+    dense_ffn_type: swiglu
+    dense_ffn_inner_dim: -1
+    moe_expert_type: exp-xelego
+    moe_gate_type: caplog-lego
+    moe_gate_metric_type: lego
+    moe_expert_exp_level: 4
+    moe_expert_exp_first_dim_factor: 1.0
+    moe_expert_exp_first_num: 2
+    moe_topk: 5
+    moe_num_expert: 0
+    moe_expert_eq_dim_factor: 0.25
+    moe_backend: default
+    moe_overlap_recomp_grad_comm: false
+    moe_expert_op_version: V1
+    moe_aux_loss_weight: 0.001
+    moe_gate_dropout: 0.0
+    moe_use_balance: false
+    moe_expert_group_capacity: 1.0
+    moe_expert_group_balance_loss_weight: 0.0
+    moe_expert_groups_in_ep_rank: 1
+    moe_enable_warmup: false
+    moe_swiglu_fc1_2_init_scale: 1.0
+    janus_use_big_op: false
+    janus_big_op_version: V1
+    janus_big_op_attn_grad_accum_fusion: true
+    convert_gate_to_fp32: false
+    moe_enable_ema_update: 1
+    query_head_scale_factor: 1
+    value_moe_num_expert: 0
+    value_moe_qkv_topk: 4
+    value_moe_qkv_times: 1
+    value_moe_is_repeat: true
+    value_moe_expert_type: linear-lego
+    moe_pr_scale_factor: 1.0
+    moe_pr_expert_type: disabled
+    value_moe_gate_type: default-lego
+    value_moe_gate_metric_type: default
+    lora_rank: 0
+    save_mixed_ckpt_in_shards: false
+    save_mixed_model_states_freq: final
+    cont_train_mode: default
+    fuse_lora_weight: true
+    rope_mode: default
+    rope_scale: 1
+    rope_base: 500000.0
+    rope_cut: false
+    rope_cut_head_dim: 0
+    rope_force_fp32: false
+    sparse_attention_window_scale: 1
+    sparse_attention_global_window_size:
+    - 0
+    use_attention_bias: false
+    layer_norm_type: rmsnorm_torch
+    exact_token_as_loss_denominator: false
+    use_key_layernorm: false
+    key_norm_after_rope: false
+    use_query_layernorm: false
+    use_context_groupnorm: false
+    use_mariana_gqa_pattern: false
+    use_sequence_parallel_attention: false
+    use_sequence_parallel_attention_a2a: false
+    context_parallel_use_all_gather: false
+    fp8_use_bf16_layers: ''
+    deterministic_mode: false
+    megatron_tensor_parallel_size: 8
+    megatron_pipeline_parallel_size: 1
+    megatron_context_parallel_size: 1
+    megatron_expert_parallel_size: 1
+    megatron_expert_parallel_size_in_dp: 1
+    megatron_context_parallel_query_only: false
+    megatron_num_layers_per_virtual_pipeline_stage: 0
+    megatron_micro_batch_size: 1
+    megatron_global_batch_size: 256
+    megatron_sequence_parallel: true
+    megatron_recompute_granularity: ''
+    megatron_use_flash_attention: true
+    megatron_recompute_method: uniform
+    megatron_recompute_num_layers: 1
+    megatron_distribute_saved_activations: false
+    megatron_enable_distributed_optimizer: true
+    megatron_use_multi_precision_ddp: false
+    megatron_sequence_parallel_as_data_parallel_in_optimizer: false
+    megatron_gather_params_use_alltoall: false
+    megatron_enable_initial_jit_warmup: true
+    megatron_accumulate_allreduce_grads_in_fp32: true
+    megatron_bf16_use_bf16_allreduce_grads: false
+    megatron_grad_comm_type: ''
+    megatron_reduce_grads_use_alltoall: false
+    megatron_scale_loss_in_gradient: false
+    megatron_scale_gradient_after_allreduce: false
+    megatron_ddp_impl: local
+    megatron_bf16_qt: false
+    megatron_empty_cache_level: 0
+    megatron_force_fp32_embed: false
+    megatron_deterministic_flash_attn: false
+    megatron_switch_pp_and_dp: false
+    megatron_timing_log_level: 2
+    megatron_no_load_rng: false
+    megatron_no_save_rng: false
+    megatron_no_load_optim: false
+    megatron_mem_efficient_column_parallel: true
+    megatron_masked_softmax_fusion: true
+    megatron_bias_gelu_fusion: false
+    megatron_bias_dropout_fusion: false
+    megatron_gradient_accumulation_fusion: true
+    megatron_overlap_p2p_comm: false
+    megatron_deallocate_pipeline_outputs: true
+    megatron_timing_log_option: local
+    megatron_barrier_with_L1_time: false
+    megatron_strict_align_diff_with_ds: false
+    megatron_parallel_linear_force_weight_contiguous: false
+    megatron_use_mariana_softmax: false
+    megatron_use_mariana_activation: false
+    megatron_overlap_data_parallel_communication: false
+    megatron_overlap_dp_grad_comm: false
+    megatron_overlap_dp_param_comm: false
+    megatron_early_prefetch_dp_allgather: true
+    megatron_use_non_sequential_block: false
+    megatron_overlap_attn_grad_input_comm: true
+    megatron_sequence_data_parallel_size: -1
+    megatron_distributed_sequence_parallel_size: -1
+    megatron_num_layers_for_pipeline_stages: []
+    megatron_vocab_parallel_embedding_fusion: false
+    megatron_embedding_reduce_scatter_for_sp: true
+    megatron_print_args: true
+    megatron_grad_norm_skip: -1.0
+    megatron_reorder_wgrad: false
+    megatron_offload_activations: false
+    megatron_offload_ratio: 1.0
+    megatron_offload_launch_ratio: 1.0
+    megatron_optimizer_offload_main_param: false
+    megatron_data_parallel_random_init: false
+    megatron_pipeline_strategy: ''
+    megatron_pipeline_wgrad_strategy: ''
+    megatron_pipeline_warmup_overlap: false
+    megatron_allow_transformer_engine: false
+    megatron_fp8_e4m3: false
+    megatron_fp8_hybrid: false
+    megatron_fp8_wgrad: true
+    megatron_fp8_dgrad: true
+    megatron_fp8_margin: 0
+    megatron_fp8_interval: 1
+    megatron_transformer_impl: local
+    megatron_fp8_amax_history_len: 1024
+    megatron_fp8_amax_compute_algo: max
+    megatron_use_qlora: false
+    megatron_qlora_quant_weight_dtype: null
+    megatron_qlora_quant_real_store: false
+    megatron_qlora_quant_groupsize: -1
+    megatron_qlora_quant_input_dtype: ''
+    megatron_qlora_quant_aware_lora: false
+    megatron_qlora_quant_aware_L4Q: false
+    megatron_terapipe_nano_batch_size: -1
+  lora_config:
+    default:
+      lora_dropout: 0.0
+      lora_rank: 64
+      layers:
+      - all
+      init_method: normal
+      init_mode: nonzero_parallel_init
+      init_kwargs: {}
+      lora_alpha: 2.0
+      use_rslora: true
+      lora_experts_appr: full
+      use_qlora: false
+      qlora_quant_weight_dtype: null
+      qlora_quant_real_store: false
+      qlora_quant_aware_L4Q: false
+      qlora_quant_groupsize: -1
+      qlora_quant_input_dtype: None
+      qlora_quant_aware_lora: false
+      post_training_quant: false
+      fully_sharded: false
+      emb_trainable: true
+    target_modules:
+    - query_key_value
+    - experts
+    - dense
+    query_key_value:
+      lora_rank: -1
+      lora_alpha: -1.0
+    experts:
+      lora_rank: -1
+      lora_alpha: -1.0
+    dense:
+      lora_rank: -1
+      lora_alpha: -1.0
+    dense_h_to_4h:
+      lora_rank: -1
+      lora_alpha: -1.0
+    dense_4h_to_h:
+      lora_rank: -1
+      lora_alpha: -1.0
+  freeze_prefix: null
+  partial_pretrain: null
+  partial_pretrain_rename: null
+  reset_global_step: -1
+  override_lr_scheduler: true
+  start_debug_server: false
+  clip_token_ids: false
+data:
+  train_path: hdfs://haruna/home/byte_data_seed/hdd_hldy/seed_code_seekpath/pretrained_yaml_new/V1_longct_datacard_hdfs_new_stage2_code_ct_fim_2.yaml
+  val_path:
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/D73_val_20240507_2_200M_token_plain_source_v2_1_part
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/human_all_lite
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/autoeval_code_val_lite
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/pretrain_auto_eval_merged_all_20240412_ceval_1_part
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/merged_few_benchmark_datasets_20240705_1_part
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/pretrain_auto_eval_merged_all_v0.3_1_part
+  - hdfs://haruna/home/byte_data_seed/ssd_hldy/p0/D73_release/val/D74_val_20240621_200M_token_tok643_sa8192_plain_source_v2_1_part_dir
+  train_size: 5000000000000
+  val_size: -1
+  train_batch_size: 32
+  train_num_workers: 4
+  val_batch_size: -1
+  val_num_workers: 1
+  max_seq_len: 32768
+  val_max_seq_len: -1
+  text_keys:
+  - content_split
+  tokenizer: hdfs://haruna/home/byte_data_seed/hl_lq/seed_code/liuyongfei/tokenizers/bbpe155k-v6.4.3-ml.pret
+  gpu_prefetch: false
+  cpu_prefetch: false
+  dyn_bsz: true
+  dyn_bsz_margin: 0.0
+  stride: -1
+  warmup_step_rate: -1.0
+  tokenizer_type: bbpe
+  bsz_warmup: false
+  bsz_warmup_rate: 0.016
+  return_source: true
+  synthetic_sample: false
+  synthetic_batch: false
+  seq_lens: null
+  seq_probs: null
+  enable_sampling_ratios: false
+  train_path_with_ratio: null
+  src_weights: null
+  parse_aug_data: false
+  loader_accumulate: -1
+  bsz_warmup_warmup_step_rate: 0.002
+  max_epochs: 1
+  pad_idx: 1
+  strategy: megatron
+  megatron_micro_batch_size: 1
+  use_rmpad: true
+  hidden_size: -1
+  megatron_sequence_parallel: false
+  max_position_embeddings: 2048
+  position_embeddings_type: absolute
+  use_sequence_parallel_attention: false
+  use_sequence_parallel_attention_a2a: false
+  resume_ckpt_path: ''
+  val_override_est_steps: false
+  init_without_cli: true
+  rope_mode: default
+  rope_scale: 1
+  rope_base: 500000.0
+  rope_cut: false
+  rope_cut_head_dim: 0
+  init_val_loader_worker_beforehand: false
+  megatron_global_batch_size: 1
+  megatron_tensor_parallel_size: 1
+  megatron_pipeline_parallel_size: 1
+  n_head: 1
+log_level: INFO
+val_only: false
+merge_model_states: false
+merge_ckpt_dtype: bf16
+merge_cache_dir: ./
+download_ckpt_in_shards: true
+gc_interval: 50
+profiler_at_iter: -1
+timer_at_iter: -1
+profile_all_ranks: false
+profile_ranks: []
+profile_every_n_steps: -1
+profiler_memory_at_iter: null
+profile_max_preview_rank: 0

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "transformers_version": "4.46.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e936d8a183cc516c3e7bc12ba683640c6389d437fddd7a7ada1dc14b55e22320
+size 4962001760

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2fabf094dcada2d9595bf7ff769b995d9d08b4a8a32035587be4653c9ab126b
+size 4915916160

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b97958c5cda4cd08ca6b59ba32e2d613158805bb5c4172b4cdf207af624a5e62
+size 4999819336

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfe5acfac5ac5231ce100c6a690d2d164693b91fe3d753d39aacc9e89ebbd663
+size 1623221024

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16500924416
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<[BOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<[EOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<[PAD_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<[SEP_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f5a7581f9bd555b11181c8fd69e9c7e6f6f5904d958f7c8771393aec7de1da9
+size 11892194

tokenizer_config.json

@@ -0,0 +1,1036 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<[BOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<[PAD_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<[EOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<[UNK_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<[CLS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<[MASK_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<[SEP_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<[PLHD7_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<[PLHD8_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<[PLHD9_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<[PLHD10_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<[PLHD11_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<[PLHD12_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<[PLHD13_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<[PLHD14_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<[PLHD15_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<[PLHD16_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<[PLHD17_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<[PLHD18_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<[PLHD19_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<[PLHD20_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<[PLHD21_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<[PLHD22_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<[PLHD23_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<[PLHD24_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<[PLHD25_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<[PLHD26_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<[PLHD27_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<[PLHD28_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<[PLHD29_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<[PLHD30_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<[PLHD31_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<[PLHD32_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<[PLHD33_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<[PLHD34_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<[PLHD35_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<[PLHD36_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "<[PLHD37_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "38": {
+      "content": "<[PLHD38_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "39": {
+      "content": "<[PLHD39_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "40": {
+      "content": "<[PLHD40_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "41": {
+      "content": "<[PLHD41_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "42": {
+      "content": "<[PLHD42_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "43": {
+      "content": "<[PLHD43_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "44": {
+      "content": "<[PLHD44_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "45": {
+      "content": "<[PLHD45_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46": {
+      "content": "<[PLHD46_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "47": {
+      "content": "<[PLHD47_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "48": {
+      "content": "<[PLHD48_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49": {
+      "content": "<[PLHD49_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50": {
+      "content": "<[PLHD50_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "51": {
+      "content": "<[PLHD51_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "52": {
+      "content": "<[PLHD52_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "53": {
+      "content": "<[PLHD53_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "54": {
+      "content": "<[PLHD54_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "55": {
+      "content": "<[PLHD55_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "56": {
+      "content": "<[PLHD56_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "57": {
+      "content": "<[PLHD57_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "58": {
+      "content": "<[PLHD58_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "59": {
+      "content": "<[PLHD59_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "60": {
+      "content": "<[PLHD60_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "61": {
+      "content": "<[PLHD61_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "62": {
+      "content": "<[PLHD62_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "63": {
+      "content": "<[PLHD63_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64": {
+      "content": "<[PLHD64_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65": {
+      "content": "<[PLHD65_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "66": {
+      "content": "<[PLHD66_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "67": {
+      "content": "<[PLHD67_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "68": {
+      "content": "<[PLHD68_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "69": {
+      "content": "<[PLHD69_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "70": {
+      "content": "<[PLHD70_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "71": {
+      "content": "<[PLHD71_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "72": {
+      "content": "<[PLHD72_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "73": {
+      "content": "<[PLHD73_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "74": {
+      "content": "<[PLHD74_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "75": {
+      "content": "<[PLHD75_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "76": {
+      "content": "<[PLHD76_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "77": {
+      "content": "<[PLHD77_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "78": {
+      "content": "<[PLHD78_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "79": {
+      "content": "<[PLHD79_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "80": {
+      "content": "<[PLHD80_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "81": {
+      "content": "<[PLHD81_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "82": {
+      "content": "<[PLHD82_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "83": {
+      "content": "<[PLHD83_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "84": {
+      "content": "<[PLHD84_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "85": {
+      "content": "<[PLHD85_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "86": {
+      "content": "<[PLHD86_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "87": {
+      "content": "<[PLHD87_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "88": {
+      "content": "<[PLHD88_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "89": {
+      "content": "<[PLHD89_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "90": {
+      "content": "<[PLHD90_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "91": {
+      "content": "<[PLHD91_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92": {
+      "content": "<[PLHD92_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "93": {
+      "content": "<[PLHD93_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "94": {
+      "content": "<[PLHD94_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "95": {
+      "content": "<[PLHD95_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "96": {
+      "content": "<[PLHD96_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "97": {
+      "content": "<[PLHD97_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "98": {
+      "content": "<[PLHD98_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "99": {
+      "content": "<[PLHD99_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "<[PLHD100_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "<[PLHD101_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "<[PLHD102_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "<[PLHD103_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "<[PLHD104_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "105": {
+      "content": "<[PLHD105_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "106": {
+      "content": "<[PLHD106_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<[PLHD107_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "<[PLHD108_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "109": {
+      "content": "<[PLHD109_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "110": {
+      "content": "<[PLHD110_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "111": {
+      "content": "<[PLHD111_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
+      "content": "<[PLHD112_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "113": {
+      "content": "<[PLHD113_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "114": {
+      "content": "<[PLHD114_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "115": {
+      "content": "<[PLHD115_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "116": {
+      "content": "<[PLHD116_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "117": {
+      "content": "<[PLHD117_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "118": {
+      "content": "<[PLHD118_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "119": {
+      "content": "<[PLHD119_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "120": {
+      "content": "<[PLHD120_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "121": {
+      "content": "<[PLHD121_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "122": {
+      "content": "<[PLHD122_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "123": {
+      "content": "<[PLHD123_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "124": {
+      "content": "<[PLHD124_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "125": {
+      "content": "<[PLHD125_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "126": {
+      "content": "<[PLHD126_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "127": {
+      "content": "<[PLHD127_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<[BOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ bos_token + role + '\n' + message['content'] | trim + eos_token }}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<[EOS_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<[PAD_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+  "sep_token": "<[SEP_never_used_51bce0c785ca2f68081bfa7d91973934]>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}