7767517
2396 2947
MemoryData model_info 0 1 model_info 0=5 21=1
Input input_0 0 1 token 0=1 1=1 2=1
Input input_1 0 1 state_0_in 0=1024
Input input_2 0 1 state_1_in 0=64 1=64 2=16
Input input_3 0 1 state_2_in 0=1024
Input input_4 0 1 state_3_in 0=1024
Input input_5 0 1 state_4_in 0=64 1=64 2=16
Input input_6 0 1 state_5_in 0=1024
Input input_7 0 1 state_6_in 0=1024
Input input_8 0 1 state_7_in 0=64 1=64 2=16
Input input_9 0 1 state_8_in 0=1024
Input input_10 0 1 state_9_in 0=1024
Input input_11 0 1 state_10_in 0=64 1=64 2=16
Input input_12 0 1 state_11_in 0=1024
Input input_13 0 1 state_12_in 0=1024
Input input_14 0 1 state_13_in 0=64 1=64 2=16
Input input_15 0 1 state_14_in 0=1024
Input input_16 0 1 state_15_in 0=1024
Input input_17 0 1 state_16_in 0=64 1=64 2=16
Input input_18 0 1 state_17_in 0=1024
Input input_19 0 1 state_18_in 0=1024
Input input_20 0 1 state_19_in 0=64 1=64 2=16
Input input_21 0 1 state_20_in 0=1024
Input input_22 0 1 state_21_in 0=1024
Input input_23 0 1 state_22_in 0=64 1=64 2=16
Input input_24 0 1 state_23_in 0=1024
Input input_25 0 1 state_24_in 0=1024
Input input_26 0 1 state_25_in 0=64 1=64 2=16
Input input_27 0 1 state_26_in 0=1024
Input input_28 0 1 state_27_in 0=1024
Input input_29 0 1 state_28_in 0=64 1=64 2=16
Input input_30 0 1 state_29_in 0=1024
Input input_31 0 1 state_30_in 0=1024
Input input_32 0 1 state_31_in 0=64 1=64 2=16
Input input_33 0 1 state_32_in 0=1024
Input input_34 0 1 state_33_in 0=1024
Input input_35 0 1 state_34_in 0=64 1=64 2=16
Input input_36 0 1 state_35_in 0=1024
Input input_37 0 1 state_36_in 0=1024
Input input_38 0 1 state_37_in 0=64 1=64 2=16
Input input_39 0 1 state_38_in 0=1024
Input input_40 0 1 state_39_in 0=1024
Input input_41 0 1 state_40_in 0=64 1=64 2=16
Input input_42 0 1 state_41_in 0=1024
Input input_43 0 1 state_42_in 0=1024
Input input_44 0 1 state_43_in 0=64 1=64 2=16
Input input_45 0 1 state_44_in 0=1024
Input input_46 0 1 state_45_in 0=1024
Input input_47 0 1 state_46_in 0=64 1=64 2=16
Input input_48 0 1 state_47_in 0=1024
Input input_49 0 1 state_48_in 0=1024
Input input_50 0 1 state_49_in 0=64 1=64 2=16
Input input_51 0 1 state_50_in 0=1024
Input input_52 0 1 state_51_in 0=1024
Input input_53 0 1 state_52_in 0=64 1=64 2=16
Input input_54 0 1 state_53_in 0=1024
Input input_55 0 1 state_54_in 0=1024
Input input_56 0 1 state_55_in 0=64 1=64 2=16
Input input_57 0 1 state_56_in 0=1024
Input input_58 0 1 state_57_in 0=1024
Input input_59 0 1 state_58_in 0=64 1=64 2=16
Input input_60 0 1 state_59_in 0=1024
Input input_61 0 1 state_60_in 0=1024
Input input_62 0 1 state_61_in 0=64 1=64 2=16
Input input_63 0 1 state_62_in 0=1024
Input input_64 0 1 state_63_in 0=1024
Input input_65 0 1 state_64_in 0=64 1=64 2=16
Input input_66 0 1 state_65_in 0=1024
Input input_67 0 1 state_66_in 0=1024
Input input_68 0 1 state_67_in 0=64 1=64 2=16
Input input_69 0 1 state_68_in 0=1024
Input input_70 0 1 state_69_in 0=1024
Input input_71 0 1 state_70_in 0=64 1=64 2=16
Input input_72 0 1 state_71_in 0=1024
Embed embedding 1 1 token emb 0=1024 1=65536 3=67108864
Split split_emb 1 2 emb att_0_x_last att_0_x
LayerNorm layernorm_0 1 1 att_0_x att_0_xx 0=1024 1=0.00001 2=1
Split split_att_0_xx 1 3 att_0_xx att_0_xx_0 att_0_xx_1 state_0_out
BinaryOp sub_0 2 1 state_0_in att_0_xx_0 att_0_sx 0=1
MemoryData data_att_0_maa 0 1 att_0_maa 0=1024 1=6 21=0
BinaryOp mul_0 2 1 att_0_sx att_0_maa att_0_maa_x 0=2
BinaryOp add_0 2 1 att_0_xx_1 att_0_maa_x att_0_maa_rwkvag 0=0
Slice slice_att_0_maa_rwkvag 1 6 att_0_maa_rwkvag att_0_mr att_0_mw att_0_mk att_0_mv att_0_ma att_0_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_0_mv 1 2 att_0_mv att_0_mv_0 att_0_mv_1
Gemm gemm_att_0_key 1 1 att_0_mk att_0_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_value 1 1 att_0_mv_0 att_0_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_receptance 1 1 att_0_mr att_0_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_ma_lora_0 1 1 att_0_ma att_0_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_0_ma_lora_1 1 1 att_0_ma_lora_0 att_0_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_0_a0 0 1 att_0_a0 0=1024 21=0
BinaryOp add_1 2 1 att_0_ma_lora_1 att_0_a0 att_0_ma_lora_2 0=0
Sigmoid sigmoid_0 1 1 att_0_ma_lora_2 att_0_a
Split split_att_0_a 1 2 att_0_a att_0_a_0 att_0_a_1
Gemm gemm_att_0_mg_lora 1 1 att_0_mg att_0_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_1 1 1 att_0_mg_lora att_0_mg_lora_sigmoid
Gemm gemm_att_0_gate 1 1 att_0_mg_lora_sigmoid att_0_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_0_k_k 0 1 att_0_k_k 0=1024 21=0
MemoryData data_att_0_k_a 0 1 att_0_k_a 0=1024 21=0
MemoryData data_att_0_r_k 0 1 att_0_r_k 0=1024 21=0
Split split_att_0_key 1 2 att_0_key att_0_key_0 att_0_key_1
BinaryOp mul_1 2 1 att_0_key_0 att_0_k_k att_0_key_k 0=2
Reshape reshape_0 1 1 att_0_key_k att_0_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_0 1 1 att_0_key_k_reshape att_0_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_1 1 1 att_0_key_k_norm att_0_key_k_norm_reshape 0=1024
BinaryOp add_2 1 1 att_0_a_0 att_0_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_2 2 1 att_0_a_0_minus att_0_k_a att_0_a_minus1_mulka 0=2
BinaryOp add_3 1 1 att_0_a_minus1_mulka att_0_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_3 2 1 att_0_a_minus1_mulka_plus1 att_0_key_1 att_0_key_a 0=2
Split split_att_0_value 1 2 att_0_value att_0_value_final v_first_0
Gemm gemm_att_0_mw_lora 1 1 att_0_mw att_0_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_0_mw_lora 1 1 att_0_mw_lora att_0_mw_lora_tanh
Gemm gemm_att_0_mw_lora_tanh_linear 1 1 att_0_mw_lora_tanh att_0_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_0_td 0 1 att_0_td 0=1024 21=0
BinaryOp add_4 2 1 att_0_mw_lora_tanh_linear att_0_td att_0_time_decay_pre 0=0
Sigmoid sigmoid_2 1 1 att_0_time_decay_pre att_0_time_decay_sigmoid
Exp exp_0 1 1 att_0_time_decay_sigmoid att_0_time_decay 1=-0.606531
Reshape reshape_2 1 1 att_0_time_decay att_0_time_decay_reshape 0=64 1=1 2=16
Split split_att_0_key_a 1 2 att_0_key_a att_0_key_a_0 att_0_key_a_1
Reshape reshape_3 1 1 att_0_key_a_0 att_0_key_a_reshape 0=64 1=1 2=16
Split split_att_0_value_final 1 3 att_0_value_final att_0_value_final_0 att_0_value_final_1 att_0_value_final_2
Reshape reshape_4 1 1 att_0_value_final_0 att_0_value_final_0_reshape 0=64 1=16
Reshape reshape_5 1 1 att_0_value_final_1 att_0_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_0_vk 2 1 att_0_value_final_1_reshape att_0_key_a_reshape att_0_vk 0=0
Split split_att_0_key_k_norm_reshape 1 2 att_0_key_k_norm_reshape att_0_kk_0 att_0_kk_1
BinaryOp mul_4 1 1 att_0_kk_0 att_0_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_6 1 1 att_0_kk_0_neg att_0_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_5 2 1 att_0_kk_1 att_0_a_1 att_0_kk_a 0=2
Reshape reshape_7 1 1 att_0_kk_a att_0_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_0_ab 2 1 att_0_kk_0_neg_reshape att_0_kk_a_reshape att_0_ab 0=0
Split split_state_1_in 1 2 state_1_in att_0_state_prev_0 att_0_state_prev_1
BinaryOp mul_6 2 1 att_0_state_prev_0 att_0_time_decay_reshape att_0_state_td 0=2
MatMul matmul_att_0_sab 2 1 att_0_state_prev_1 att_0_ab att_0_sab 0=0
BinaryOp add_5 2 1 att_0_sab att_0_vk att_0_sab_vk 0=0
BinaryOp add_6 2 1 att_0_sab_vk att_0_state_td att_0_state_new 0=0
Split split_att_0_state_new 1 2 att_0_state_new att_0_state_new_0 state_1_out
Split split_att_0_receptance 1 2 att_0_receptance att_0_receptance_0 att_0_receptance_1
Reshape reshape_8 1 1 att_0_receptance_0 att_0_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_0_wkv_out 2 1 att_0_state_new_0 att_0_receptance_reshape att_0_wkv_out 0=0
Reshape reshape_9 1 1 att_0_wkv_out att_0_wkv_out_flatten 0=1024
GroupNorm groupnorm_0 1 1 att_0_wkv_out_flatten att_0_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_7 2 1 att_0_receptance_1 att_0_key_a_1 att_0_receptance_key_a 0=2
BinaryOp mul_8 2 1 att_0_receptance_key_a att_0_r_k att_0_rk 0=2
Reshape reshape_10 1 1 att_0_rk att_0_rk_reshape 0=64 1=16
Reduction sum_0 1 1 att_0_rk_reshape att_0_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_9 2 1 att_0_value_final_0_reshape att_0_rk_sum att_0_value_final_0_rk 0=2
Reshape reshape_11 1 1 att_0_value_final_0_rk att_0_value_final_0_rk_reshape 0=1024
BinaryOp add_7 2 1 att_0_value_final_0_rk_reshape att_0_x_gn att_0_x_gn_rkv 0=0
BinaryOp mul_10 2 1 att_0_x_gn_rkv att_0_gate att_0_x_gate 0=2
Gemm gemm_att_0_x_out 1 1 att_0_x_gate att_0_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_8 2 1 att_0_x_out att_0_x_last time_mixing_0_out 0=0
Split split_time_mixing_0_out 1 2 time_mixing_0_out ffn_0_x_last ffn_0_x
LayerNorm layernorm_1 1 1 ffn_0_x ffn_0_xx 0=1024 1=0.00001 2=1
Split split_ffn_0_xx 1 3 ffn_0_xx ffn_0_xx_0 ffn_0_xx_1 state_2_out
BinaryOp sub_1 2 1 state_2_in ffn_0_xx_0 ffn_0_sx 0=1
Split split_ffn_0_sx 1 1 ffn_0_sx ffn_0_sx_0
MemoryData data_ffn_0_x_k 0 1 ffn_0_x_k 0=1024 21=0
BinaryOp mul_11 2 1 ffn_0_sx_0 ffn_0_x_k ffn_0_xk 0=2
BinaryOp add_9 2 1 ffn_0_xk ffn_0_xx_1 ffn_0_xxk 0=0
Gemm gemm_ffn_0_key 1 1 ffn_0_xxk ffn_0_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_0 1 1 ffn_0_key ffn_0_key_relu
UnaryOp square_ffn_0_key_relu 1 1 ffn_0_key_relu ffn_0_key_relu_square 0=4
Gemm gemm_ffn_0_value 1 1 ffn_0_key_relu_square ffn_0_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_10 2 1 ffn_0_value ffn_0_x_last channel_mixing_0_out 0=0
Split split_channel_mixing_0_out 1 2 channel_mixing_0_out att_1_x_last att_1_x
LayerNorm layernorm_2 1 1 att_1_x att_1_xx 0=1024 1=0.00001 2=1
Split split_att_1_xx 1 3 att_1_xx att_1_xx_0 att_1_xx_1 state_3_out
BinaryOp sub_2 2 1 state_3_in att_1_xx_0 att_1_sx 0=1
MemoryData data_att_1_maa 0 1 att_1_maa 0=1024 1=6 21=0
BinaryOp mul_12 2 1 att_1_sx att_1_maa att_1_maa_x 0=2
BinaryOp add_11 2 1 att_1_xx_1 att_1_maa_x att_1_maa_rwkvag 0=0
Slice slice_att_1_maa_rwkvag 1 6 att_1_maa_rwkvag att_1_mr att_1_mw att_1_mk att_1_mv att_1_ma att_1_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_1_mv 1 2 att_1_mv att_1_mv_0 att_1_mv_1
Gemm gemm_att_1_key 1 1 att_1_mk att_1_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_value 1 1 att_1_mv_0 att_1_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_receptance 1 1 att_1_mr att_1_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_ma_lora_0 1 1 att_1_ma att_1_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_1_ma_lora_1 1 1 att_1_ma_lora_0 att_1_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_1_a0 0 1 att_1_a0 0=1024 21=0
BinaryOp add_12 2 1 att_1_ma_lora_1 att_1_a0 att_1_ma_lora_2 0=0
Sigmoid sigmoid_3 1 1 att_1_ma_lora_2 att_1_a
Split split_att_1_a 1 2 att_1_a att_1_a_0 att_1_a_1
Gemm gemm_att_1_mg_lora 1 1 att_1_mg att_1_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_4 1 1 att_1_mg_lora att_1_mg_lora_sigmoid
Gemm gemm_att_1_gate 1 1 att_1_mg_lora_sigmoid att_1_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_1_k_k 0 1 att_1_k_k 0=1024 21=0
MemoryData data_att_1_k_a 0 1 att_1_k_a 0=1024 21=0
MemoryData data_att_1_r_k 0 1 att_1_r_k 0=1024 21=0
Split split_att_1_key 1 2 att_1_key att_1_key_0 att_1_key_1
BinaryOp mul_13 2 1 att_1_key_0 att_1_k_k att_1_key_k 0=2
Reshape reshape_12 1 1 att_1_key_k att_1_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_1 1 1 att_1_key_k_reshape att_1_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_13 1 1 att_1_key_k_norm att_1_key_k_norm_reshape 0=1024
BinaryOp add_13 1 1 att_1_a_0 att_1_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_14 2 1 att_1_a_0_minus att_1_k_a att_1_a_minus1_mulka 0=2
BinaryOp add_14 1 1 att_1_a_minus1_mulka att_1_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_15 2 1 att_1_a_minus1_mulka_plus1 att_1_key_1 att_1_key_a 0=2
Split split_att_1_value 1 2 att_1_value att_1_value_0 att_1_value_1
Split split_v_first_0 1 2 v_first_0 att_1_v_first v_first_1
BinaryOp minus_0 2 1 att_1_v_first att_1_value_1 att_1_vfirst_minus_value 0=1
Gemm gemm_att_1_mv_lora_0 1 1 att_1_mv_1 att_1_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_1_mv_lora_1 1 1 att_1_mv_lora_0 att_1_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_1_v0 0 1 att_1_v0 0=1024 21=0
BinaryOp add_15 2 1 att_1_mv_lora_1 att_1_v0 att_1_mv_lora_2 0=0
Sigmoid sigmoid_5 1 1 att_1_mv_lora_2 att_1_mv_lora_sigmoid
BinaryOp mul_16 2 1 att_1_mv_lora_sigmoid att_1_vfirst_minus_value att_1_vfirst_minus_value_mul 0=2
BinaryOp add_16 2 1 att_1_value_0 att_1_vfirst_minus_value_mul att_1_value_final 0=0
Gemm gemm_att_1_mw_lora 1 1 att_1_mw att_1_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_1_mw_lora 1 1 att_1_mw_lora att_1_mw_lora_tanh
Gemm gemm_att_1_mw_lora_tanh_linear 1 1 att_1_mw_lora_tanh att_1_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_1_td 0 1 att_1_td 0=1024 21=0
BinaryOp add_17 2 1 att_1_mw_lora_tanh_linear att_1_td att_1_time_decay_pre 0=0
Sigmoid sigmoid_6 1 1 att_1_time_decay_pre att_1_time_decay_sigmoid
Exp exp_1 1 1 att_1_time_decay_sigmoid att_1_time_decay 1=-0.606531
Reshape reshape_14 1 1 att_1_time_decay att_1_time_decay_reshape 0=64 1=1 2=16
Split split_att_1_key_a 1 2 att_1_key_a att_1_key_a_0 att_1_key_a_1
Reshape reshape_15 1 1 att_1_key_a_0 att_1_key_a_reshape 0=64 1=1 2=16
Split split_att_1_value_final 1 3 att_1_value_final att_1_value_final_0 att_1_value_final_1 att_1_value_final_2
Reshape reshape_16 1 1 att_1_value_final_0 att_1_value_final_0_reshape 0=64 1=16
Reshape reshape_17 1 1 att_1_value_final_1 att_1_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_1_vk 2 1 att_1_value_final_1_reshape att_1_key_a_reshape att_1_vk 0=0
Split split_att_1_key_k_norm_reshape 1 2 att_1_key_k_norm_reshape att_1_kk_0 att_1_kk_1
BinaryOp mul_17 1 1 att_1_kk_0 att_1_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_18 1 1 att_1_kk_0_neg att_1_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_18 2 1 att_1_kk_1 att_1_a_1 att_1_kk_a 0=2
Reshape reshape_19 1 1 att_1_kk_a att_1_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_1_ab 2 1 att_1_kk_0_neg_reshape att_1_kk_a_reshape att_1_ab 0=0
Split split_state_4_in 1 2 state_4_in att_1_state_prev_0 att_1_state_prev_1
BinaryOp mul_19 2 1 att_1_state_prev_0 att_1_time_decay_reshape att_1_state_td 0=2
MatMul matmul_att_1_sab 2 1 att_1_state_prev_1 att_1_ab att_1_sab 0=0
BinaryOp add_18 2 1 att_1_sab att_1_vk att_1_sab_vk 0=0
BinaryOp add_19 2 1 att_1_sab_vk att_1_state_td att_1_state_new 0=0
Split split_att_1_state_new 1 2 att_1_state_new att_1_state_new_0 state_4_out
Split split_att_1_receptance 1 2 att_1_receptance att_1_receptance_0 att_1_receptance_1
Reshape reshape_20 1 1 att_1_receptance_0 att_1_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_1_wkv_out 2 1 att_1_state_new_0 att_1_receptance_reshape att_1_wkv_out 0=0
Reshape reshape_21 1 1 att_1_wkv_out att_1_wkv_out_flatten 0=1024
GroupNorm groupnorm_1 1 1 att_1_wkv_out_flatten att_1_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_20 2 1 att_1_receptance_1 att_1_key_a_1 att_1_receptance_key_a 0=2
BinaryOp mul_21 2 1 att_1_receptance_key_a att_1_r_k att_1_rk 0=2
Reshape reshape_22 1 1 att_1_rk att_1_rk_reshape 0=64 1=16
Reduction sum_1 1 1 att_1_rk_reshape att_1_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_22 2 1 att_1_value_final_0_reshape att_1_rk_sum att_1_value_final_0_rk 0=2
Reshape reshape_23 1 1 att_1_value_final_0_rk att_1_value_final_0_rk_reshape 0=1024
BinaryOp add_20 2 1 att_1_value_final_0_rk_reshape att_1_x_gn att_1_x_gn_rkv 0=0
BinaryOp mul_23 2 1 att_1_x_gn_rkv att_1_gate att_1_x_gate 0=2
Gemm gemm_att_1_x_out 1 1 att_1_x_gate att_1_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_21 2 1 att_1_x_out att_1_x_last time_mixing_1_out 0=0
Split split_time_mixing_1_out 1 2 time_mixing_1_out ffn_1_x_last ffn_1_x
LayerNorm layernorm_3 1 1 ffn_1_x ffn_1_xx 0=1024 1=0.00001 2=1
Split split_ffn_1_xx 1 3 ffn_1_xx ffn_1_xx_0 ffn_1_xx_1 state_5_out
BinaryOp sub_3 2 1 state_5_in ffn_1_xx_0 ffn_1_sx 0=1
Split split_ffn_1_sx 1 1 ffn_1_sx ffn_1_sx_0
MemoryData data_ffn_1_x_k 0 1 ffn_1_x_k 0=1024 21=0
BinaryOp mul_24 2 1 ffn_1_sx_0 ffn_1_x_k ffn_1_xk 0=2
BinaryOp add_22 2 1 ffn_1_xk ffn_1_xx_1 ffn_1_xxk 0=0
Gemm gemm_ffn_1_key 1 1 ffn_1_xxk ffn_1_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_1 1 1 ffn_1_key ffn_1_key_relu
UnaryOp square_ffn_1_key_relu 1 1 ffn_1_key_relu ffn_1_key_relu_square 0=4
Gemm gemm_ffn_1_value 1 1 ffn_1_key_relu_square ffn_1_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_23 2 1 ffn_1_value ffn_1_x_last channel_mixing_1_out 0=0
Split split_channel_mixing_1_out 1 2 channel_mixing_1_out att_2_x_last att_2_x
LayerNorm layernorm_4 1 1 att_2_x att_2_xx 0=1024 1=0.00001 2=1
Split split_att_2_xx 1 3 att_2_xx att_2_xx_0 att_2_xx_1 state_6_out
BinaryOp sub_4 2 1 state_6_in att_2_xx_0 att_2_sx 0=1
MemoryData data_att_2_maa 0 1 att_2_maa 0=1024 1=6 21=0
BinaryOp mul_25 2 1 att_2_sx att_2_maa att_2_maa_x 0=2
BinaryOp add_24 2 1 att_2_xx_1 att_2_maa_x att_2_maa_rwkvag 0=0
Slice slice_att_2_maa_rwkvag 1 6 att_2_maa_rwkvag att_2_mr att_2_mw att_2_mk att_2_mv att_2_ma att_2_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_2_mv 1 2 att_2_mv att_2_mv_0 att_2_mv_1
Gemm gemm_att_2_key 1 1 att_2_mk att_2_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_value 1 1 att_2_mv_0 att_2_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_receptance 1 1 att_2_mr att_2_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_ma_lora_0 1 1 att_2_ma att_2_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_2_ma_lora_1 1 1 att_2_ma_lora_0 att_2_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_2_a0 0 1 att_2_a0 0=1024 21=0
BinaryOp add_25 2 1 att_2_ma_lora_1 att_2_a0 att_2_ma_lora_2 0=0
Sigmoid sigmoid_7 1 1 att_2_ma_lora_2 att_2_a
Split split_att_2_a 1 2 att_2_a att_2_a_0 att_2_a_1
Gemm gemm_att_2_mg_lora 1 1 att_2_mg att_2_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_8 1 1 att_2_mg_lora att_2_mg_lora_sigmoid
Gemm gemm_att_2_gate 1 1 att_2_mg_lora_sigmoid att_2_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_2_k_k 0 1 att_2_k_k 0=1024 21=0
MemoryData data_att_2_k_a 0 1 att_2_k_a 0=1024 21=0
MemoryData data_att_2_r_k 0 1 att_2_r_k 0=1024 21=0
Split split_att_2_key 1 2 att_2_key att_2_key_0 att_2_key_1
BinaryOp mul_26 2 1 att_2_key_0 att_2_k_k att_2_key_k 0=2
Reshape reshape_24 1 1 att_2_key_k att_2_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_2 1 1 att_2_key_k_reshape att_2_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_25 1 1 att_2_key_k_norm att_2_key_k_norm_reshape 0=1024
BinaryOp add_26 1 1 att_2_a_0 att_2_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_27 2 1 att_2_a_0_minus att_2_k_a att_2_a_minus1_mulka 0=2
BinaryOp add_27 1 1 att_2_a_minus1_mulka att_2_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_28 2 1 att_2_a_minus1_mulka_plus1 att_2_key_1 att_2_key_a 0=2
Split split_att_2_value 1 2 att_2_value att_2_value_0 att_2_value_1
Split split_v_first_1 1 2 v_first_1 att_2_v_first v_first_2
BinaryOp minus_1 2 1 att_2_v_first att_2_value_1 att_2_vfirst_minus_value 0=1
Gemm gemm_att_2_mv_lora_0 1 1 att_2_mv_1 att_2_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_2_mv_lora_1 1 1 att_2_mv_lora_0 att_2_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_2_v0 0 1 att_2_v0 0=1024 21=0
BinaryOp add_28 2 1 att_2_mv_lora_1 att_2_v0 att_2_mv_lora_2 0=0
Sigmoid sigmoid_9 1 1 att_2_mv_lora_2 att_2_mv_lora_sigmoid
BinaryOp mul_29 2 1 att_2_mv_lora_sigmoid att_2_vfirst_minus_value att_2_vfirst_minus_value_mul 0=2
BinaryOp add_29 2 1 att_2_value_0 att_2_vfirst_minus_value_mul att_2_value_final 0=0
Gemm gemm_att_2_mw_lora 1 1 att_2_mw att_2_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_2_mw_lora 1 1 att_2_mw_lora att_2_mw_lora_tanh
Gemm gemm_att_2_mw_lora_tanh_linear 1 1 att_2_mw_lora_tanh att_2_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_2_td 0 1 att_2_td 0=1024 21=0
BinaryOp add_30 2 1 att_2_mw_lora_tanh_linear att_2_td att_2_time_decay_pre 0=0
Sigmoid sigmoid_10 1 1 att_2_time_decay_pre att_2_time_decay_sigmoid
Exp exp_2 1 1 att_2_time_decay_sigmoid att_2_time_decay 1=-0.606531
Reshape reshape_26 1 1 att_2_time_decay att_2_time_decay_reshape 0=64 1=1 2=16
Split split_att_2_key_a 1 2 att_2_key_a att_2_key_a_0 att_2_key_a_1
Reshape reshape_27 1 1 att_2_key_a_0 att_2_key_a_reshape 0=64 1=1 2=16
Split split_att_2_value_final 1 3 att_2_value_final att_2_value_final_0 att_2_value_final_1 att_2_value_final_2
Reshape reshape_28 1 1 att_2_value_final_0 att_2_value_final_0_reshape 0=64 1=16
Reshape reshape_29 1 1 att_2_value_final_1 att_2_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_2_vk 2 1 att_2_value_final_1_reshape att_2_key_a_reshape att_2_vk 0=0
Split split_att_2_key_k_norm_reshape 1 2 att_2_key_k_norm_reshape att_2_kk_0 att_2_kk_1
BinaryOp mul_30 1 1 att_2_kk_0 att_2_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_30 1 1 att_2_kk_0_neg att_2_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_31 2 1 att_2_kk_1 att_2_a_1 att_2_kk_a 0=2
Reshape reshape_31 1 1 att_2_kk_a att_2_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_2_ab 2 1 att_2_kk_0_neg_reshape att_2_kk_a_reshape att_2_ab 0=0
Split split_state_7_in 1 2 state_7_in att_2_state_prev_0 att_2_state_prev_1
BinaryOp mul_32 2 1 att_2_state_prev_0 att_2_time_decay_reshape att_2_state_td 0=2
MatMul matmul_att_2_sab 2 1 att_2_state_prev_1 att_2_ab att_2_sab 0=0
BinaryOp add_31 2 1 att_2_sab att_2_vk att_2_sab_vk 0=0
BinaryOp add_32 2 1 att_2_sab_vk att_2_state_td att_2_state_new 0=0
Split split_att_2_state_new 1 2 att_2_state_new att_2_state_new_0 state_7_out
Split split_att_2_receptance 1 2 att_2_receptance att_2_receptance_0 att_2_receptance_1
Reshape reshape_32 1 1 att_2_receptance_0 att_2_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_2_wkv_out 2 1 att_2_state_new_0 att_2_receptance_reshape att_2_wkv_out 0=0
Reshape reshape_33 1 1 att_2_wkv_out att_2_wkv_out_flatten 0=1024
GroupNorm groupnorm_2 1 1 att_2_wkv_out_flatten att_2_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_33 2 1 att_2_receptance_1 att_2_key_a_1 att_2_receptance_key_a 0=2
BinaryOp mul_34 2 1 att_2_receptance_key_a att_2_r_k att_2_rk 0=2
Reshape reshape_34 1 1 att_2_rk att_2_rk_reshape 0=64 1=16
Reduction sum_2 1 1 att_2_rk_reshape att_2_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_35 2 1 att_2_value_final_0_reshape att_2_rk_sum att_2_value_final_0_rk 0=2
Reshape reshape_35 1 1 att_2_value_final_0_rk att_2_value_final_0_rk_reshape 0=1024
BinaryOp add_33 2 1 att_2_value_final_0_rk_reshape att_2_x_gn att_2_x_gn_rkv 0=0
BinaryOp mul_36 2 1 att_2_x_gn_rkv att_2_gate att_2_x_gate 0=2
Gemm gemm_att_2_x_out 1 1 att_2_x_gate att_2_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_34 2 1 att_2_x_out att_2_x_last time_mixing_2_out 0=0
Split split_time_mixing_2_out 1 2 time_mixing_2_out ffn_2_x_last ffn_2_x
LayerNorm layernorm_5 1 1 ffn_2_x ffn_2_xx 0=1024 1=0.00001 2=1
Split split_ffn_2_xx 1 3 ffn_2_xx ffn_2_xx_0 ffn_2_xx_1 state_8_out
BinaryOp sub_5 2 1 state_8_in ffn_2_xx_0 ffn_2_sx 0=1
Split split_ffn_2_sx 1 1 ffn_2_sx ffn_2_sx_0
MemoryData data_ffn_2_x_k 0 1 ffn_2_x_k 0=1024 21=0
BinaryOp mul_37 2 1 ffn_2_sx_0 ffn_2_x_k ffn_2_xk 0=2
BinaryOp add_35 2 1 ffn_2_xk ffn_2_xx_1 ffn_2_xxk 0=0
Gemm gemm_ffn_2_key 1 1 ffn_2_xxk ffn_2_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_2 1 1 ffn_2_key ffn_2_key_relu
UnaryOp square_ffn_2_key_relu 1 1 ffn_2_key_relu ffn_2_key_relu_square 0=4
Gemm gemm_ffn_2_value 1 1 ffn_2_key_relu_square ffn_2_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_36 2 1 ffn_2_value ffn_2_x_last channel_mixing_2_out 0=0
Split split_channel_mixing_2_out 1 2 channel_mixing_2_out att_3_x_last att_3_x
LayerNorm layernorm_6 1 1 att_3_x att_3_xx 0=1024 1=0.00001 2=1
Split split_att_3_xx 1 3 att_3_xx att_3_xx_0 att_3_xx_1 state_9_out
BinaryOp sub_6 2 1 state_9_in att_3_xx_0 att_3_sx 0=1
MemoryData data_att_3_maa 0 1 att_3_maa 0=1024 1=6 21=0
BinaryOp mul_38 2 1 att_3_sx att_3_maa att_3_maa_x 0=2
BinaryOp add_37 2 1 att_3_xx_1 att_3_maa_x att_3_maa_rwkvag 0=0
Slice slice_att_3_maa_rwkvag 1 6 att_3_maa_rwkvag att_3_mr att_3_mw att_3_mk att_3_mv att_3_ma att_3_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_3_mv 1 2 att_3_mv att_3_mv_0 att_3_mv_1
Gemm gemm_att_3_key 1 1 att_3_mk att_3_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_value 1 1 att_3_mv_0 att_3_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_receptance 1 1 att_3_mr att_3_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_ma_lora_0 1 1 att_3_ma att_3_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_3_ma_lora_1 1 1 att_3_ma_lora_0 att_3_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_3_a0 0 1 att_3_a0 0=1024 21=0
BinaryOp add_38 2 1 att_3_ma_lora_1 att_3_a0 att_3_ma_lora_2 0=0
Sigmoid sigmoid_11 1 1 att_3_ma_lora_2 att_3_a
Split split_att_3_a 1 2 att_3_a att_3_a_0 att_3_a_1
Gemm gemm_att_3_mg_lora 1 1 att_3_mg att_3_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_12 1 1 att_3_mg_lora att_3_mg_lora_sigmoid
Gemm gemm_att_3_gate 1 1 att_3_mg_lora_sigmoid att_3_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_3_k_k 0 1 att_3_k_k 0=1024 21=0
MemoryData data_att_3_k_a 0 1 att_3_k_a 0=1024 21=0
MemoryData data_att_3_r_k 0 1 att_3_r_k 0=1024 21=0
Split split_att_3_key 1 2 att_3_key att_3_key_0 att_3_key_1
BinaryOp mul_39 2 1 att_3_key_0 att_3_k_k att_3_key_k 0=2
Reshape reshape_36 1 1 att_3_key_k att_3_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_3 1 1 att_3_key_k_reshape att_3_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_37 1 1 att_3_key_k_norm att_3_key_k_norm_reshape 0=1024
BinaryOp add_39 1 1 att_3_a_0 att_3_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_40 2 1 att_3_a_0_minus att_3_k_a att_3_a_minus1_mulka 0=2
BinaryOp add_40 1 1 att_3_a_minus1_mulka att_3_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_41 2 1 att_3_a_minus1_mulka_plus1 att_3_key_1 att_3_key_a 0=2
Split split_att_3_value 1 2 att_3_value att_3_value_0 att_3_value_1
Split split_v_first_2 1 2 v_first_2 att_3_v_first v_first_3
BinaryOp minus_2 2 1 att_3_v_first att_3_value_1 att_3_vfirst_minus_value 0=1
Gemm gemm_att_3_mv_lora_0 1 1 att_3_mv_1 att_3_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_3_mv_lora_1 1 1 att_3_mv_lora_0 att_3_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_3_v0 0 1 att_3_v0 0=1024 21=0
BinaryOp add_41 2 1 att_3_mv_lora_1 att_3_v0 att_3_mv_lora_2 0=0
Sigmoid sigmoid_13 1 1 att_3_mv_lora_2 att_3_mv_lora_sigmoid
BinaryOp mul_42 2 1 att_3_mv_lora_sigmoid att_3_vfirst_minus_value att_3_vfirst_minus_value_mul 0=2
BinaryOp add_42 2 1 att_3_value_0 att_3_vfirst_minus_value_mul att_3_value_final 0=0
Gemm gemm_att_3_mw_lora 1 1 att_3_mw att_3_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_3_mw_lora 1 1 att_3_mw_lora att_3_mw_lora_tanh
Gemm gemm_att_3_mw_lora_tanh_linear 1 1 att_3_mw_lora_tanh att_3_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_3_td 0 1 att_3_td 0=1024 21=0
BinaryOp add_43 2 1 att_3_mw_lora_tanh_linear att_3_td att_3_time_decay_pre 0=0
Sigmoid sigmoid_14 1 1 att_3_time_decay_pre att_3_time_decay_sigmoid
Exp exp_3 1 1 att_3_time_decay_sigmoid att_3_time_decay 1=-0.606531
Reshape reshape_38 1 1 att_3_time_decay att_3_time_decay_reshape 0=64 1=1 2=16
Split split_att_3_key_a 1 2 att_3_key_a att_3_key_a_0 att_3_key_a_1
Reshape reshape_39 1 1 att_3_key_a_0 att_3_key_a_reshape 0=64 1=1 2=16
Split split_att_3_value_final 1 3 att_3_value_final att_3_value_final_0 att_3_value_final_1 att_3_value_final_2
Reshape reshape_40 1 1 att_3_value_final_0 att_3_value_final_0_reshape 0=64 1=16
Reshape reshape_41 1 1 att_3_value_final_1 att_3_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_3_vk 2 1 att_3_value_final_1_reshape att_3_key_a_reshape att_3_vk 0=0
Split split_att_3_key_k_norm_reshape 1 2 att_3_key_k_norm_reshape att_3_kk_0 att_3_kk_1
BinaryOp mul_43 1 1 att_3_kk_0 att_3_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_42 1 1 att_3_kk_0_neg att_3_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_44 2 1 att_3_kk_1 att_3_a_1 att_3_kk_a 0=2
Reshape reshape_43 1 1 att_3_kk_a att_3_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_3_ab 2 1 att_3_kk_0_neg_reshape att_3_kk_a_reshape att_3_ab 0=0
Split split_state_10_in 1 2 state_10_in att_3_state_prev_0 att_3_state_prev_1
BinaryOp mul_45 2 1 att_3_state_prev_0 att_3_time_decay_reshape att_3_state_td 0=2
MatMul matmul_att_3_sab 2 1 att_3_state_prev_1 att_3_ab att_3_sab 0=0
BinaryOp add_44 2 1 att_3_sab att_3_vk att_3_sab_vk 0=0
BinaryOp add_45 2 1 att_3_sab_vk att_3_state_td att_3_state_new 0=0
Split split_att_3_state_new 1 2 att_3_state_new att_3_state_new_0 state_10_out
Split split_att_3_receptance 1 2 att_3_receptance att_3_receptance_0 att_3_receptance_1
Reshape reshape_44 1 1 att_3_receptance_0 att_3_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_3_wkv_out 2 1 att_3_state_new_0 att_3_receptance_reshape att_3_wkv_out 0=0
Reshape reshape_45 1 1 att_3_wkv_out att_3_wkv_out_flatten 0=1024
GroupNorm groupnorm_3 1 1 att_3_wkv_out_flatten att_3_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_46 2 1 att_3_receptance_1 att_3_key_a_1 att_3_receptance_key_a 0=2
BinaryOp mul_47 2 1 att_3_receptance_key_a att_3_r_k att_3_rk 0=2
Reshape reshape_46 1 1 att_3_rk att_3_rk_reshape 0=64 1=16
Reduction sum_3 1 1 att_3_rk_reshape att_3_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_48 2 1 att_3_value_final_0_reshape att_3_rk_sum att_3_value_final_0_rk 0=2
Reshape reshape_47 1 1 att_3_value_final_0_rk att_3_value_final_0_rk_reshape 0=1024
BinaryOp add_46 2 1 att_3_value_final_0_rk_reshape att_3_x_gn att_3_x_gn_rkv 0=0
BinaryOp mul_49 2 1 att_3_x_gn_rkv att_3_gate att_3_x_gate 0=2
Gemm gemm_att_3_x_out 1 1 att_3_x_gate att_3_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_47 2 1 att_3_x_out att_3_x_last time_mixing_3_out 0=0
Split split_time_mixing_3_out 1 2 time_mixing_3_out ffn_3_x_last ffn_3_x
LayerNorm layernorm_7 1 1 ffn_3_x ffn_3_xx 0=1024 1=0.00001 2=1
Split split_ffn_3_xx 1 3 ffn_3_xx ffn_3_xx_0 ffn_3_xx_1 state_11_out
BinaryOp sub_7 2 1 state_11_in ffn_3_xx_0 ffn_3_sx 0=1
Split split_ffn_3_sx 1 1 ffn_3_sx ffn_3_sx_0
MemoryData data_ffn_3_x_k 0 1 ffn_3_x_k 0=1024 21=0
BinaryOp mul_50 2 1 ffn_3_sx_0 ffn_3_x_k ffn_3_xk 0=2
BinaryOp add_48 2 1 ffn_3_xk ffn_3_xx_1 ffn_3_xxk 0=0
Gemm gemm_ffn_3_key 1 1 ffn_3_xxk ffn_3_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_3 1 1 ffn_3_key ffn_3_key_relu
UnaryOp square_ffn_3_key_relu 1 1 ffn_3_key_relu ffn_3_key_relu_square 0=4
Gemm gemm_ffn_3_value 1 1 ffn_3_key_relu_square ffn_3_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_49 2 1 ffn_3_value ffn_3_x_last channel_mixing_3_out 0=0
Split split_channel_mixing_3_out 1 2 channel_mixing_3_out att_4_x_last att_4_x
LayerNorm layernorm_8 1 1 att_4_x att_4_xx 0=1024 1=0.00001 2=1
Split split_att_4_xx 1 3 att_4_xx att_4_xx_0 att_4_xx_1 state_12_out
BinaryOp sub_8 2 1 state_12_in att_4_xx_0 att_4_sx 0=1
MemoryData data_att_4_maa 0 1 att_4_maa 0=1024 1=6 21=0
BinaryOp mul_51 2 1 att_4_sx att_4_maa att_4_maa_x 0=2
BinaryOp add_50 2 1 att_4_xx_1 att_4_maa_x att_4_maa_rwkvag 0=0
Slice slice_att_4_maa_rwkvag 1 6 att_4_maa_rwkvag att_4_mr att_4_mw att_4_mk att_4_mv att_4_ma att_4_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_4_mv 1 2 att_4_mv att_4_mv_0 att_4_mv_1
Gemm gemm_att_4_key 1 1 att_4_mk att_4_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_value 1 1 att_4_mv_0 att_4_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_receptance 1 1 att_4_mr att_4_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_ma_lora_0 1 1 att_4_ma att_4_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_4_ma_lora_1 1 1 att_4_ma_lora_0 att_4_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_4_a0 0 1 att_4_a0 0=1024 21=0
BinaryOp add_51 2 1 att_4_ma_lora_1 att_4_a0 att_4_ma_lora_2 0=0
Sigmoid sigmoid_15 1 1 att_4_ma_lora_2 att_4_a
Split split_att_4_a 1 2 att_4_a att_4_a_0 att_4_a_1
Gemm gemm_att_4_mg_lora 1 1 att_4_mg att_4_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_16 1 1 att_4_mg_lora att_4_mg_lora_sigmoid
Gemm gemm_att_4_gate 1 1 att_4_mg_lora_sigmoid att_4_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_4_k_k 0 1 att_4_k_k 0=1024 21=0
MemoryData data_att_4_k_a 0 1 att_4_k_a 0=1024 21=0
MemoryData data_att_4_r_k 0 1 att_4_r_k 0=1024 21=0
Split split_att_4_key 1 2 att_4_key att_4_key_0 att_4_key_1
BinaryOp mul_52 2 1 att_4_key_0 att_4_k_k att_4_key_k 0=2
Reshape reshape_48 1 1 att_4_key_k att_4_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_4 1 1 att_4_key_k_reshape att_4_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_49 1 1 att_4_key_k_norm att_4_key_k_norm_reshape 0=1024
BinaryOp add_52 1 1 att_4_a_0 att_4_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_53 2 1 att_4_a_0_minus att_4_k_a att_4_a_minus1_mulka 0=2
BinaryOp add_53 1 1 att_4_a_minus1_mulka att_4_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_54 2 1 att_4_a_minus1_mulka_plus1 att_4_key_1 att_4_key_a 0=2
Split split_att_4_value 1 2 att_4_value att_4_value_0 att_4_value_1
Split split_v_first_3 1 2 v_first_3 att_4_v_first v_first_4
BinaryOp minus_3 2 1 att_4_v_first att_4_value_1 att_4_vfirst_minus_value 0=1
Gemm gemm_att_4_mv_lora_0 1 1 att_4_mv_1 att_4_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_4_mv_lora_1 1 1 att_4_mv_lora_0 att_4_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_4_v0 0 1 att_4_v0 0=1024 21=0
BinaryOp add_54 2 1 att_4_mv_lora_1 att_4_v0 att_4_mv_lora_2 0=0
Sigmoid sigmoid_17 1 1 att_4_mv_lora_2 att_4_mv_lora_sigmoid
BinaryOp mul_55 2 1 att_4_mv_lora_sigmoid att_4_vfirst_minus_value att_4_vfirst_minus_value_mul 0=2
BinaryOp add_55 2 1 att_4_value_0 att_4_vfirst_minus_value_mul att_4_value_final 0=0
Gemm gemm_att_4_mw_lora 1 1 att_4_mw att_4_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_4_mw_lora 1 1 att_4_mw_lora att_4_mw_lora_tanh
Gemm gemm_att_4_mw_lora_tanh_linear 1 1 att_4_mw_lora_tanh att_4_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_4_td 0 1 att_4_td 0=1024 21=0
BinaryOp add_56 2 1 att_4_mw_lora_tanh_linear att_4_td att_4_time_decay_pre 0=0
Sigmoid sigmoid_18 1 1 att_4_time_decay_pre att_4_time_decay_sigmoid
Exp exp_4 1 1 att_4_time_decay_sigmoid att_4_time_decay 1=-0.606531
Reshape reshape_50 1 1 att_4_time_decay att_4_time_decay_reshape 0=64 1=1 2=16
Split split_att_4_key_a 1 2 att_4_key_a att_4_key_a_0 att_4_key_a_1
Reshape reshape_51 1 1 att_4_key_a_0 att_4_key_a_reshape 0=64 1=1 2=16
Split split_att_4_value_final 1 3 att_4_value_final att_4_value_final_0 att_4_value_final_1 att_4_value_final_2
Reshape reshape_52 1 1 att_4_value_final_0 att_4_value_final_0_reshape 0=64 1=16
Reshape reshape_53 1 1 att_4_value_final_1 att_4_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_4_vk 2 1 att_4_value_final_1_reshape att_4_key_a_reshape att_4_vk 0=0
Split split_att_4_key_k_norm_reshape 1 2 att_4_key_k_norm_reshape att_4_kk_0 att_4_kk_1
BinaryOp mul_56 1 1 att_4_kk_0 att_4_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_54 1 1 att_4_kk_0_neg att_4_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_57 2 1 att_4_kk_1 att_4_a_1 att_4_kk_a 0=2
Reshape reshape_55 1 1 att_4_kk_a att_4_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_4_ab 2 1 att_4_kk_0_neg_reshape att_4_kk_a_reshape att_4_ab 0=0
Split split_state_13_in 1 2 state_13_in att_4_state_prev_0 att_4_state_prev_1
BinaryOp mul_58 2 1 att_4_state_prev_0 att_4_time_decay_reshape att_4_state_td 0=2
MatMul matmul_att_4_sab 2 1 att_4_state_prev_1 att_4_ab att_4_sab 0=0
BinaryOp add_57 2 1 att_4_sab att_4_vk att_4_sab_vk 0=0
BinaryOp add_58 2 1 att_4_sab_vk att_4_state_td att_4_state_new 0=0
Split split_att_4_state_new 1 2 att_4_state_new att_4_state_new_0 state_13_out
Split split_att_4_receptance 1 2 att_4_receptance att_4_receptance_0 att_4_receptance_1
Reshape reshape_56 1 1 att_4_receptance_0 att_4_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_4_wkv_out 2 1 att_4_state_new_0 att_4_receptance_reshape att_4_wkv_out 0=0
Reshape reshape_57 1 1 att_4_wkv_out att_4_wkv_out_flatten 0=1024
GroupNorm groupnorm_4 1 1 att_4_wkv_out_flatten att_4_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_59 2 1 att_4_receptance_1 att_4_key_a_1 att_4_receptance_key_a 0=2
BinaryOp mul_60 2 1 att_4_receptance_key_a att_4_r_k att_4_rk 0=2
Reshape reshape_58 1 1 att_4_rk att_4_rk_reshape 0=64 1=16
Reduction sum_4 1 1 att_4_rk_reshape att_4_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_61 2 1 att_4_value_final_0_reshape att_4_rk_sum att_4_value_final_0_rk 0=2
Reshape reshape_59 1 1 att_4_value_final_0_rk att_4_value_final_0_rk_reshape 0=1024
BinaryOp add_59 2 1 att_4_value_final_0_rk_reshape att_4_x_gn att_4_x_gn_rkv 0=0
BinaryOp mul_62 2 1 att_4_x_gn_rkv att_4_gate att_4_x_gate 0=2
Gemm gemm_att_4_x_out 1 1 att_4_x_gate att_4_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_60 2 1 att_4_x_out att_4_x_last time_mixing_4_out 0=0
Split split_time_mixing_4_out 1 2 time_mixing_4_out ffn_4_x_last ffn_4_x
LayerNorm layernorm_9 1 1 ffn_4_x ffn_4_xx 0=1024 1=0.00001 2=1
Split split_ffn_4_xx 1 3 ffn_4_xx ffn_4_xx_0 ffn_4_xx_1 state_14_out
BinaryOp sub_9 2 1 state_14_in ffn_4_xx_0 ffn_4_sx 0=1
Split split_ffn_4_sx 1 1 ffn_4_sx ffn_4_sx_0
MemoryData data_ffn_4_x_k 0 1 ffn_4_x_k 0=1024 21=0
BinaryOp mul_63 2 1 ffn_4_sx_0 ffn_4_x_k ffn_4_xk 0=2
BinaryOp add_61 2 1 ffn_4_xk ffn_4_xx_1 ffn_4_xxk 0=0
Gemm gemm_ffn_4_key 1 1 ffn_4_xxk ffn_4_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_4 1 1 ffn_4_key ffn_4_key_relu
UnaryOp square_ffn_4_key_relu 1 1 ffn_4_key_relu ffn_4_key_relu_square 0=4
Gemm gemm_ffn_4_value 1 1 ffn_4_key_relu_square ffn_4_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_62 2 1 ffn_4_value ffn_4_x_last channel_mixing_4_out 0=0
Split split_channel_mixing_4_out 1 2 channel_mixing_4_out att_5_x_last att_5_x
LayerNorm layernorm_10 1 1 att_5_x att_5_xx 0=1024 1=0.00001 2=1
Split split_att_5_xx 1 3 att_5_xx att_5_xx_0 att_5_xx_1 state_15_out
BinaryOp sub_10 2 1 state_15_in att_5_xx_0 att_5_sx 0=1
MemoryData data_att_5_maa 0 1 att_5_maa 0=1024 1=6 21=0
BinaryOp mul_64 2 1 att_5_sx att_5_maa att_5_maa_x 0=2
BinaryOp add_63 2 1 att_5_xx_1 att_5_maa_x att_5_maa_rwkvag 0=0
Slice slice_att_5_maa_rwkvag 1 6 att_5_maa_rwkvag att_5_mr att_5_mw att_5_mk att_5_mv att_5_ma att_5_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_5_mv 1 2 att_5_mv att_5_mv_0 att_5_mv_1
Gemm gemm_att_5_key 1 1 att_5_mk att_5_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_value 1 1 att_5_mv_0 att_5_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_receptance 1 1 att_5_mr att_5_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_ma_lora_0 1 1 att_5_ma att_5_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_5_ma_lora_1 1 1 att_5_ma_lora_0 att_5_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_5_a0 0 1 att_5_a0 0=1024 21=0
BinaryOp add_64 2 1 att_5_ma_lora_1 att_5_a0 att_5_ma_lora_2 0=0
Sigmoid sigmoid_19 1 1 att_5_ma_lora_2 att_5_a
Split split_att_5_a 1 2 att_5_a att_5_a_0 att_5_a_1
Gemm gemm_att_5_mg_lora 1 1 att_5_mg att_5_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_20 1 1 att_5_mg_lora att_5_mg_lora_sigmoid
Gemm gemm_att_5_gate 1 1 att_5_mg_lora_sigmoid att_5_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_5_k_k 0 1 att_5_k_k 0=1024 21=0
MemoryData data_att_5_k_a 0 1 att_5_k_a 0=1024 21=0
MemoryData data_att_5_r_k 0 1 att_5_r_k 0=1024 21=0
Split split_att_5_key 1 2 att_5_key att_5_key_0 att_5_key_1
BinaryOp mul_65 2 1 att_5_key_0 att_5_k_k att_5_key_k 0=2
Reshape reshape_60 1 1 att_5_key_k att_5_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_5 1 1 att_5_key_k_reshape att_5_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_61 1 1 att_5_key_k_norm att_5_key_k_norm_reshape 0=1024
BinaryOp add_65 1 1 att_5_a_0 att_5_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_66 2 1 att_5_a_0_minus att_5_k_a att_5_a_minus1_mulka 0=2
BinaryOp add_66 1 1 att_5_a_minus1_mulka att_5_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_67 2 1 att_5_a_minus1_mulka_plus1 att_5_key_1 att_5_key_a 0=2
Split split_att_5_value 1 2 att_5_value att_5_value_0 att_5_value_1
Split split_v_first_4 1 2 v_first_4 att_5_v_first v_first_5
BinaryOp minus_4 2 1 att_5_v_first att_5_value_1 att_5_vfirst_minus_value 0=1
Gemm gemm_att_5_mv_lora_0 1 1 att_5_mv_1 att_5_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_5_mv_lora_1 1 1 att_5_mv_lora_0 att_5_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_5_v0 0 1 att_5_v0 0=1024 21=0
BinaryOp add_67 2 1 att_5_mv_lora_1 att_5_v0 att_5_mv_lora_2 0=0
Sigmoid sigmoid_21 1 1 att_5_mv_lora_2 att_5_mv_lora_sigmoid
BinaryOp mul_68 2 1 att_5_mv_lora_sigmoid att_5_vfirst_minus_value att_5_vfirst_minus_value_mul 0=2
BinaryOp add_68 2 1 att_5_value_0 att_5_vfirst_minus_value_mul att_5_value_final 0=0
Gemm gemm_att_5_mw_lora 1 1 att_5_mw att_5_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_5_mw_lora 1 1 att_5_mw_lora att_5_mw_lora_tanh
Gemm gemm_att_5_mw_lora_tanh_linear 1 1 att_5_mw_lora_tanh att_5_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_5_td 0 1 att_5_td 0=1024 21=0
BinaryOp add_69 2 1 att_5_mw_lora_tanh_linear att_5_td att_5_time_decay_pre 0=0
Sigmoid sigmoid_22 1 1 att_5_time_decay_pre att_5_time_decay_sigmoid
Exp exp_5 1 1 att_5_time_decay_sigmoid att_5_time_decay 1=-0.606531
Reshape reshape_62 1 1 att_5_time_decay att_5_time_decay_reshape 0=64 1=1 2=16
Split split_att_5_key_a 1 2 att_5_key_a att_5_key_a_0 att_5_key_a_1
Reshape reshape_63 1 1 att_5_key_a_0 att_5_key_a_reshape 0=64 1=1 2=16
Split split_att_5_value_final 1 3 att_5_value_final att_5_value_final_0 att_5_value_final_1 att_5_value_final_2
Reshape reshape_64 1 1 att_5_value_final_0 att_5_value_final_0_reshape 0=64 1=16
Reshape reshape_65 1 1 att_5_value_final_1 att_5_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_5_vk 2 1 att_5_value_final_1_reshape att_5_key_a_reshape att_5_vk 0=0
Split split_att_5_key_k_norm_reshape 1 2 att_5_key_k_norm_reshape att_5_kk_0 att_5_kk_1
BinaryOp mul_69 1 1 att_5_kk_0 att_5_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_66 1 1 att_5_kk_0_neg att_5_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_70 2 1 att_5_kk_1 att_5_a_1 att_5_kk_a 0=2
Reshape reshape_67 1 1 att_5_kk_a att_5_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_5_ab 2 1 att_5_kk_0_neg_reshape att_5_kk_a_reshape att_5_ab 0=0
Split split_state_16_in 1 2 state_16_in att_5_state_prev_0 att_5_state_prev_1
BinaryOp mul_71 2 1 att_5_state_prev_0 att_5_time_decay_reshape att_5_state_td 0=2
MatMul matmul_att_5_sab 2 1 att_5_state_prev_1 att_5_ab att_5_sab 0=0
BinaryOp add_70 2 1 att_5_sab att_5_vk att_5_sab_vk 0=0
BinaryOp add_71 2 1 att_5_sab_vk att_5_state_td att_5_state_new 0=0
Split split_att_5_state_new 1 2 att_5_state_new att_5_state_new_0 state_16_out
Split split_att_5_receptance 1 2 att_5_receptance att_5_receptance_0 att_5_receptance_1
Reshape reshape_68 1 1 att_5_receptance_0 att_5_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_5_wkv_out 2 1 att_5_state_new_0 att_5_receptance_reshape att_5_wkv_out 0=0
Reshape reshape_69 1 1 att_5_wkv_out att_5_wkv_out_flatten 0=1024
GroupNorm groupnorm_5 1 1 att_5_wkv_out_flatten att_5_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_72 2 1 att_5_receptance_1 att_5_key_a_1 att_5_receptance_key_a 0=2
BinaryOp mul_73 2 1 att_5_receptance_key_a att_5_r_k att_5_rk 0=2
Reshape reshape_70 1 1 att_5_rk att_5_rk_reshape 0=64 1=16
Reduction sum_5 1 1 att_5_rk_reshape att_5_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_74 2 1 att_5_value_final_0_reshape att_5_rk_sum att_5_value_final_0_rk 0=2
Reshape reshape_71 1 1 att_5_value_final_0_rk att_5_value_final_0_rk_reshape 0=1024
BinaryOp add_72 2 1 att_5_value_final_0_rk_reshape att_5_x_gn att_5_x_gn_rkv 0=0
BinaryOp mul_75 2 1 att_5_x_gn_rkv att_5_gate att_5_x_gate 0=2
Gemm gemm_att_5_x_out 1 1 att_5_x_gate att_5_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_73 2 1 att_5_x_out att_5_x_last time_mixing_5_out 0=0
Split split_time_mixing_5_out 1 2 time_mixing_5_out ffn_5_x_last ffn_5_x
LayerNorm layernorm_11 1 1 ffn_5_x ffn_5_xx 0=1024 1=0.00001 2=1
Split split_ffn_5_xx 1 3 ffn_5_xx ffn_5_xx_0 ffn_5_xx_1 state_17_out
BinaryOp sub_11 2 1 state_17_in ffn_5_xx_0 ffn_5_sx 0=1
Split split_ffn_5_sx 1 1 ffn_5_sx ffn_5_sx_0
MemoryData data_ffn_5_x_k 0 1 ffn_5_x_k 0=1024 21=0
BinaryOp mul_76 2 1 ffn_5_sx_0 ffn_5_x_k ffn_5_xk 0=2
BinaryOp add_74 2 1 ffn_5_xk ffn_5_xx_1 ffn_5_xxk 0=0
Gemm gemm_ffn_5_key 1 1 ffn_5_xxk ffn_5_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_5 1 1 ffn_5_key ffn_5_key_relu
UnaryOp square_ffn_5_key_relu 1 1 ffn_5_key_relu ffn_5_key_relu_square 0=4
Gemm gemm_ffn_5_value 1 1 ffn_5_key_relu_square ffn_5_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_75 2 1 ffn_5_value ffn_5_x_last channel_mixing_5_out 0=0
Split split_channel_mixing_5_out 1 2 channel_mixing_5_out att_6_x_last att_6_x
LayerNorm layernorm_12 1 1 att_6_x att_6_xx 0=1024 1=0.00001 2=1
Split split_att_6_xx 1 3 att_6_xx att_6_xx_0 att_6_xx_1 state_18_out
BinaryOp sub_12 2 1 state_18_in att_6_xx_0 att_6_sx 0=1
MemoryData data_att_6_maa 0 1 att_6_maa 0=1024 1=6 21=0
BinaryOp mul_77 2 1 att_6_sx att_6_maa att_6_maa_x 0=2
BinaryOp add_76 2 1 att_6_xx_1 att_6_maa_x att_6_maa_rwkvag 0=0
Slice slice_att_6_maa_rwkvag 1 6 att_6_maa_rwkvag att_6_mr att_6_mw att_6_mk att_6_mv att_6_ma att_6_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_6_mv 1 2 att_6_mv att_6_mv_0 att_6_mv_1
Gemm gemm_att_6_key 1 1 att_6_mk att_6_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_value 1 1 att_6_mv_0 att_6_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_receptance 1 1 att_6_mr att_6_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_ma_lora_0 1 1 att_6_ma att_6_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_6_ma_lora_1 1 1 att_6_ma_lora_0 att_6_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_6_a0 0 1 att_6_a0 0=1024 21=0
BinaryOp add_77 2 1 att_6_ma_lora_1 att_6_a0 att_6_ma_lora_2 0=0
Sigmoid sigmoid_23 1 1 att_6_ma_lora_2 att_6_a
Split split_att_6_a 1 2 att_6_a att_6_a_0 att_6_a_1
Gemm gemm_att_6_mg_lora 1 1 att_6_mg att_6_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_24 1 1 att_6_mg_lora att_6_mg_lora_sigmoid
Gemm gemm_att_6_gate 1 1 att_6_mg_lora_sigmoid att_6_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_6_k_k 0 1 att_6_k_k 0=1024 21=0
MemoryData data_att_6_k_a 0 1 att_6_k_a 0=1024 21=0
MemoryData data_att_6_r_k 0 1 att_6_r_k 0=1024 21=0
Split split_att_6_key 1 2 att_6_key att_6_key_0 att_6_key_1
BinaryOp mul_78 2 1 att_6_key_0 att_6_k_k att_6_key_k 0=2
Reshape reshape_72 1 1 att_6_key_k att_6_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_6 1 1 att_6_key_k_reshape att_6_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_73 1 1 att_6_key_k_norm att_6_key_k_norm_reshape 0=1024
BinaryOp add_78 1 1 att_6_a_0 att_6_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_79 2 1 att_6_a_0_minus att_6_k_a att_6_a_minus1_mulka 0=2
BinaryOp add_79 1 1 att_6_a_minus1_mulka att_6_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_80 2 1 att_6_a_minus1_mulka_plus1 att_6_key_1 att_6_key_a 0=2
Split split_att_6_value 1 2 att_6_value att_6_value_0 att_6_value_1
Split split_v_first_5 1 2 v_first_5 att_6_v_first v_first_6
BinaryOp minus_5 2 1 att_6_v_first att_6_value_1 att_6_vfirst_minus_value 0=1
Gemm gemm_att_6_mv_lora_0 1 1 att_6_mv_1 att_6_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_6_mv_lora_1 1 1 att_6_mv_lora_0 att_6_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_6_v0 0 1 att_6_v0 0=1024 21=0
BinaryOp add_80 2 1 att_6_mv_lora_1 att_6_v0 att_6_mv_lora_2 0=0
Sigmoid sigmoid_25 1 1 att_6_mv_lora_2 att_6_mv_lora_sigmoid
BinaryOp mul_81 2 1 att_6_mv_lora_sigmoid att_6_vfirst_minus_value att_6_vfirst_minus_value_mul 0=2
BinaryOp add_81 2 1 att_6_value_0 att_6_vfirst_minus_value_mul att_6_value_final 0=0
Gemm gemm_att_6_mw_lora 1 1 att_6_mw att_6_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_6_mw_lora 1 1 att_6_mw_lora att_6_mw_lora_tanh
Gemm gemm_att_6_mw_lora_tanh_linear 1 1 att_6_mw_lora_tanh att_6_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_6_td 0 1 att_6_td 0=1024 21=0
BinaryOp add_82 2 1 att_6_mw_lora_tanh_linear att_6_td att_6_time_decay_pre 0=0
Sigmoid sigmoid_26 1 1 att_6_time_decay_pre att_6_time_decay_sigmoid
Exp exp_6 1 1 att_6_time_decay_sigmoid att_6_time_decay 1=-0.606531
Reshape reshape_74 1 1 att_6_time_decay att_6_time_decay_reshape 0=64 1=1 2=16
Split split_att_6_key_a 1 2 att_6_key_a att_6_key_a_0 att_6_key_a_1
Reshape reshape_75 1 1 att_6_key_a_0 att_6_key_a_reshape 0=64 1=1 2=16
Split split_att_6_value_final 1 3 att_6_value_final att_6_value_final_0 att_6_value_final_1 att_6_value_final_2
Reshape reshape_76 1 1 att_6_value_final_0 att_6_value_final_0_reshape 0=64 1=16
Reshape reshape_77 1 1 att_6_value_final_1 att_6_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_6_vk 2 1 att_6_value_final_1_reshape att_6_key_a_reshape att_6_vk 0=0
Split split_att_6_key_k_norm_reshape 1 2 att_6_key_k_norm_reshape att_6_kk_0 att_6_kk_1
BinaryOp mul_82 1 1 att_6_kk_0 att_6_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_78 1 1 att_6_kk_0_neg att_6_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_83 2 1 att_6_kk_1 att_6_a_1 att_6_kk_a 0=2
Reshape reshape_79 1 1 att_6_kk_a att_6_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_6_ab 2 1 att_6_kk_0_neg_reshape att_6_kk_a_reshape att_6_ab 0=0
Split split_state_19_in 1 2 state_19_in att_6_state_prev_0 att_6_state_prev_1
BinaryOp mul_84 2 1 att_6_state_prev_0 att_6_time_decay_reshape att_6_state_td 0=2
MatMul matmul_att_6_sab 2 1 att_6_state_prev_1 att_6_ab att_6_sab 0=0
BinaryOp add_83 2 1 att_6_sab att_6_vk att_6_sab_vk 0=0
BinaryOp add_84 2 1 att_6_sab_vk att_6_state_td att_6_state_new 0=0
Split split_att_6_state_new 1 2 att_6_state_new att_6_state_new_0 state_19_out
Split split_att_6_receptance 1 2 att_6_receptance att_6_receptance_0 att_6_receptance_1
Reshape reshape_80 1 1 att_6_receptance_0 att_6_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_6_wkv_out 2 1 att_6_state_new_0 att_6_receptance_reshape att_6_wkv_out 0=0
Reshape reshape_81 1 1 att_6_wkv_out att_6_wkv_out_flatten 0=1024
GroupNorm groupnorm_6 1 1 att_6_wkv_out_flatten att_6_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_85 2 1 att_6_receptance_1 att_6_key_a_1 att_6_receptance_key_a 0=2
BinaryOp mul_86 2 1 att_6_receptance_key_a att_6_r_k att_6_rk 0=2
Reshape reshape_82 1 1 att_6_rk att_6_rk_reshape 0=64 1=16
Reduction sum_6 1 1 att_6_rk_reshape att_6_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_87 2 1 att_6_value_final_0_reshape att_6_rk_sum att_6_value_final_0_rk 0=2
Reshape reshape_83 1 1 att_6_value_final_0_rk att_6_value_final_0_rk_reshape 0=1024
BinaryOp add_85 2 1 att_6_value_final_0_rk_reshape att_6_x_gn att_6_x_gn_rkv 0=0
BinaryOp mul_88 2 1 att_6_x_gn_rkv att_6_gate att_6_x_gate 0=2
Gemm gemm_att_6_x_out 1 1 att_6_x_gate att_6_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_86 2 1 att_6_x_out att_6_x_last time_mixing_6_out 0=0
Split split_time_mixing_6_out 1 2 time_mixing_6_out ffn_6_x_last ffn_6_x
LayerNorm layernorm_13 1 1 ffn_6_x ffn_6_xx 0=1024 1=0.00001 2=1
Split split_ffn_6_xx 1 3 ffn_6_xx ffn_6_xx_0 ffn_6_xx_1 state_20_out
BinaryOp sub_13 2 1 state_20_in ffn_6_xx_0 ffn_6_sx 0=1
Split split_ffn_6_sx 1 1 ffn_6_sx ffn_6_sx_0
MemoryData data_ffn_6_x_k 0 1 ffn_6_x_k 0=1024 21=0
BinaryOp mul_89 2 1 ffn_6_sx_0 ffn_6_x_k ffn_6_xk 0=2
BinaryOp add_87 2 1 ffn_6_xk ffn_6_xx_1 ffn_6_xxk 0=0
Gemm gemm_ffn_6_key 1 1 ffn_6_xxk ffn_6_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_6 1 1 ffn_6_key ffn_6_key_relu
UnaryOp square_ffn_6_key_relu 1 1 ffn_6_key_relu ffn_6_key_relu_square 0=4
Gemm gemm_ffn_6_value 1 1 ffn_6_key_relu_square ffn_6_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_88 2 1 ffn_6_value ffn_6_x_last channel_mixing_6_out 0=0
Split split_channel_mixing_6_out 1 2 channel_mixing_6_out att_7_x_last att_7_x
LayerNorm layernorm_14 1 1 att_7_x att_7_xx 0=1024 1=0.00001 2=1
Split split_att_7_xx 1 3 att_7_xx att_7_xx_0 att_7_xx_1 state_21_out
BinaryOp sub_14 2 1 state_21_in att_7_xx_0 att_7_sx 0=1
MemoryData data_att_7_maa 0 1 att_7_maa 0=1024 1=6 21=0
BinaryOp mul_90 2 1 att_7_sx att_7_maa att_7_maa_x 0=2
BinaryOp add_89 2 1 att_7_xx_1 att_7_maa_x att_7_maa_rwkvag 0=0
Slice slice_att_7_maa_rwkvag 1 6 att_7_maa_rwkvag att_7_mr att_7_mw att_7_mk att_7_mv att_7_ma att_7_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_7_mv 1 2 att_7_mv att_7_mv_0 att_7_mv_1
Gemm gemm_att_7_key 1 1 att_7_mk att_7_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_value 1 1 att_7_mv_0 att_7_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_receptance 1 1 att_7_mr att_7_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_ma_lora_0 1 1 att_7_ma att_7_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_7_ma_lora_1 1 1 att_7_ma_lora_0 att_7_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_7_a0 0 1 att_7_a0 0=1024 21=0
BinaryOp add_90 2 1 att_7_ma_lora_1 att_7_a0 att_7_ma_lora_2 0=0
Sigmoid sigmoid_27 1 1 att_7_ma_lora_2 att_7_a
Split split_att_7_a 1 2 att_7_a att_7_a_0 att_7_a_1
Gemm gemm_att_7_mg_lora 1 1 att_7_mg att_7_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_28 1 1 att_7_mg_lora att_7_mg_lora_sigmoid
Gemm gemm_att_7_gate 1 1 att_7_mg_lora_sigmoid att_7_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_7_k_k 0 1 att_7_k_k 0=1024 21=0
MemoryData data_att_7_k_a 0 1 att_7_k_a 0=1024 21=0
MemoryData data_att_7_r_k 0 1 att_7_r_k 0=1024 21=0
Split split_att_7_key 1 2 att_7_key att_7_key_0 att_7_key_1
BinaryOp mul_91 2 1 att_7_key_0 att_7_k_k att_7_key_k 0=2
Reshape reshape_84 1 1 att_7_key_k att_7_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_7 1 1 att_7_key_k_reshape att_7_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_85 1 1 att_7_key_k_norm att_7_key_k_norm_reshape 0=1024
BinaryOp add_91 1 1 att_7_a_0 att_7_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_92 2 1 att_7_a_0_minus att_7_k_a att_7_a_minus1_mulka 0=2
BinaryOp add_92 1 1 att_7_a_minus1_mulka att_7_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_93 2 1 att_7_a_minus1_mulka_plus1 att_7_key_1 att_7_key_a 0=2
Split split_att_7_value 1 2 att_7_value att_7_value_0 att_7_value_1
Split split_v_first_6 1 2 v_first_6 att_7_v_first v_first_7
BinaryOp minus_6 2 1 att_7_v_first att_7_value_1 att_7_vfirst_minus_value 0=1
Gemm gemm_att_7_mv_lora_0 1 1 att_7_mv_1 att_7_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_7_mv_lora_1 1 1 att_7_mv_lora_0 att_7_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_7_v0 0 1 att_7_v0 0=1024 21=0
BinaryOp add_93 2 1 att_7_mv_lora_1 att_7_v0 att_7_mv_lora_2 0=0
Sigmoid sigmoid_29 1 1 att_7_mv_lora_2 att_7_mv_lora_sigmoid
BinaryOp mul_94 2 1 att_7_mv_lora_sigmoid att_7_vfirst_minus_value att_7_vfirst_minus_value_mul 0=2
BinaryOp add_94 2 1 att_7_value_0 att_7_vfirst_minus_value_mul att_7_value_final 0=0
Gemm gemm_att_7_mw_lora 1 1 att_7_mw att_7_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_7_mw_lora 1 1 att_7_mw_lora att_7_mw_lora_tanh
Gemm gemm_att_7_mw_lora_tanh_linear 1 1 att_7_mw_lora_tanh att_7_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_7_td 0 1 att_7_td 0=1024 21=0
BinaryOp add_95 2 1 att_7_mw_lora_tanh_linear att_7_td att_7_time_decay_pre 0=0
Sigmoid sigmoid_30 1 1 att_7_time_decay_pre att_7_time_decay_sigmoid
Exp exp_7 1 1 att_7_time_decay_sigmoid att_7_time_decay 1=-0.606531
Reshape reshape_86 1 1 att_7_time_decay att_7_time_decay_reshape 0=64 1=1 2=16
Split split_att_7_key_a 1 2 att_7_key_a att_7_key_a_0 att_7_key_a_1
Reshape reshape_87 1 1 att_7_key_a_0 att_7_key_a_reshape 0=64 1=1 2=16
Split split_att_7_value_final 1 3 att_7_value_final att_7_value_final_0 att_7_value_final_1 att_7_value_final_2
Reshape reshape_88 1 1 att_7_value_final_0 att_7_value_final_0_reshape 0=64 1=16
Reshape reshape_89 1 1 att_7_value_final_1 att_7_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_7_vk 2 1 att_7_value_final_1_reshape att_7_key_a_reshape att_7_vk 0=0
Split split_att_7_key_k_norm_reshape 1 2 att_7_key_k_norm_reshape att_7_kk_0 att_7_kk_1
BinaryOp mul_95 1 1 att_7_kk_0 att_7_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_90 1 1 att_7_kk_0_neg att_7_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_96 2 1 att_7_kk_1 att_7_a_1 att_7_kk_a 0=2
Reshape reshape_91 1 1 att_7_kk_a att_7_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_7_ab 2 1 att_7_kk_0_neg_reshape att_7_kk_a_reshape att_7_ab 0=0
Split split_state_22_in 1 2 state_22_in att_7_state_prev_0 att_7_state_prev_1
BinaryOp mul_97 2 1 att_7_state_prev_0 att_7_time_decay_reshape att_7_state_td 0=2
MatMul matmul_att_7_sab 2 1 att_7_state_prev_1 att_7_ab att_7_sab 0=0
BinaryOp add_96 2 1 att_7_sab att_7_vk att_7_sab_vk 0=0
BinaryOp add_97 2 1 att_7_sab_vk att_7_state_td att_7_state_new 0=0
Split split_att_7_state_new 1 2 att_7_state_new att_7_state_new_0 state_22_out
Split split_att_7_receptance 1 2 att_7_receptance att_7_receptance_0 att_7_receptance_1
Reshape reshape_92 1 1 att_7_receptance_0 att_7_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_7_wkv_out 2 1 att_7_state_new_0 att_7_receptance_reshape att_7_wkv_out 0=0
Reshape reshape_93 1 1 att_7_wkv_out att_7_wkv_out_flatten 0=1024
GroupNorm groupnorm_7 1 1 att_7_wkv_out_flatten att_7_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_98 2 1 att_7_receptance_1 att_7_key_a_1 att_7_receptance_key_a 0=2
BinaryOp mul_99 2 1 att_7_receptance_key_a att_7_r_k att_7_rk 0=2
Reshape reshape_94 1 1 att_7_rk att_7_rk_reshape 0=64 1=16
Reduction sum_7 1 1 att_7_rk_reshape att_7_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_100 2 1 att_7_value_final_0_reshape att_7_rk_sum att_7_value_final_0_rk 0=2
Reshape reshape_95 1 1 att_7_value_final_0_rk att_7_value_final_0_rk_reshape 0=1024
BinaryOp add_98 2 1 att_7_value_final_0_rk_reshape att_7_x_gn att_7_x_gn_rkv 0=0
BinaryOp mul_101 2 1 att_7_x_gn_rkv att_7_gate att_7_x_gate 0=2
Gemm gemm_att_7_x_out 1 1 att_7_x_gate att_7_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_99 2 1 att_7_x_out att_7_x_last time_mixing_7_out 0=0
Split split_time_mixing_7_out 1 2 time_mixing_7_out ffn_7_x_last ffn_7_x
LayerNorm layernorm_15 1 1 ffn_7_x ffn_7_xx 0=1024 1=0.00001 2=1
Split split_ffn_7_xx 1 3 ffn_7_xx ffn_7_xx_0 ffn_7_xx_1 state_23_out
BinaryOp sub_15 2 1 state_23_in ffn_7_xx_0 ffn_7_sx 0=1
Split split_ffn_7_sx 1 1 ffn_7_sx ffn_7_sx_0
MemoryData data_ffn_7_x_k 0 1 ffn_7_x_k 0=1024 21=0
BinaryOp mul_102 2 1 ffn_7_sx_0 ffn_7_x_k ffn_7_xk 0=2
BinaryOp add_100 2 1 ffn_7_xk ffn_7_xx_1 ffn_7_xxk 0=0
Gemm gemm_ffn_7_key 1 1 ffn_7_xxk ffn_7_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_7 1 1 ffn_7_key ffn_7_key_relu
UnaryOp square_ffn_7_key_relu 1 1 ffn_7_key_relu ffn_7_key_relu_square 0=4
Gemm gemm_ffn_7_value 1 1 ffn_7_key_relu_square ffn_7_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_101 2 1 ffn_7_value ffn_7_x_last channel_mixing_7_out 0=0
Split split_channel_mixing_7_out 1 2 channel_mixing_7_out att_8_x_last att_8_x
LayerNorm layernorm_16 1 1 att_8_x att_8_xx 0=1024 1=0.00001 2=1
Split split_att_8_xx 1 3 att_8_xx att_8_xx_0 att_8_xx_1 state_24_out
BinaryOp sub_16 2 1 state_24_in att_8_xx_0 att_8_sx 0=1
MemoryData data_att_8_maa 0 1 att_8_maa 0=1024 1=6 21=0
BinaryOp mul_103 2 1 att_8_sx att_8_maa att_8_maa_x 0=2
BinaryOp add_102 2 1 att_8_xx_1 att_8_maa_x att_8_maa_rwkvag 0=0
Slice slice_att_8_maa_rwkvag 1 6 att_8_maa_rwkvag att_8_mr att_8_mw att_8_mk att_8_mv att_8_ma att_8_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_8_mv 1 2 att_8_mv att_8_mv_0 att_8_mv_1
Gemm gemm_att_8_key 1 1 att_8_mk att_8_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_value 1 1 att_8_mv_0 att_8_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_receptance 1 1 att_8_mr att_8_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_ma_lora_0 1 1 att_8_ma att_8_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_8_ma_lora_1 1 1 att_8_ma_lora_0 att_8_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_8_a0 0 1 att_8_a0 0=1024 21=0
BinaryOp add_103 2 1 att_8_ma_lora_1 att_8_a0 att_8_ma_lora_2 0=0
Sigmoid sigmoid_31 1 1 att_8_ma_lora_2 att_8_a
Split split_att_8_a 1 2 att_8_a att_8_a_0 att_8_a_1
Gemm gemm_att_8_mg_lora 1 1 att_8_mg att_8_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_32 1 1 att_8_mg_lora att_8_mg_lora_sigmoid
Gemm gemm_att_8_gate 1 1 att_8_mg_lora_sigmoid att_8_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_8_k_k 0 1 att_8_k_k 0=1024 21=0
MemoryData data_att_8_k_a 0 1 att_8_k_a 0=1024 21=0
MemoryData data_att_8_r_k 0 1 att_8_r_k 0=1024 21=0
Split split_att_8_key 1 2 att_8_key att_8_key_0 att_8_key_1
BinaryOp mul_104 2 1 att_8_key_0 att_8_k_k att_8_key_k 0=2
Reshape reshape_96 1 1 att_8_key_k att_8_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_8 1 1 att_8_key_k_reshape att_8_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_97 1 1 att_8_key_k_norm att_8_key_k_norm_reshape 0=1024
BinaryOp add_104 1 1 att_8_a_0 att_8_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_105 2 1 att_8_a_0_minus att_8_k_a att_8_a_minus1_mulka 0=2
BinaryOp add_105 1 1 att_8_a_minus1_mulka att_8_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_106 2 1 att_8_a_minus1_mulka_plus1 att_8_key_1 att_8_key_a 0=2
Split split_att_8_value 1 2 att_8_value att_8_value_0 att_8_value_1
Split split_v_first_7 1 2 v_first_7 att_8_v_first v_first_8
BinaryOp minus_7 2 1 att_8_v_first att_8_value_1 att_8_vfirst_minus_value 0=1
Gemm gemm_att_8_mv_lora_0 1 1 att_8_mv_1 att_8_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_8_mv_lora_1 1 1 att_8_mv_lora_0 att_8_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_8_v0 0 1 att_8_v0 0=1024 21=0
BinaryOp add_106 2 1 att_8_mv_lora_1 att_8_v0 att_8_mv_lora_2 0=0
Sigmoid sigmoid_33 1 1 att_8_mv_lora_2 att_8_mv_lora_sigmoid
BinaryOp mul_107 2 1 att_8_mv_lora_sigmoid att_8_vfirst_minus_value att_8_vfirst_minus_value_mul 0=2
BinaryOp add_107 2 1 att_8_value_0 att_8_vfirst_minus_value_mul att_8_value_final 0=0
Gemm gemm_att_8_mw_lora 1 1 att_8_mw att_8_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_8_mw_lora 1 1 att_8_mw_lora att_8_mw_lora_tanh
Gemm gemm_att_8_mw_lora_tanh_linear 1 1 att_8_mw_lora_tanh att_8_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_8_td 0 1 att_8_td 0=1024 21=0
BinaryOp add_108 2 1 att_8_mw_lora_tanh_linear att_8_td att_8_time_decay_pre 0=0
Sigmoid sigmoid_34 1 1 att_8_time_decay_pre att_8_time_decay_sigmoid
Exp exp_8 1 1 att_8_time_decay_sigmoid att_8_time_decay 1=-0.606531
Reshape reshape_98 1 1 att_8_time_decay att_8_time_decay_reshape 0=64 1=1 2=16
Split split_att_8_key_a 1 2 att_8_key_a att_8_key_a_0 att_8_key_a_1
Reshape reshape_99 1 1 att_8_key_a_0 att_8_key_a_reshape 0=64 1=1 2=16
Split split_att_8_value_final 1 3 att_8_value_final att_8_value_final_0 att_8_value_final_1 att_8_value_final_2
Reshape reshape_100 1 1 att_8_value_final_0 att_8_value_final_0_reshape 0=64 1=16
Reshape reshape_101 1 1 att_8_value_final_1 att_8_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_8_vk 2 1 att_8_value_final_1_reshape att_8_key_a_reshape att_8_vk 0=0
Split split_att_8_key_k_norm_reshape 1 2 att_8_key_k_norm_reshape att_8_kk_0 att_8_kk_1
BinaryOp mul_108 1 1 att_8_kk_0 att_8_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_102 1 1 att_8_kk_0_neg att_8_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_109 2 1 att_8_kk_1 att_8_a_1 att_8_kk_a 0=2
Reshape reshape_103 1 1 att_8_kk_a att_8_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_8_ab 2 1 att_8_kk_0_neg_reshape att_8_kk_a_reshape att_8_ab 0=0
Split split_state_25_in 1 2 state_25_in att_8_state_prev_0 att_8_state_prev_1
BinaryOp mul_110 2 1 att_8_state_prev_0 att_8_time_decay_reshape att_8_state_td 0=2
MatMul matmul_att_8_sab 2 1 att_8_state_prev_1 att_8_ab att_8_sab 0=0
BinaryOp add_109 2 1 att_8_sab att_8_vk att_8_sab_vk 0=0
BinaryOp add_110 2 1 att_8_sab_vk att_8_state_td att_8_state_new 0=0
Split split_att_8_state_new 1 2 att_8_state_new att_8_state_new_0 state_25_out
Split split_att_8_receptance 1 2 att_8_receptance att_8_receptance_0 att_8_receptance_1
Reshape reshape_104 1 1 att_8_receptance_0 att_8_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_8_wkv_out 2 1 att_8_state_new_0 att_8_receptance_reshape att_8_wkv_out 0=0
Reshape reshape_105 1 1 att_8_wkv_out att_8_wkv_out_flatten 0=1024
GroupNorm groupnorm_8 1 1 att_8_wkv_out_flatten att_8_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_111 2 1 att_8_receptance_1 att_8_key_a_1 att_8_receptance_key_a 0=2
BinaryOp mul_112 2 1 att_8_receptance_key_a att_8_r_k att_8_rk 0=2
Reshape reshape_106 1 1 att_8_rk att_8_rk_reshape 0=64 1=16
Reduction sum_8 1 1 att_8_rk_reshape att_8_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_113 2 1 att_8_value_final_0_reshape att_8_rk_sum att_8_value_final_0_rk 0=2
Reshape reshape_107 1 1 att_8_value_final_0_rk att_8_value_final_0_rk_reshape 0=1024
BinaryOp add_111 2 1 att_8_value_final_0_rk_reshape att_8_x_gn att_8_x_gn_rkv 0=0
BinaryOp mul_114 2 1 att_8_x_gn_rkv att_8_gate att_8_x_gate 0=2
Gemm gemm_att_8_x_out 1 1 att_8_x_gate att_8_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_112 2 1 att_8_x_out att_8_x_last time_mixing_8_out 0=0
Split split_time_mixing_8_out 1 2 time_mixing_8_out ffn_8_x_last ffn_8_x
LayerNorm layernorm_17 1 1 ffn_8_x ffn_8_xx 0=1024 1=0.00001 2=1
Split split_ffn_8_xx 1 3 ffn_8_xx ffn_8_xx_0 ffn_8_xx_1 state_26_out
BinaryOp sub_17 2 1 state_26_in ffn_8_xx_0 ffn_8_sx 0=1
Split split_ffn_8_sx 1 1 ffn_8_sx ffn_8_sx_0
MemoryData data_ffn_8_x_k 0 1 ffn_8_x_k 0=1024 21=0
BinaryOp mul_115 2 1 ffn_8_sx_0 ffn_8_x_k ffn_8_xk 0=2
BinaryOp add_113 2 1 ffn_8_xk ffn_8_xx_1 ffn_8_xxk 0=0
Gemm gemm_ffn_8_key 1 1 ffn_8_xxk ffn_8_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_8 1 1 ffn_8_key ffn_8_key_relu
UnaryOp square_ffn_8_key_relu 1 1 ffn_8_key_relu ffn_8_key_relu_square 0=4
Gemm gemm_ffn_8_value 1 1 ffn_8_key_relu_square ffn_8_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_114 2 1 ffn_8_value ffn_8_x_last channel_mixing_8_out 0=0
Split split_channel_mixing_8_out 1 2 channel_mixing_8_out att_9_x_last att_9_x
LayerNorm layernorm_18 1 1 att_9_x att_9_xx 0=1024 1=0.00001 2=1
Split split_att_9_xx 1 3 att_9_xx att_9_xx_0 att_9_xx_1 state_27_out
BinaryOp sub_18 2 1 state_27_in att_9_xx_0 att_9_sx 0=1
MemoryData data_att_9_maa 0 1 att_9_maa 0=1024 1=6 21=0
BinaryOp mul_116 2 1 att_9_sx att_9_maa att_9_maa_x 0=2
BinaryOp add_115 2 1 att_9_xx_1 att_9_maa_x att_9_maa_rwkvag 0=0
Slice slice_att_9_maa_rwkvag 1 6 att_9_maa_rwkvag att_9_mr att_9_mw att_9_mk att_9_mv att_9_ma att_9_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_9_mv 1 2 att_9_mv att_9_mv_0 att_9_mv_1
Gemm gemm_att_9_key 1 1 att_9_mk att_9_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_value 1 1 att_9_mv_0 att_9_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_receptance 1 1 att_9_mr att_9_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_ma_lora_0 1 1 att_9_ma att_9_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_9_ma_lora_1 1 1 att_9_ma_lora_0 att_9_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_9_a0 0 1 att_9_a0 0=1024 21=0
BinaryOp add_116 2 1 att_9_ma_lora_1 att_9_a0 att_9_ma_lora_2 0=0
Sigmoid sigmoid_35 1 1 att_9_ma_lora_2 att_9_a
Split split_att_9_a 1 2 att_9_a att_9_a_0 att_9_a_1
Gemm gemm_att_9_mg_lora 1 1 att_9_mg att_9_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_36 1 1 att_9_mg_lora att_9_mg_lora_sigmoid
Gemm gemm_att_9_gate 1 1 att_9_mg_lora_sigmoid att_9_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_9_k_k 0 1 att_9_k_k 0=1024 21=0
MemoryData data_att_9_k_a 0 1 att_9_k_a 0=1024 21=0
MemoryData data_att_9_r_k 0 1 att_9_r_k 0=1024 21=0
Split split_att_9_key 1 2 att_9_key att_9_key_0 att_9_key_1
BinaryOp mul_117 2 1 att_9_key_0 att_9_k_k att_9_key_k 0=2
Reshape reshape_108 1 1 att_9_key_k att_9_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_9 1 1 att_9_key_k_reshape att_9_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_109 1 1 att_9_key_k_norm att_9_key_k_norm_reshape 0=1024
BinaryOp add_117 1 1 att_9_a_0 att_9_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_118 2 1 att_9_a_0_minus att_9_k_a att_9_a_minus1_mulka 0=2
BinaryOp add_118 1 1 att_9_a_minus1_mulka att_9_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_119 2 1 att_9_a_minus1_mulka_plus1 att_9_key_1 att_9_key_a 0=2
Split split_att_9_value 1 2 att_9_value att_9_value_0 att_9_value_1
Split split_v_first_8 1 2 v_first_8 att_9_v_first v_first_9
BinaryOp minus_8 2 1 att_9_v_first att_9_value_1 att_9_vfirst_minus_value 0=1
Gemm gemm_att_9_mv_lora_0 1 1 att_9_mv_1 att_9_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_9_mv_lora_1 1 1 att_9_mv_lora_0 att_9_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_9_v0 0 1 att_9_v0 0=1024 21=0
BinaryOp add_119 2 1 att_9_mv_lora_1 att_9_v0 att_9_mv_lora_2 0=0
Sigmoid sigmoid_37 1 1 att_9_mv_lora_2 att_9_mv_lora_sigmoid
BinaryOp mul_120 2 1 att_9_mv_lora_sigmoid att_9_vfirst_minus_value att_9_vfirst_minus_value_mul 0=2
BinaryOp add_120 2 1 att_9_value_0 att_9_vfirst_minus_value_mul att_9_value_final 0=0
Gemm gemm_att_9_mw_lora 1 1 att_9_mw att_9_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_9_mw_lora 1 1 att_9_mw_lora att_9_mw_lora_tanh
Gemm gemm_att_9_mw_lora_tanh_linear 1 1 att_9_mw_lora_tanh att_9_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_9_td 0 1 att_9_td 0=1024 21=0
BinaryOp add_121 2 1 att_9_mw_lora_tanh_linear att_9_td att_9_time_decay_pre 0=0
Sigmoid sigmoid_38 1 1 att_9_time_decay_pre att_9_time_decay_sigmoid
Exp exp_9 1 1 att_9_time_decay_sigmoid att_9_time_decay 1=-0.606531
Reshape reshape_110 1 1 att_9_time_decay att_9_time_decay_reshape 0=64 1=1 2=16
Split split_att_9_key_a 1 2 att_9_key_a att_9_key_a_0 att_9_key_a_1
Reshape reshape_111 1 1 att_9_key_a_0 att_9_key_a_reshape 0=64 1=1 2=16
Split split_att_9_value_final 1 3 att_9_value_final att_9_value_final_0 att_9_value_final_1 att_9_value_final_2
Reshape reshape_112 1 1 att_9_value_final_0 att_9_value_final_0_reshape 0=64 1=16
Reshape reshape_113 1 1 att_9_value_final_1 att_9_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_9_vk 2 1 att_9_value_final_1_reshape att_9_key_a_reshape att_9_vk 0=0
Split split_att_9_key_k_norm_reshape 1 2 att_9_key_k_norm_reshape att_9_kk_0 att_9_kk_1
BinaryOp mul_121 1 1 att_9_kk_0 att_9_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_114 1 1 att_9_kk_0_neg att_9_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_122 2 1 att_9_kk_1 att_9_a_1 att_9_kk_a 0=2
Reshape reshape_115 1 1 att_9_kk_a att_9_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_9_ab 2 1 att_9_kk_0_neg_reshape att_9_kk_a_reshape att_9_ab 0=0
Split split_state_28_in 1 2 state_28_in att_9_state_prev_0 att_9_state_prev_1
BinaryOp mul_123 2 1 att_9_state_prev_0 att_9_time_decay_reshape att_9_state_td 0=2
MatMul matmul_att_9_sab 2 1 att_9_state_prev_1 att_9_ab att_9_sab 0=0
BinaryOp add_122 2 1 att_9_sab att_9_vk att_9_sab_vk 0=0
BinaryOp add_123 2 1 att_9_sab_vk att_9_state_td att_9_state_new 0=0
Split split_att_9_state_new 1 2 att_9_state_new att_9_state_new_0 state_28_out
Split split_att_9_receptance 1 2 att_9_receptance att_9_receptance_0 att_9_receptance_1
Reshape reshape_116 1 1 att_9_receptance_0 att_9_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_9_wkv_out 2 1 att_9_state_new_0 att_9_receptance_reshape att_9_wkv_out 0=0
Reshape reshape_117 1 1 att_9_wkv_out att_9_wkv_out_flatten 0=1024
GroupNorm groupnorm_9 1 1 att_9_wkv_out_flatten att_9_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_124 2 1 att_9_receptance_1 att_9_key_a_1 att_9_receptance_key_a 0=2
BinaryOp mul_125 2 1 att_9_receptance_key_a att_9_r_k att_9_rk 0=2
Reshape reshape_118 1 1 att_9_rk att_9_rk_reshape 0=64 1=16
Reduction sum_9 1 1 att_9_rk_reshape att_9_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_126 2 1 att_9_value_final_0_reshape att_9_rk_sum att_9_value_final_0_rk 0=2
Reshape reshape_119 1 1 att_9_value_final_0_rk att_9_value_final_0_rk_reshape 0=1024
BinaryOp add_124 2 1 att_9_value_final_0_rk_reshape att_9_x_gn att_9_x_gn_rkv 0=0
BinaryOp mul_127 2 1 att_9_x_gn_rkv att_9_gate att_9_x_gate 0=2
Gemm gemm_att_9_x_out 1 1 att_9_x_gate att_9_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_125 2 1 att_9_x_out att_9_x_last time_mixing_9_out 0=0
Split split_time_mixing_9_out 1 2 time_mixing_9_out ffn_9_x_last ffn_9_x
LayerNorm layernorm_19 1 1 ffn_9_x ffn_9_xx 0=1024 1=0.00001 2=1
Split split_ffn_9_xx 1 3 ffn_9_xx ffn_9_xx_0 ffn_9_xx_1 state_29_out
BinaryOp sub_19 2 1 state_29_in ffn_9_xx_0 ffn_9_sx 0=1
Split split_ffn_9_sx 1 1 ffn_9_sx ffn_9_sx_0
MemoryData data_ffn_9_x_k 0 1 ffn_9_x_k 0=1024 21=0
BinaryOp mul_128 2 1 ffn_9_sx_0 ffn_9_x_k ffn_9_xk 0=2
BinaryOp add_126 2 1 ffn_9_xk ffn_9_xx_1 ffn_9_xxk 0=0
Gemm gemm_ffn_9_key 1 1 ffn_9_xxk ffn_9_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_9 1 1 ffn_9_key ffn_9_key_relu
UnaryOp square_ffn_9_key_relu 1 1 ffn_9_key_relu ffn_9_key_relu_square 0=4
Gemm gemm_ffn_9_value 1 1 ffn_9_key_relu_square ffn_9_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_127 2 1 ffn_9_value ffn_9_x_last channel_mixing_9_out 0=0
Split split_channel_mixing_9_out 1 2 channel_mixing_9_out att_10_x_last att_10_x
LayerNorm layernorm_20 1 1 att_10_x att_10_xx 0=1024 1=0.00001 2=1
Split split_att_10_xx 1 3 att_10_xx att_10_xx_0 att_10_xx_1 state_30_out
BinaryOp sub_20 2 1 state_30_in att_10_xx_0 att_10_sx 0=1
MemoryData data_att_10_maa 0 1 att_10_maa 0=1024 1=6 21=0
BinaryOp mul_129 2 1 att_10_sx att_10_maa att_10_maa_x 0=2
BinaryOp add_128 2 1 att_10_xx_1 att_10_maa_x att_10_maa_rwkvag 0=0
Slice slice_att_10_maa_rwkvag 1 6 att_10_maa_rwkvag att_10_mr att_10_mw att_10_mk att_10_mv att_10_ma att_10_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_10_mv 1 2 att_10_mv att_10_mv_0 att_10_mv_1
Gemm gemm_att_10_key 1 1 att_10_mk att_10_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_value 1 1 att_10_mv_0 att_10_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_receptance 1 1 att_10_mr att_10_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_ma_lora_0 1 1 att_10_ma att_10_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_10_ma_lora_1 1 1 att_10_ma_lora_0 att_10_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_10_a0 0 1 att_10_a0 0=1024 21=0
BinaryOp add_129 2 1 att_10_ma_lora_1 att_10_a0 att_10_ma_lora_2 0=0
Sigmoid sigmoid_39 1 1 att_10_ma_lora_2 att_10_a
Split split_att_10_a 1 2 att_10_a att_10_a_0 att_10_a_1
Gemm gemm_att_10_mg_lora 1 1 att_10_mg att_10_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_40 1 1 att_10_mg_lora att_10_mg_lora_sigmoid
Gemm gemm_att_10_gate 1 1 att_10_mg_lora_sigmoid att_10_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_10_k_k 0 1 att_10_k_k 0=1024 21=0
MemoryData data_att_10_k_a 0 1 att_10_k_a 0=1024 21=0
MemoryData data_att_10_r_k 0 1 att_10_r_k 0=1024 21=0
Split split_att_10_key 1 2 att_10_key att_10_key_0 att_10_key_1
BinaryOp mul_130 2 1 att_10_key_0 att_10_k_k att_10_key_k 0=2
Reshape reshape_120 1 1 att_10_key_k att_10_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_10 1 1 att_10_key_k_reshape att_10_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_121 1 1 att_10_key_k_norm att_10_key_k_norm_reshape 0=1024
BinaryOp add_130 1 1 att_10_a_0 att_10_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_131 2 1 att_10_a_0_minus att_10_k_a att_10_a_minus1_mulka 0=2
BinaryOp add_131 1 1 att_10_a_minus1_mulka att_10_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_132 2 1 att_10_a_minus1_mulka_plus1 att_10_key_1 att_10_key_a 0=2
Split split_att_10_value 1 2 att_10_value att_10_value_0 att_10_value_1
Split split_v_first_9 1 2 v_first_9 att_10_v_first v_first_10
BinaryOp minus_9 2 1 att_10_v_first att_10_value_1 att_10_vfirst_minus_value 0=1
Gemm gemm_att_10_mv_lora_0 1 1 att_10_mv_1 att_10_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_10_mv_lora_1 1 1 att_10_mv_lora_0 att_10_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_10_v0 0 1 att_10_v0 0=1024 21=0
BinaryOp add_132 2 1 att_10_mv_lora_1 att_10_v0 att_10_mv_lora_2 0=0
Sigmoid sigmoid_41 1 1 att_10_mv_lora_2 att_10_mv_lora_sigmoid
BinaryOp mul_133 2 1 att_10_mv_lora_sigmoid att_10_vfirst_minus_value att_10_vfirst_minus_value_mul 0=2
BinaryOp add_133 2 1 att_10_value_0 att_10_vfirst_minus_value_mul att_10_value_final 0=0
Gemm gemm_att_10_mw_lora 1 1 att_10_mw att_10_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_10_mw_lora 1 1 att_10_mw_lora att_10_mw_lora_tanh
Gemm gemm_att_10_mw_lora_tanh_linear 1 1 att_10_mw_lora_tanh att_10_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_10_td 0 1 att_10_td 0=1024 21=0
BinaryOp add_134 2 1 att_10_mw_lora_tanh_linear att_10_td att_10_time_decay_pre 0=0
Sigmoid sigmoid_42 1 1 att_10_time_decay_pre att_10_time_decay_sigmoid
Exp exp_10 1 1 att_10_time_decay_sigmoid att_10_time_decay 1=-0.606531
Reshape reshape_122 1 1 att_10_time_decay att_10_time_decay_reshape 0=64 1=1 2=16
Split split_att_10_key_a 1 2 att_10_key_a att_10_key_a_0 att_10_key_a_1
Reshape reshape_123 1 1 att_10_key_a_0 att_10_key_a_reshape 0=64 1=1 2=16
Split split_att_10_value_final 1 3 att_10_value_final att_10_value_final_0 att_10_value_final_1 att_10_value_final_2
Reshape reshape_124 1 1 att_10_value_final_0 att_10_value_final_0_reshape 0=64 1=16
Reshape reshape_125 1 1 att_10_value_final_1 att_10_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_10_vk 2 1 att_10_value_final_1_reshape att_10_key_a_reshape att_10_vk 0=0
Split split_att_10_key_k_norm_reshape 1 2 att_10_key_k_norm_reshape att_10_kk_0 att_10_kk_1
BinaryOp mul_134 1 1 att_10_kk_0 att_10_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_126 1 1 att_10_kk_0_neg att_10_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_135 2 1 att_10_kk_1 att_10_a_1 att_10_kk_a 0=2
Reshape reshape_127 1 1 att_10_kk_a att_10_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_10_ab 2 1 att_10_kk_0_neg_reshape att_10_kk_a_reshape att_10_ab 0=0
Split split_state_31_in 1 2 state_31_in att_10_state_prev_0 att_10_state_prev_1
BinaryOp mul_136 2 1 att_10_state_prev_0 att_10_time_decay_reshape att_10_state_td 0=2
MatMul matmul_att_10_sab 2 1 att_10_state_prev_1 att_10_ab att_10_sab 0=0
BinaryOp add_135 2 1 att_10_sab att_10_vk att_10_sab_vk 0=0
BinaryOp add_136 2 1 att_10_sab_vk att_10_state_td att_10_state_new 0=0
Split split_att_10_state_new 1 2 att_10_state_new att_10_state_new_0 state_31_out
Split split_att_10_receptance 1 2 att_10_receptance att_10_receptance_0 att_10_receptance_1
Reshape reshape_128 1 1 att_10_receptance_0 att_10_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_10_wkv_out 2 1 att_10_state_new_0 att_10_receptance_reshape att_10_wkv_out 0=0
Reshape reshape_129 1 1 att_10_wkv_out att_10_wkv_out_flatten 0=1024
GroupNorm groupnorm_10 1 1 att_10_wkv_out_flatten att_10_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_137 2 1 att_10_receptance_1 att_10_key_a_1 att_10_receptance_key_a 0=2
BinaryOp mul_138 2 1 att_10_receptance_key_a att_10_r_k att_10_rk 0=2
Reshape reshape_130 1 1 att_10_rk att_10_rk_reshape 0=64 1=16
Reduction sum_10 1 1 att_10_rk_reshape att_10_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_139 2 1 att_10_value_final_0_reshape att_10_rk_sum att_10_value_final_0_rk 0=2
Reshape reshape_131 1 1 att_10_value_final_0_rk att_10_value_final_0_rk_reshape 0=1024
BinaryOp add_137 2 1 att_10_value_final_0_rk_reshape att_10_x_gn att_10_x_gn_rkv 0=0
BinaryOp mul_140 2 1 att_10_x_gn_rkv att_10_gate att_10_x_gate 0=2
Gemm gemm_att_10_x_out 1 1 att_10_x_gate att_10_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_138 2 1 att_10_x_out att_10_x_last time_mixing_10_out 0=0
Split split_time_mixing_10_out 1 2 time_mixing_10_out ffn_10_x_last ffn_10_x
LayerNorm layernorm_21 1 1 ffn_10_x ffn_10_xx 0=1024 1=0.00001 2=1
Split split_ffn_10_xx 1 3 ffn_10_xx ffn_10_xx_0 ffn_10_xx_1 state_32_out
BinaryOp sub_21 2 1 state_32_in ffn_10_xx_0 ffn_10_sx 0=1
Split split_ffn_10_sx 1 1 ffn_10_sx ffn_10_sx_0
MemoryData data_ffn_10_x_k 0 1 ffn_10_x_k 0=1024 21=0
BinaryOp mul_141 2 1 ffn_10_sx_0 ffn_10_x_k ffn_10_xk 0=2
BinaryOp add_139 2 1 ffn_10_xk ffn_10_xx_1 ffn_10_xxk 0=0
Gemm gemm_ffn_10_key 1 1 ffn_10_xxk ffn_10_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_10 1 1 ffn_10_key ffn_10_key_relu
UnaryOp square_ffn_10_key_relu 1 1 ffn_10_key_relu ffn_10_key_relu_square 0=4
Gemm gemm_ffn_10_value 1 1 ffn_10_key_relu_square ffn_10_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_140 2 1 ffn_10_value ffn_10_x_last channel_mixing_10_out 0=0
Split split_channel_mixing_10_out 1 2 channel_mixing_10_out att_11_x_last att_11_x
LayerNorm layernorm_22 1 1 att_11_x att_11_xx 0=1024 1=0.00001 2=1
Split split_att_11_xx 1 3 att_11_xx att_11_xx_0 att_11_xx_1 state_33_out
BinaryOp sub_22 2 1 state_33_in att_11_xx_0 att_11_sx 0=1
MemoryData data_att_11_maa 0 1 att_11_maa 0=1024 1=6 21=0
BinaryOp mul_142 2 1 att_11_sx att_11_maa att_11_maa_x 0=2
BinaryOp add_141 2 1 att_11_xx_1 att_11_maa_x att_11_maa_rwkvag 0=0
Slice slice_att_11_maa_rwkvag 1 6 att_11_maa_rwkvag att_11_mr att_11_mw att_11_mk att_11_mv att_11_ma att_11_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_11_mv 1 2 att_11_mv att_11_mv_0 att_11_mv_1
Gemm gemm_att_11_key 1 1 att_11_mk att_11_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_value 1 1 att_11_mv_0 att_11_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_receptance 1 1 att_11_mr att_11_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_ma_lora_0 1 1 att_11_ma att_11_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_11_ma_lora_1 1 1 att_11_ma_lora_0 att_11_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_11_a0 0 1 att_11_a0 0=1024 21=0
BinaryOp add_142 2 1 att_11_ma_lora_1 att_11_a0 att_11_ma_lora_2 0=0
Sigmoid sigmoid_43 1 1 att_11_ma_lora_2 att_11_a
Split split_att_11_a 1 2 att_11_a att_11_a_0 att_11_a_1
Gemm gemm_att_11_mg_lora 1 1 att_11_mg att_11_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_44 1 1 att_11_mg_lora att_11_mg_lora_sigmoid
Gemm gemm_att_11_gate 1 1 att_11_mg_lora_sigmoid att_11_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_11_k_k 0 1 att_11_k_k 0=1024 21=0
MemoryData data_att_11_k_a 0 1 att_11_k_a 0=1024 21=0
MemoryData data_att_11_r_k 0 1 att_11_r_k 0=1024 21=0
Split split_att_11_key 1 2 att_11_key att_11_key_0 att_11_key_1
BinaryOp mul_143 2 1 att_11_key_0 att_11_k_k att_11_key_k 0=2
Reshape reshape_132 1 1 att_11_key_k att_11_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_11 1 1 att_11_key_k_reshape att_11_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_133 1 1 att_11_key_k_norm att_11_key_k_norm_reshape 0=1024
BinaryOp add_143 1 1 att_11_a_0 att_11_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_144 2 1 att_11_a_0_minus att_11_k_a att_11_a_minus1_mulka 0=2
BinaryOp add_144 1 1 att_11_a_minus1_mulka att_11_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_145 2 1 att_11_a_minus1_mulka_plus1 att_11_key_1 att_11_key_a 0=2
Split split_att_11_value 1 2 att_11_value att_11_value_0 att_11_value_1
Split split_v_first_10 1 2 v_first_10 att_11_v_first v_first_11
BinaryOp minus_10 2 1 att_11_v_first att_11_value_1 att_11_vfirst_minus_value 0=1
Gemm gemm_att_11_mv_lora_0 1 1 att_11_mv_1 att_11_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_11_mv_lora_1 1 1 att_11_mv_lora_0 att_11_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_11_v0 0 1 att_11_v0 0=1024 21=0
BinaryOp add_145 2 1 att_11_mv_lora_1 att_11_v0 att_11_mv_lora_2 0=0
Sigmoid sigmoid_45 1 1 att_11_mv_lora_2 att_11_mv_lora_sigmoid
BinaryOp mul_146 2 1 att_11_mv_lora_sigmoid att_11_vfirst_minus_value att_11_vfirst_minus_value_mul 0=2
BinaryOp add_146 2 1 att_11_value_0 att_11_vfirst_minus_value_mul att_11_value_final 0=0
Gemm gemm_att_11_mw_lora 1 1 att_11_mw att_11_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_11_mw_lora 1 1 att_11_mw_lora att_11_mw_lora_tanh
Gemm gemm_att_11_mw_lora_tanh_linear 1 1 att_11_mw_lora_tanh att_11_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_11_td 0 1 att_11_td 0=1024 21=0
BinaryOp add_147 2 1 att_11_mw_lora_tanh_linear att_11_td att_11_time_decay_pre 0=0
Sigmoid sigmoid_46 1 1 att_11_time_decay_pre att_11_time_decay_sigmoid
Exp exp_11 1 1 att_11_time_decay_sigmoid att_11_time_decay 1=-0.606531
Reshape reshape_134 1 1 att_11_time_decay att_11_time_decay_reshape 0=64 1=1 2=16
Split split_att_11_key_a 1 2 att_11_key_a att_11_key_a_0 att_11_key_a_1
Reshape reshape_135 1 1 att_11_key_a_0 att_11_key_a_reshape 0=64 1=1 2=16
Split split_att_11_value_final 1 3 att_11_value_final att_11_value_final_0 att_11_value_final_1 att_11_value_final_2
Reshape reshape_136 1 1 att_11_value_final_0 att_11_value_final_0_reshape 0=64 1=16
Reshape reshape_137 1 1 att_11_value_final_1 att_11_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_11_vk 2 1 att_11_value_final_1_reshape att_11_key_a_reshape att_11_vk 0=0
Split split_att_11_key_k_norm_reshape 1 2 att_11_key_k_norm_reshape att_11_kk_0 att_11_kk_1
BinaryOp mul_147 1 1 att_11_kk_0 att_11_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_138 1 1 att_11_kk_0_neg att_11_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_148 2 1 att_11_kk_1 att_11_a_1 att_11_kk_a 0=2
Reshape reshape_139 1 1 att_11_kk_a att_11_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_11_ab 2 1 att_11_kk_0_neg_reshape att_11_kk_a_reshape att_11_ab 0=0
Split split_state_34_in 1 2 state_34_in att_11_state_prev_0 att_11_state_prev_1
BinaryOp mul_149 2 1 att_11_state_prev_0 att_11_time_decay_reshape att_11_state_td 0=2
MatMul matmul_att_11_sab 2 1 att_11_state_prev_1 att_11_ab att_11_sab 0=0
BinaryOp add_148 2 1 att_11_sab att_11_vk att_11_sab_vk 0=0
BinaryOp add_149 2 1 att_11_sab_vk att_11_state_td att_11_state_new 0=0
Split split_att_11_state_new 1 2 att_11_state_new att_11_state_new_0 state_34_out
Split split_att_11_receptance 1 2 att_11_receptance att_11_receptance_0 att_11_receptance_1
Reshape reshape_140 1 1 att_11_receptance_0 att_11_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_11_wkv_out 2 1 att_11_state_new_0 att_11_receptance_reshape att_11_wkv_out 0=0
Reshape reshape_141 1 1 att_11_wkv_out att_11_wkv_out_flatten 0=1024
GroupNorm groupnorm_11 1 1 att_11_wkv_out_flatten att_11_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_150 2 1 att_11_receptance_1 att_11_key_a_1 att_11_receptance_key_a 0=2
BinaryOp mul_151 2 1 att_11_receptance_key_a att_11_r_k att_11_rk 0=2
Reshape reshape_142 1 1 att_11_rk att_11_rk_reshape 0=64 1=16
Reduction sum_11 1 1 att_11_rk_reshape att_11_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_152 2 1 att_11_value_final_0_reshape att_11_rk_sum att_11_value_final_0_rk 0=2
Reshape reshape_143 1 1 att_11_value_final_0_rk att_11_value_final_0_rk_reshape 0=1024
BinaryOp add_150 2 1 att_11_value_final_0_rk_reshape att_11_x_gn att_11_x_gn_rkv 0=0
BinaryOp mul_153 2 1 att_11_x_gn_rkv att_11_gate att_11_x_gate 0=2
Gemm gemm_att_11_x_out 1 1 att_11_x_gate att_11_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_151 2 1 att_11_x_out att_11_x_last time_mixing_11_out 0=0
Split split_time_mixing_11_out 1 2 time_mixing_11_out ffn_11_x_last ffn_11_x
LayerNorm layernorm_23 1 1 ffn_11_x ffn_11_xx 0=1024 1=0.00001 2=1
Split split_ffn_11_xx 1 3 ffn_11_xx ffn_11_xx_0 ffn_11_xx_1 state_35_out
BinaryOp sub_23 2 1 state_35_in ffn_11_xx_0 ffn_11_sx 0=1
Split split_ffn_11_sx 1 1 ffn_11_sx ffn_11_sx_0
MemoryData data_ffn_11_x_k 0 1 ffn_11_x_k 0=1024 21=0
BinaryOp mul_154 2 1 ffn_11_sx_0 ffn_11_x_k ffn_11_xk 0=2
BinaryOp add_152 2 1 ffn_11_xk ffn_11_xx_1 ffn_11_xxk 0=0
Gemm gemm_ffn_11_key 1 1 ffn_11_xxk ffn_11_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_11 1 1 ffn_11_key ffn_11_key_relu
UnaryOp square_ffn_11_key_relu 1 1 ffn_11_key_relu ffn_11_key_relu_square 0=4
Gemm gemm_ffn_11_value 1 1 ffn_11_key_relu_square ffn_11_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_153 2 1 ffn_11_value ffn_11_x_last channel_mixing_11_out 0=0
Split split_channel_mixing_11_out 1 2 channel_mixing_11_out att_12_x_last att_12_x
LayerNorm layernorm_24 1 1 att_12_x att_12_xx 0=1024 1=0.00001 2=1
Split split_att_12_xx 1 3 att_12_xx att_12_xx_0 att_12_xx_1 state_36_out
BinaryOp sub_24 2 1 state_36_in att_12_xx_0 att_12_sx 0=1
MemoryData data_att_12_maa 0 1 att_12_maa 0=1024 1=6 21=0
BinaryOp mul_155 2 1 att_12_sx att_12_maa att_12_maa_x 0=2
BinaryOp add_154 2 1 att_12_xx_1 att_12_maa_x att_12_maa_rwkvag 0=0
Slice slice_att_12_maa_rwkvag 1 6 att_12_maa_rwkvag att_12_mr att_12_mw att_12_mk att_12_mv att_12_ma att_12_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_12_mv 1 2 att_12_mv att_12_mv_0 att_12_mv_1
Gemm gemm_att_12_key 1 1 att_12_mk att_12_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_value 1 1 att_12_mv_0 att_12_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_receptance 1 1 att_12_mr att_12_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_ma_lora_0 1 1 att_12_ma att_12_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_12_ma_lora_1 1 1 att_12_ma_lora_0 att_12_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_12_a0 0 1 att_12_a0 0=1024 21=0
BinaryOp add_155 2 1 att_12_ma_lora_1 att_12_a0 att_12_ma_lora_2 0=0
Sigmoid sigmoid_47 1 1 att_12_ma_lora_2 att_12_a
Split split_att_12_a 1 2 att_12_a att_12_a_0 att_12_a_1
Gemm gemm_att_12_mg_lora 1 1 att_12_mg att_12_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_48 1 1 att_12_mg_lora att_12_mg_lora_sigmoid
Gemm gemm_att_12_gate 1 1 att_12_mg_lora_sigmoid att_12_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_12_k_k 0 1 att_12_k_k 0=1024 21=0
MemoryData data_att_12_k_a 0 1 att_12_k_a 0=1024 21=0
MemoryData data_att_12_r_k 0 1 att_12_r_k 0=1024 21=0
Split split_att_12_key 1 2 att_12_key att_12_key_0 att_12_key_1
BinaryOp mul_156 2 1 att_12_key_0 att_12_k_k att_12_key_k 0=2
Reshape reshape_144 1 1 att_12_key_k att_12_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_12 1 1 att_12_key_k_reshape att_12_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_145 1 1 att_12_key_k_norm att_12_key_k_norm_reshape 0=1024
BinaryOp add_156 1 1 att_12_a_0 att_12_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_157 2 1 att_12_a_0_minus att_12_k_a att_12_a_minus1_mulka 0=2
BinaryOp add_157 1 1 att_12_a_minus1_mulka att_12_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_158 2 1 att_12_a_minus1_mulka_plus1 att_12_key_1 att_12_key_a 0=2
Split split_att_12_value 1 2 att_12_value att_12_value_0 att_12_value_1
Split split_v_first_11 1 2 v_first_11 att_12_v_first v_first_12
BinaryOp minus_11 2 1 att_12_v_first att_12_value_1 att_12_vfirst_minus_value 0=1
Gemm gemm_att_12_mv_lora_0 1 1 att_12_mv_1 att_12_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_12_mv_lora_1 1 1 att_12_mv_lora_0 att_12_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_12_v0 0 1 att_12_v0 0=1024 21=0
BinaryOp add_158 2 1 att_12_mv_lora_1 att_12_v0 att_12_mv_lora_2 0=0
Sigmoid sigmoid_49 1 1 att_12_mv_lora_2 att_12_mv_lora_sigmoid
BinaryOp mul_159 2 1 att_12_mv_lora_sigmoid att_12_vfirst_minus_value att_12_vfirst_minus_value_mul 0=2
BinaryOp add_159 2 1 att_12_value_0 att_12_vfirst_minus_value_mul att_12_value_final 0=0
Gemm gemm_att_12_mw_lora 1 1 att_12_mw att_12_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_12_mw_lora 1 1 att_12_mw_lora att_12_mw_lora_tanh
Gemm gemm_att_12_mw_lora_tanh_linear 1 1 att_12_mw_lora_tanh att_12_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_12_td 0 1 att_12_td 0=1024 21=0
BinaryOp add_160 2 1 att_12_mw_lora_tanh_linear att_12_td att_12_time_decay_pre 0=0
Sigmoid sigmoid_50 1 1 att_12_time_decay_pre att_12_time_decay_sigmoid
Exp exp_12 1 1 att_12_time_decay_sigmoid att_12_time_decay 1=-0.606531
Reshape reshape_146 1 1 att_12_time_decay att_12_time_decay_reshape 0=64 1=1 2=16
Split split_att_12_key_a 1 2 att_12_key_a att_12_key_a_0 att_12_key_a_1
Reshape reshape_147 1 1 att_12_key_a_0 att_12_key_a_reshape 0=64 1=1 2=16
Split split_att_12_value_final 1 3 att_12_value_final att_12_value_final_0 att_12_value_final_1 att_12_value_final_2
Reshape reshape_148 1 1 att_12_value_final_0 att_12_value_final_0_reshape 0=64 1=16
Reshape reshape_149 1 1 att_12_value_final_1 att_12_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_12_vk 2 1 att_12_value_final_1_reshape att_12_key_a_reshape att_12_vk 0=0
Split split_att_12_key_k_norm_reshape 1 2 att_12_key_k_norm_reshape att_12_kk_0 att_12_kk_1
BinaryOp mul_160 1 1 att_12_kk_0 att_12_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_150 1 1 att_12_kk_0_neg att_12_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_161 2 1 att_12_kk_1 att_12_a_1 att_12_kk_a 0=2
Reshape reshape_151 1 1 att_12_kk_a att_12_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_12_ab 2 1 att_12_kk_0_neg_reshape att_12_kk_a_reshape att_12_ab 0=0
Split split_state_37_in 1 2 state_37_in att_12_state_prev_0 att_12_state_prev_1
BinaryOp mul_162 2 1 att_12_state_prev_0 att_12_time_decay_reshape att_12_state_td 0=2
MatMul matmul_att_12_sab 2 1 att_12_state_prev_1 att_12_ab att_12_sab 0=0
BinaryOp add_161 2 1 att_12_sab att_12_vk att_12_sab_vk 0=0
BinaryOp add_162 2 1 att_12_sab_vk att_12_state_td att_12_state_new 0=0
Split split_att_12_state_new 1 2 att_12_state_new att_12_state_new_0 state_37_out
Split split_att_12_receptance 1 2 att_12_receptance att_12_receptance_0 att_12_receptance_1
Reshape reshape_152 1 1 att_12_receptance_0 att_12_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_12_wkv_out 2 1 att_12_state_new_0 att_12_receptance_reshape att_12_wkv_out 0=0
Reshape reshape_153 1 1 att_12_wkv_out att_12_wkv_out_flatten 0=1024
GroupNorm groupnorm_12 1 1 att_12_wkv_out_flatten att_12_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_163 2 1 att_12_receptance_1 att_12_key_a_1 att_12_receptance_key_a 0=2
BinaryOp mul_164 2 1 att_12_receptance_key_a att_12_r_k att_12_rk 0=2
Reshape reshape_154 1 1 att_12_rk att_12_rk_reshape 0=64 1=16
Reduction sum_12 1 1 att_12_rk_reshape att_12_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_165 2 1 att_12_value_final_0_reshape att_12_rk_sum att_12_value_final_0_rk 0=2
Reshape reshape_155 1 1 att_12_value_final_0_rk att_12_value_final_0_rk_reshape 0=1024
BinaryOp add_163 2 1 att_12_value_final_0_rk_reshape att_12_x_gn att_12_x_gn_rkv 0=0
BinaryOp mul_166 2 1 att_12_x_gn_rkv att_12_gate att_12_x_gate 0=2
Gemm gemm_att_12_x_out 1 1 att_12_x_gate att_12_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_164 2 1 att_12_x_out att_12_x_last time_mixing_12_out 0=0
Split split_time_mixing_12_out 1 2 time_mixing_12_out ffn_12_x_last ffn_12_x
LayerNorm layernorm_25 1 1 ffn_12_x ffn_12_xx 0=1024 1=0.00001 2=1
Split split_ffn_12_xx 1 3 ffn_12_xx ffn_12_xx_0 ffn_12_xx_1 state_38_out
BinaryOp sub_25 2 1 state_38_in ffn_12_xx_0 ffn_12_sx 0=1
Split split_ffn_12_sx 1 1 ffn_12_sx ffn_12_sx_0
MemoryData data_ffn_12_x_k 0 1 ffn_12_x_k 0=1024 21=0
BinaryOp mul_167 2 1 ffn_12_sx_0 ffn_12_x_k ffn_12_xk 0=2
BinaryOp add_165 2 1 ffn_12_xk ffn_12_xx_1 ffn_12_xxk 0=0
Gemm gemm_ffn_12_key 1 1 ffn_12_xxk ffn_12_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_12 1 1 ffn_12_key ffn_12_key_relu
UnaryOp square_ffn_12_key_relu 1 1 ffn_12_key_relu ffn_12_key_relu_square 0=4
Gemm gemm_ffn_12_value 1 1 ffn_12_key_relu_square ffn_12_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_166 2 1 ffn_12_value ffn_12_x_last channel_mixing_12_out 0=0
Split split_channel_mixing_12_out 1 2 channel_mixing_12_out att_13_x_last att_13_x
LayerNorm layernorm_26 1 1 att_13_x att_13_xx 0=1024 1=0.00001 2=1
Split split_att_13_xx 1 3 att_13_xx att_13_xx_0 att_13_xx_1 state_39_out
BinaryOp sub_26 2 1 state_39_in att_13_xx_0 att_13_sx 0=1
MemoryData data_att_13_maa 0 1 att_13_maa 0=1024 1=6 21=0
BinaryOp mul_168 2 1 att_13_sx att_13_maa att_13_maa_x 0=2
BinaryOp add_167 2 1 att_13_xx_1 att_13_maa_x att_13_maa_rwkvag 0=0
Slice slice_att_13_maa_rwkvag 1 6 att_13_maa_rwkvag att_13_mr att_13_mw att_13_mk att_13_mv att_13_ma att_13_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_13_mv 1 2 att_13_mv att_13_mv_0 att_13_mv_1
Gemm gemm_att_13_key 1 1 att_13_mk att_13_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_value 1 1 att_13_mv_0 att_13_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_receptance 1 1 att_13_mr att_13_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_ma_lora_0 1 1 att_13_ma att_13_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_13_ma_lora_1 1 1 att_13_ma_lora_0 att_13_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_13_a0 0 1 att_13_a0 0=1024 21=0
BinaryOp add_168 2 1 att_13_ma_lora_1 att_13_a0 att_13_ma_lora_2 0=0
Sigmoid sigmoid_51 1 1 att_13_ma_lora_2 att_13_a
Split split_att_13_a 1 2 att_13_a att_13_a_0 att_13_a_1
Gemm gemm_att_13_mg_lora 1 1 att_13_mg att_13_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_52 1 1 att_13_mg_lora att_13_mg_lora_sigmoid
Gemm gemm_att_13_gate 1 1 att_13_mg_lora_sigmoid att_13_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_13_k_k 0 1 att_13_k_k 0=1024 21=0
MemoryData data_att_13_k_a 0 1 att_13_k_a 0=1024 21=0
MemoryData data_att_13_r_k 0 1 att_13_r_k 0=1024 21=0
Split split_att_13_key 1 2 att_13_key att_13_key_0 att_13_key_1
BinaryOp mul_169 2 1 att_13_key_0 att_13_k_k att_13_key_k 0=2
Reshape reshape_156 1 1 att_13_key_k att_13_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_13 1 1 att_13_key_k_reshape att_13_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_157 1 1 att_13_key_k_norm att_13_key_k_norm_reshape 0=1024
BinaryOp add_169 1 1 att_13_a_0 att_13_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_170 2 1 att_13_a_0_minus att_13_k_a att_13_a_minus1_mulka 0=2
BinaryOp add_170 1 1 att_13_a_minus1_mulka att_13_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_171 2 1 att_13_a_minus1_mulka_plus1 att_13_key_1 att_13_key_a 0=2
Split split_att_13_value 1 2 att_13_value att_13_value_0 att_13_value_1
Split split_v_first_12 1 2 v_first_12 att_13_v_first v_first_13
BinaryOp minus_12 2 1 att_13_v_first att_13_value_1 att_13_vfirst_minus_value 0=1
Gemm gemm_att_13_mv_lora_0 1 1 att_13_mv_1 att_13_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_13_mv_lora_1 1 1 att_13_mv_lora_0 att_13_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_13_v0 0 1 att_13_v0 0=1024 21=0
BinaryOp add_171 2 1 att_13_mv_lora_1 att_13_v0 att_13_mv_lora_2 0=0
Sigmoid sigmoid_53 1 1 att_13_mv_lora_2 att_13_mv_lora_sigmoid
BinaryOp mul_172 2 1 att_13_mv_lora_sigmoid att_13_vfirst_minus_value att_13_vfirst_minus_value_mul 0=2
BinaryOp add_172 2 1 att_13_value_0 att_13_vfirst_minus_value_mul att_13_value_final 0=0
Gemm gemm_att_13_mw_lora 1 1 att_13_mw att_13_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_13_mw_lora 1 1 att_13_mw_lora att_13_mw_lora_tanh
Gemm gemm_att_13_mw_lora_tanh_linear 1 1 att_13_mw_lora_tanh att_13_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_13_td 0 1 att_13_td 0=1024 21=0
BinaryOp add_173 2 1 att_13_mw_lora_tanh_linear att_13_td att_13_time_decay_pre 0=0
Sigmoid sigmoid_54 1 1 att_13_time_decay_pre att_13_time_decay_sigmoid
Exp exp_13 1 1 att_13_time_decay_sigmoid att_13_time_decay 1=-0.606531
Reshape reshape_158 1 1 att_13_time_decay att_13_time_decay_reshape 0=64 1=1 2=16
Split split_att_13_key_a 1 2 att_13_key_a att_13_key_a_0 att_13_key_a_1
Reshape reshape_159 1 1 att_13_key_a_0 att_13_key_a_reshape 0=64 1=1 2=16
Split split_att_13_value_final 1 3 att_13_value_final att_13_value_final_0 att_13_value_final_1 att_13_value_final_2
Reshape reshape_160 1 1 att_13_value_final_0 att_13_value_final_0_reshape 0=64 1=16
Reshape reshape_161 1 1 att_13_value_final_1 att_13_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_13_vk 2 1 att_13_value_final_1_reshape att_13_key_a_reshape att_13_vk 0=0
Split split_att_13_key_k_norm_reshape 1 2 att_13_key_k_norm_reshape att_13_kk_0 att_13_kk_1
BinaryOp mul_173 1 1 att_13_kk_0 att_13_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_162 1 1 att_13_kk_0_neg att_13_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_174 2 1 att_13_kk_1 att_13_a_1 att_13_kk_a 0=2
Reshape reshape_163 1 1 att_13_kk_a att_13_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_13_ab 2 1 att_13_kk_0_neg_reshape att_13_kk_a_reshape att_13_ab 0=0
Split split_state_40_in 1 2 state_40_in att_13_state_prev_0 att_13_state_prev_1
BinaryOp mul_175 2 1 att_13_state_prev_0 att_13_time_decay_reshape att_13_state_td 0=2
MatMul matmul_att_13_sab 2 1 att_13_state_prev_1 att_13_ab att_13_sab 0=0
BinaryOp add_174 2 1 att_13_sab att_13_vk att_13_sab_vk 0=0
BinaryOp add_175 2 1 att_13_sab_vk att_13_state_td att_13_state_new 0=0
Split split_att_13_state_new 1 2 att_13_state_new att_13_state_new_0 state_40_out
Split split_att_13_receptance 1 2 att_13_receptance att_13_receptance_0 att_13_receptance_1
Reshape reshape_164 1 1 att_13_receptance_0 att_13_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_13_wkv_out 2 1 att_13_state_new_0 att_13_receptance_reshape att_13_wkv_out 0=0
Reshape reshape_165 1 1 att_13_wkv_out att_13_wkv_out_flatten 0=1024
GroupNorm groupnorm_13 1 1 att_13_wkv_out_flatten att_13_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_176 2 1 att_13_receptance_1 att_13_key_a_1 att_13_receptance_key_a 0=2
BinaryOp mul_177 2 1 att_13_receptance_key_a att_13_r_k att_13_rk 0=2
Reshape reshape_166 1 1 att_13_rk att_13_rk_reshape 0=64 1=16
Reduction sum_13 1 1 att_13_rk_reshape att_13_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_178 2 1 att_13_value_final_0_reshape att_13_rk_sum att_13_value_final_0_rk 0=2
Reshape reshape_167 1 1 att_13_value_final_0_rk att_13_value_final_0_rk_reshape 0=1024
BinaryOp add_176 2 1 att_13_value_final_0_rk_reshape att_13_x_gn att_13_x_gn_rkv 0=0
BinaryOp mul_179 2 1 att_13_x_gn_rkv att_13_gate att_13_x_gate 0=2
Gemm gemm_att_13_x_out 1 1 att_13_x_gate att_13_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_177 2 1 att_13_x_out att_13_x_last time_mixing_13_out 0=0
Split split_time_mixing_13_out 1 2 time_mixing_13_out ffn_13_x_last ffn_13_x
LayerNorm layernorm_27 1 1 ffn_13_x ffn_13_xx 0=1024 1=0.00001 2=1
Split split_ffn_13_xx 1 3 ffn_13_xx ffn_13_xx_0 ffn_13_xx_1 state_41_out
BinaryOp sub_27 2 1 state_41_in ffn_13_xx_0 ffn_13_sx 0=1
Split split_ffn_13_sx 1 1 ffn_13_sx ffn_13_sx_0
MemoryData data_ffn_13_x_k 0 1 ffn_13_x_k 0=1024 21=0
BinaryOp mul_180 2 1 ffn_13_sx_0 ffn_13_x_k ffn_13_xk 0=2
BinaryOp add_178 2 1 ffn_13_xk ffn_13_xx_1 ffn_13_xxk 0=0
Gemm gemm_ffn_13_key 1 1 ffn_13_xxk ffn_13_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_13 1 1 ffn_13_key ffn_13_key_relu
UnaryOp square_ffn_13_key_relu 1 1 ffn_13_key_relu ffn_13_key_relu_square 0=4
Gemm gemm_ffn_13_value 1 1 ffn_13_key_relu_square ffn_13_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_179 2 1 ffn_13_value ffn_13_x_last channel_mixing_13_out 0=0
Split split_channel_mixing_13_out 1 2 channel_mixing_13_out att_14_x_last att_14_x
LayerNorm layernorm_28 1 1 att_14_x att_14_xx 0=1024 1=0.00001 2=1
Split split_att_14_xx 1 3 att_14_xx att_14_xx_0 att_14_xx_1 state_42_out
BinaryOp sub_28 2 1 state_42_in att_14_xx_0 att_14_sx 0=1
MemoryData data_att_14_maa 0 1 att_14_maa 0=1024 1=6 21=0
BinaryOp mul_181 2 1 att_14_sx att_14_maa att_14_maa_x 0=2
BinaryOp add_180 2 1 att_14_xx_1 att_14_maa_x att_14_maa_rwkvag 0=0
Slice slice_att_14_maa_rwkvag 1 6 att_14_maa_rwkvag att_14_mr att_14_mw att_14_mk att_14_mv att_14_ma att_14_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_14_mv 1 2 att_14_mv att_14_mv_0 att_14_mv_1
Gemm gemm_att_14_key 1 1 att_14_mk att_14_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_value 1 1 att_14_mv_0 att_14_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_receptance 1 1 att_14_mr att_14_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_ma_lora_0 1 1 att_14_ma att_14_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_14_ma_lora_1 1 1 att_14_ma_lora_0 att_14_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_14_a0 0 1 att_14_a0 0=1024 21=0
BinaryOp add_181 2 1 att_14_ma_lora_1 att_14_a0 att_14_ma_lora_2 0=0
Sigmoid sigmoid_55 1 1 att_14_ma_lora_2 att_14_a
Split split_att_14_a 1 2 att_14_a att_14_a_0 att_14_a_1
Gemm gemm_att_14_mg_lora 1 1 att_14_mg att_14_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_56 1 1 att_14_mg_lora att_14_mg_lora_sigmoid
Gemm gemm_att_14_gate 1 1 att_14_mg_lora_sigmoid att_14_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_14_k_k 0 1 att_14_k_k 0=1024 21=0
MemoryData data_att_14_k_a 0 1 att_14_k_a 0=1024 21=0
MemoryData data_att_14_r_k 0 1 att_14_r_k 0=1024 21=0
Split split_att_14_key 1 2 att_14_key att_14_key_0 att_14_key_1
BinaryOp mul_182 2 1 att_14_key_0 att_14_k_k att_14_key_k 0=2
Reshape reshape_168 1 1 att_14_key_k att_14_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_14 1 1 att_14_key_k_reshape att_14_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_169 1 1 att_14_key_k_norm att_14_key_k_norm_reshape 0=1024
BinaryOp add_182 1 1 att_14_a_0 att_14_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_183 2 1 att_14_a_0_minus att_14_k_a att_14_a_minus1_mulka 0=2
BinaryOp add_183 1 1 att_14_a_minus1_mulka att_14_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_184 2 1 att_14_a_minus1_mulka_plus1 att_14_key_1 att_14_key_a 0=2
Split split_att_14_value 1 2 att_14_value att_14_value_0 att_14_value_1
Split split_v_first_13 1 2 v_first_13 att_14_v_first v_first_14
BinaryOp minus_13 2 1 att_14_v_first att_14_value_1 att_14_vfirst_minus_value 0=1
Gemm gemm_att_14_mv_lora_0 1 1 att_14_mv_1 att_14_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_14_mv_lora_1 1 1 att_14_mv_lora_0 att_14_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_14_v0 0 1 att_14_v0 0=1024 21=0
BinaryOp add_184 2 1 att_14_mv_lora_1 att_14_v0 att_14_mv_lora_2 0=0
Sigmoid sigmoid_57 1 1 att_14_mv_lora_2 att_14_mv_lora_sigmoid
BinaryOp mul_185 2 1 att_14_mv_lora_sigmoid att_14_vfirst_minus_value att_14_vfirst_minus_value_mul 0=2
BinaryOp add_185 2 1 att_14_value_0 att_14_vfirst_minus_value_mul att_14_value_final 0=0
Gemm gemm_att_14_mw_lora 1 1 att_14_mw att_14_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_14_mw_lora 1 1 att_14_mw_lora att_14_mw_lora_tanh
Gemm gemm_att_14_mw_lora_tanh_linear 1 1 att_14_mw_lora_tanh att_14_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_14_td 0 1 att_14_td 0=1024 21=0
BinaryOp add_186 2 1 att_14_mw_lora_tanh_linear att_14_td att_14_time_decay_pre 0=0
Sigmoid sigmoid_58 1 1 att_14_time_decay_pre att_14_time_decay_sigmoid
Exp exp_14 1 1 att_14_time_decay_sigmoid att_14_time_decay 1=-0.606531
Reshape reshape_170 1 1 att_14_time_decay att_14_time_decay_reshape 0=64 1=1 2=16
Split split_att_14_key_a 1 2 att_14_key_a att_14_key_a_0 att_14_key_a_1
Reshape reshape_171 1 1 att_14_key_a_0 att_14_key_a_reshape 0=64 1=1 2=16
Split split_att_14_value_final 1 3 att_14_value_final att_14_value_final_0 att_14_value_final_1 att_14_value_final_2
Reshape reshape_172 1 1 att_14_value_final_0 att_14_value_final_0_reshape 0=64 1=16
Reshape reshape_173 1 1 att_14_value_final_1 att_14_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_14_vk 2 1 att_14_value_final_1_reshape att_14_key_a_reshape att_14_vk 0=0
Split split_att_14_key_k_norm_reshape 1 2 att_14_key_k_norm_reshape att_14_kk_0 att_14_kk_1
BinaryOp mul_186 1 1 att_14_kk_0 att_14_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_174 1 1 att_14_kk_0_neg att_14_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_187 2 1 att_14_kk_1 att_14_a_1 att_14_kk_a 0=2
Reshape reshape_175 1 1 att_14_kk_a att_14_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_14_ab 2 1 att_14_kk_0_neg_reshape att_14_kk_a_reshape att_14_ab 0=0
Split split_state_43_in 1 2 state_43_in att_14_state_prev_0 att_14_state_prev_1
BinaryOp mul_188 2 1 att_14_state_prev_0 att_14_time_decay_reshape att_14_state_td 0=2
MatMul matmul_att_14_sab 2 1 att_14_state_prev_1 att_14_ab att_14_sab 0=0
BinaryOp add_187 2 1 att_14_sab att_14_vk att_14_sab_vk 0=0
BinaryOp add_188 2 1 att_14_sab_vk att_14_state_td att_14_state_new 0=0
Split split_att_14_state_new 1 2 att_14_state_new att_14_state_new_0 state_43_out
Split split_att_14_receptance 1 2 att_14_receptance att_14_receptance_0 att_14_receptance_1
Reshape reshape_176 1 1 att_14_receptance_0 att_14_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_14_wkv_out 2 1 att_14_state_new_0 att_14_receptance_reshape att_14_wkv_out 0=0
Reshape reshape_177 1 1 att_14_wkv_out att_14_wkv_out_flatten 0=1024
GroupNorm groupnorm_14 1 1 att_14_wkv_out_flatten att_14_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_189 2 1 att_14_receptance_1 att_14_key_a_1 att_14_receptance_key_a 0=2
BinaryOp mul_190 2 1 att_14_receptance_key_a att_14_r_k att_14_rk 0=2
Reshape reshape_178 1 1 att_14_rk att_14_rk_reshape 0=64 1=16
Reduction sum_14 1 1 att_14_rk_reshape att_14_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_191 2 1 att_14_value_final_0_reshape att_14_rk_sum att_14_value_final_0_rk 0=2
Reshape reshape_179 1 1 att_14_value_final_0_rk att_14_value_final_0_rk_reshape 0=1024
BinaryOp add_189 2 1 att_14_value_final_0_rk_reshape att_14_x_gn att_14_x_gn_rkv 0=0
BinaryOp mul_192 2 1 att_14_x_gn_rkv att_14_gate att_14_x_gate 0=2
Gemm gemm_att_14_x_out 1 1 att_14_x_gate att_14_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_190 2 1 att_14_x_out att_14_x_last time_mixing_14_out 0=0
Split split_time_mixing_14_out 1 2 time_mixing_14_out ffn_14_x_last ffn_14_x
LayerNorm layernorm_29 1 1 ffn_14_x ffn_14_xx 0=1024 1=0.00001 2=1
Split split_ffn_14_xx 1 3 ffn_14_xx ffn_14_xx_0 ffn_14_xx_1 state_44_out
BinaryOp sub_29 2 1 state_44_in ffn_14_xx_0 ffn_14_sx 0=1
Split split_ffn_14_sx 1 1 ffn_14_sx ffn_14_sx_0
MemoryData data_ffn_14_x_k 0 1 ffn_14_x_k 0=1024 21=0
BinaryOp mul_193 2 1 ffn_14_sx_0 ffn_14_x_k ffn_14_xk 0=2
BinaryOp add_191 2 1 ffn_14_xk ffn_14_xx_1 ffn_14_xxk 0=0
Gemm gemm_ffn_14_key 1 1 ffn_14_xxk ffn_14_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_14 1 1 ffn_14_key ffn_14_key_relu
UnaryOp square_ffn_14_key_relu 1 1 ffn_14_key_relu ffn_14_key_relu_square 0=4
Gemm gemm_ffn_14_value 1 1 ffn_14_key_relu_square ffn_14_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_192 2 1 ffn_14_value ffn_14_x_last channel_mixing_14_out 0=0
Split split_channel_mixing_14_out 1 2 channel_mixing_14_out att_15_x_last att_15_x
LayerNorm layernorm_30 1 1 att_15_x att_15_xx 0=1024 1=0.00001 2=1
Split split_att_15_xx 1 3 att_15_xx att_15_xx_0 att_15_xx_1 state_45_out
BinaryOp sub_30 2 1 state_45_in att_15_xx_0 att_15_sx 0=1
MemoryData data_att_15_maa 0 1 att_15_maa 0=1024 1=6 21=0
BinaryOp mul_194 2 1 att_15_sx att_15_maa att_15_maa_x 0=2
BinaryOp add_193 2 1 att_15_xx_1 att_15_maa_x att_15_maa_rwkvag 0=0
Slice slice_att_15_maa_rwkvag 1 6 att_15_maa_rwkvag att_15_mr att_15_mw att_15_mk att_15_mv att_15_ma att_15_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_15_mv 1 2 att_15_mv att_15_mv_0 att_15_mv_1
Gemm gemm_att_15_key 1 1 att_15_mk att_15_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_value 1 1 att_15_mv_0 att_15_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_receptance 1 1 att_15_mr att_15_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_ma_lora_0 1 1 att_15_ma att_15_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_15_ma_lora_1 1 1 att_15_ma_lora_0 att_15_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_15_a0 0 1 att_15_a0 0=1024 21=0
BinaryOp add_194 2 1 att_15_ma_lora_1 att_15_a0 att_15_ma_lora_2 0=0
Sigmoid sigmoid_59 1 1 att_15_ma_lora_2 att_15_a
Split split_att_15_a 1 2 att_15_a att_15_a_0 att_15_a_1
Gemm gemm_att_15_mg_lora 1 1 att_15_mg att_15_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_60 1 1 att_15_mg_lora att_15_mg_lora_sigmoid
Gemm gemm_att_15_gate 1 1 att_15_mg_lora_sigmoid att_15_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_15_k_k 0 1 att_15_k_k 0=1024 21=0
MemoryData data_att_15_k_a 0 1 att_15_k_a 0=1024 21=0
MemoryData data_att_15_r_k 0 1 att_15_r_k 0=1024 21=0
Split split_att_15_key 1 2 att_15_key att_15_key_0 att_15_key_1
BinaryOp mul_195 2 1 att_15_key_0 att_15_k_k att_15_key_k 0=2
Reshape reshape_180 1 1 att_15_key_k att_15_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_15 1 1 att_15_key_k_reshape att_15_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_181 1 1 att_15_key_k_norm att_15_key_k_norm_reshape 0=1024
BinaryOp add_195 1 1 att_15_a_0 att_15_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_196 2 1 att_15_a_0_minus att_15_k_a att_15_a_minus1_mulka 0=2
BinaryOp add_196 1 1 att_15_a_minus1_mulka att_15_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_197 2 1 att_15_a_minus1_mulka_plus1 att_15_key_1 att_15_key_a 0=2
Split split_att_15_value 1 2 att_15_value att_15_value_0 att_15_value_1
Split split_v_first_14 1 2 v_first_14 att_15_v_first v_first_15
BinaryOp minus_14 2 1 att_15_v_first att_15_value_1 att_15_vfirst_minus_value 0=1
Gemm gemm_att_15_mv_lora_0 1 1 att_15_mv_1 att_15_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_15_mv_lora_1 1 1 att_15_mv_lora_0 att_15_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_15_v0 0 1 att_15_v0 0=1024 21=0
BinaryOp add_197 2 1 att_15_mv_lora_1 att_15_v0 att_15_mv_lora_2 0=0
Sigmoid sigmoid_61 1 1 att_15_mv_lora_2 att_15_mv_lora_sigmoid
BinaryOp mul_198 2 1 att_15_mv_lora_sigmoid att_15_vfirst_minus_value att_15_vfirst_minus_value_mul 0=2
BinaryOp add_198 2 1 att_15_value_0 att_15_vfirst_minus_value_mul att_15_value_final 0=0
Gemm gemm_att_15_mw_lora 1 1 att_15_mw att_15_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_15_mw_lora 1 1 att_15_mw_lora att_15_mw_lora_tanh
Gemm gemm_att_15_mw_lora_tanh_linear 1 1 att_15_mw_lora_tanh att_15_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_15_td 0 1 att_15_td 0=1024 21=0
BinaryOp add_199 2 1 att_15_mw_lora_tanh_linear att_15_td att_15_time_decay_pre 0=0
Sigmoid sigmoid_62 1 1 att_15_time_decay_pre att_15_time_decay_sigmoid
Exp exp_15 1 1 att_15_time_decay_sigmoid att_15_time_decay 1=-0.606531
Reshape reshape_182 1 1 att_15_time_decay att_15_time_decay_reshape 0=64 1=1 2=16
Split split_att_15_key_a 1 2 att_15_key_a att_15_key_a_0 att_15_key_a_1
Reshape reshape_183 1 1 att_15_key_a_0 att_15_key_a_reshape 0=64 1=1 2=16
Split split_att_15_value_final 1 3 att_15_value_final att_15_value_final_0 att_15_value_final_1 att_15_value_final_2
Reshape reshape_184 1 1 att_15_value_final_0 att_15_value_final_0_reshape 0=64 1=16
Reshape reshape_185 1 1 att_15_value_final_1 att_15_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_15_vk 2 1 att_15_value_final_1_reshape att_15_key_a_reshape att_15_vk 0=0
Split split_att_15_key_k_norm_reshape 1 2 att_15_key_k_norm_reshape att_15_kk_0 att_15_kk_1
BinaryOp mul_199 1 1 att_15_kk_0 att_15_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_186 1 1 att_15_kk_0_neg att_15_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_200 2 1 att_15_kk_1 att_15_a_1 att_15_kk_a 0=2
Reshape reshape_187 1 1 att_15_kk_a att_15_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_15_ab 2 1 att_15_kk_0_neg_reshape att_15_kk_a_reshape att_15_ab 0=0
Split split_state_46_in 1 2 state_46_in att_15_state_prev_0 att_15_state_prev_1
BinaryOp mul_201 2 1 att_15_state_prev_0 att_15_time_decay_reshape att_15_state_td 0=2
MatMul matmul_att_15_sab 2 1 att_15_state_prev_1 att_15_ab att_15_sab 0=0
BinaryOp add_200 2 1 att_15_sab att_15_vk att_15_sab_vk 0=0
BinaryOp add_201 2 1 att_15_sab_vk att_15_state_td att_15_state_new 0=0
Split split_att_15_state_new 1 2 att_15_state_new att_15_state_new_0 state_46_out
Split split_att_15_receptance 1 2 att_15_receptance att_15_receptance_0 att_15_receptance_1
Reshape reshape_188 1 1 att_15_receptance_0 att_15_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_15_wkv_out 2 1 att_15_state_new_0 att_15_receptance_reshape att_15_wkv_out 0=0
Reshape reshape_189 1 1 att_15_wkv_out att_15_wkv_out_flatten 0=1024
GroupNorm groupnorm_15 1 1 att_15_wkv_out_flatten att_15_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_202 2 1 att_15_receptance_1 att_15_key_a_1 att_15_receptance_key_a 0=2
BinaryOp mul_203 2 1 att_15_receptance_key_a att_15_r_k att_15_rk 0=2
Reshape reshape_190 1 1 att_15_rk att_15_rk_reshape 0=64 1=16
Reduction sum_15 1 1 att_15_rk_reshape att_15_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_204 2 1 att_15_value_final_0_reshape att_15_rk_sum att_15_value_final_0_rk 0=2
Reshape reshape_191 1 1 att_15_value_final_0_rk att_15_value_final_0_rk_reshape 0=1024
BinaryOp add_202 2 1 att_15_value_final_0_rk_reshape att_15_x_gn att_15_x_gn_rkv 0=0
BinaryOp mul_205 2 1 att_15_x_gn_rkv att_15_gate att_15_x_gate 0=2
Gemm gemm_att_15_x_out 1 1 att_15_x_gate att_15_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_203 2 1 att_15_x_out att_15_x_last time_mixing_15_out 0=0
Split split_time_mixing_15_out 1 2 time_mixing_15_out ffn_15_x_last ffn_15_x
LayerNorm layernorm_31 1 1 ffn_15_x ffn_15_xx 0=1024 1=0.00001 2=1
Split split_ffn_15_xx 1 3 ffn_15_xx ffn_15_xx_0 ffn_15_xx_1 state_47_out
BinaryOp sub_31 2 1 state_47_in ffn_15_xx_0 ffn_15_sx 0=1
Split split_ffn_15_sx 1 1 ffn_15_sx ffn_15_sx_0
MemoryData data_ffn_15_x_k 0 1 ffn_15_x_k 0=1024 21=0
BinaryOp mul_206 2 1 ffn_15_sx_0 ffn_15_x_k ffn_15_xk 0=2
BinaryOp add_204 2 1 ffn_15_xk ffn_15_xx_1 ffn_15_xxk 0=0
Gemm gemm_ffn_15_key 1 1 ffn_15_xxk ffn_15_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_15 1 1 ffn_15_key ffn_15_key_relu
UnaryOp square_ffn_15_key_relu 1 1 ffn_15_key_relu ffn_15_key_relu_square 0=4
Gemm gemm_ffn_15_value 1 1 ffn_15_key_relu_square ffn_15_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_205 2 1 ffn_15_value ffn_15_x_last channel_mixing_15_out 0=0
Split split_channel_mixing_15_out 1 2 channel_mixing_15_out att_16_x_last att_16_x
LayerNorm layernorm_32 1 1 att_16_x att_16_xx 0=1024 1=0.00001 2=1
Split split_att_16_xx 1 3 att_16_xx att_16_xx_0 att_16_xx_1 state_48_out
BinaryOp sub_32 2 1 state_48_in att_16_xx_0 att_16_sx 0=1
MemoryData data_att_16_maa 0 1 att_16_maa 0=1024 1=6 21=0
BinaryOp mul_207 2 1 att_16_sx att_16_maa att_16_maa_x 0=2
BinaryOp add_206 2 1 att_16_xx_1 att_16_maa_x att_16_maa_rwkvag 0=0
Slice slice_att_16_maa_rwkvag 1 6 att_16_maa_rwkvag att_16_mr att_16_mw att_16_mk att_16_mv att_16_ma att_16_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_16_mv 1 2 att_16_mv att_16_mv_0 att_16_mv_1
Gemm gemm_att_16_key 1 1 att_16_mk att_16_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_value 1 1 att_16_mv_0 att_16_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_receptance 1 1 att_16_mr att_16_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_ma_lora_0 1 1 att_16_ma att_16_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_16_ma_lora_1 1 1 att_16_ma_lora_0 att_16_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_16_a0 0 1 att_16_a0 0=1024 21=0
BinaryOp add_207 2 1 att_16_ma_lora_1 att_16_a0 att_16_ma_lora_2 0=0
Sigmoid sigmoid_63 1 1 att_16_ma_lora_2 att_16_a
Split split_att_16_a 1 2 att_16_a att_16_a_0 att_16_a_1
Gemm gemm_att_16_mg_lora 1 1 att_16_mg att_16_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_64 1 1 att_16_mg_lora att_16_mg_lora_sigmoid
Gemm gemm_att_16_gate 1 1 att_16_mg_lora_sigmoid att_16_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_16_k_k 0 1 att_16_k_k 0=1024 21=0
MemoryData data_att_16_k_a 0 1 att_16_k_a 0=1024 21=0
MemoryData data_att_16_r_k 0 1 att_16_r_k 0=1024 21=0
Split split_att_16_key 1 2 att_16_key att_16_key_0 att_16_key_1
BinaryOp mul_208 2 1 att_16_key_0 att_16_k_k att_16_key_k 0=2
Reshape reshape_192 1 1 att_16_key_k att_16_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_16 1 1 att_16_key_k_reshape att_16_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_193 1 1 att_16_key_k_norm att_16_key_k_norm_reshape 0=1024
BinaryOp add_208 1 1 att_16_a_0 att_16_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_209 2 1 att_16_a_0_minus att_16_k_a att_16_a_minus1_mulka 0=2
BinaryOp add_209 1 1 att_16_a_minus1_mulka att_16_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_210 2 1 att_16_a_minus1_mulka_plus1 att_16_key_1 att_16_key_a 0=2
Split split_att_16_value 1 2 att_16_value att_16_value_0 att_16_value_1
Split split_v_first_15 1 2 v_first_15 att_16_v_first v_first_16
BinaryOp minus_15 2 1 att_16_v_first att_16_value_1 att_16_vfirst_minus_value 0=1
Gemm gemm_att_16_mv_lora_0 1 1 att_16_mv_1 att_16_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_16_mv_lora_1 1 1 att_16_mv_lora_0 att_16_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_16_v0 0 1 att_16_v0 0=1024 21=0
BinaryOp add_210 2 1 att_16_mv_lora_1 att_16_v0 att_16_mv_lora_2 0=0
Sigmoid sigmoid_65 1 1 att_16_mv_lora_2 att_16_mv_lora_sigmoid
BinaryOp mul_211 2 1 att_16_mv_lora_sigmoid att_16_vfirst_minus_value att_16_vfirst_minus_value_mul 0=2
BinaryOp add_211 2 1 att_16_value_0 att_16_vfirst_minus_value_mul att_16_value_final 0=0
Gemm gemm_att_16_mw_lora 1 1 att_16_mw att_16_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_16_mw_lora 1 1 att_16_mw_lora att_16_mw_lora_tanh
Gemm gemm_att_16_mw_lora_tanh_linear 1 1 att_16_mw_lora_tanh att_16_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_16_td 0 1 att_16_td 0=1024 21=0
BinaryOp add_212 2 1 att_16_mw_lora_tanh_linear att_16_td att_16_time_decay_pre 0=0
Sigmoid sigmoid_66 1 1 att_16_time_decay_pre att_16_time_decay_sigmoid
Exp exp_16 1 1 att_16_time_decay_sigmoid att_16_time_decay 1=-0.606531
Reshape reshape_194 1 1 att_16_time_decay att_16_time_decay_reshape 0=64 1=1 2=16
Split split_att_16_key_a 1 2 att_16_key_a att_16_key_a_0 att_16_key_a_1
Reshape reshape_195 1 1 att_16_key_a_0 att_16_key_a_reshape 0=64 1=1 2=16
Split split_att_16_value_final 1 3 att_16_value_final att_16_value_final_0 att_16_value_final_1 att_16_value_final_2
Reshape reshape_196 1 1 att_16_value_final_0 att_16_value_final_0_reshape 0=64 1=16
Reshape reshape_197 1 1 att_16_value_final_1 att_16_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_16_vk 2 1 att_16_value_final_1_reshape att_16_key_a_reshape att_16_vk 0=0
Split split_att_16_key_k_norm_reshape 1 2 att_16_key_k_norm_reshape att_16_kk_0 att_16_kk_1
BinaryOp mul_212 1 1 att_16_kk_0 att_16_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_198 1 1 att_16_kk_0_neg att_16_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_213 2 1 att_16_kk_1 att_16_a_1 att_16_kk_a 0=2
Reshape reshape_199 1 1 att_16_kk_a att_16_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_16_ab 2 1 att_16_kk_0_neg_reshape att_16_kk_a_reshape att_16_ab 0=0
Split split_state_49_in 1 2 state_49_in att_16_state_prev_0 att_16_state_prev_1
BinaryOp mul_214 2 1 att_16_state_prev_0 att_16_time_decay_reshape att_16_state_td 0=2
MatMul matmul_att_16_sab 2 1 att_16_state_prev_1 att_16_ab att_16_sab 0=0
BinaryOp add_213 2 1 att_16_sab att_16_vk att_16_sab_vk 0=0
BinaryOp add_214 2 1 att_16_sab_vk att_16_state_td att_16_state_new 0=0
Split split_att_16_state_new 1 2 att_16_state_new att_16_state_new_0 state_49_out
Split split_att_16_receptance 1 2 att_16_receptance att_16_receptance_0 att_16_receptance_1
Reshape reshape_200 1 1 att_16_receptance_0 att_16_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_16_wkv_out 2 1 att_16_state_new_0 att_16_receptance_reshape att_16_wkv_out 0=0
Reshape reshape_201 1 1 att_16_wkv_out att_16_wkv_out_flatten 0=1024
GroupNorm groupnorm_16 1 1 att_16_wkv_out_flatten att_16_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_215 2 1 att_16_receptance_1 att_16_key_a_1 att_16_receptance_key_a 0=2
BinaryOp mul_216 2 1 att_16_receptance_key_a att_16_r_k att_16_rk 0=2
Reshape reshape_202 1 1 att_16_rk att_16_rk_reshape 0=64 1=16
Reduction sum_16 1 1 att_16_rk_reshape att_16_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_217 2 1 att_16_value_final_0_reshape att_16_rk_sum att_16_value_final_0_rk 0=2
Reshape reshape_203 1 1 att_16_value_final_0_rk att_16_value_final_0_rk_reshape 0=1024
BinaryOp add_215 2 1 att_16_value_final_0_rk_reshape att_16_x_gn att_16_x_gn_rkv 0=0
BinaryOp mul_218 2 1 att_16_x_gn_rkv att_16_gate att_16_x_gate 0=2
Gemm gemm_att_16_x_out 1 1 att_16_x_gate att_16_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_216 2 1 att_16_x_out att_16_x_last time_mixing_16_out 0=0
Split split_time_mixing_16_out 1 2 time_mixing_16_out ffn_16_x_last ffn_16_x
LayerNorm layernorm_33 1 1 ffn_16_x ffn_16_xx 0=1024 1=0.00001 2=1
Split split_ffn_16_xx 1 3 ffn_16_xx ffn_16_xx_0 ffn_16_xx_1 state_50_out
BinaryOp sub_33 2 1 state_50_in ffn_16_xx_0 ffn_16_sx 0=1
Split split_ffn_16_sx 1 1 ffn_16_sx ffn_16_sx_0
MemoryData data_ffn_16_x_k 0 1 ffn_16_x_k 0=1024 21=0
BinaryOp mul_219 2 1 ffn_16_sx_0 ffn_16_x_k ffn_16_xk 0=2
BinaryOp add_217 2 1 ffn_16_xk ffn_16_xx_1 ffn_16_xxk 0=0
Gemm gemm_ffn_16_key 1 1 ffn_16_xxk ffn_16_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_16 1 1 ffn_16_key ffn_16_key_relu
UnaryOp square_ffn_16_key_relu 1 1 ffn_16_key_relu ffn_16_key_relu_square 0=4
Gemm gemm_ffn_16_value 1 1 ffn_16_key_relu_square ffn_16_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_218 2 1 ffn_16_value ffn_16_x_last channel_mixing_16_out 0=0
Split split_channel_mixing_16_out 1 2 channel_mixing_16_out att_17_x_last att_17_x
LayerNorm layernorm_34 1 1 att_17_x att_17_xx 0=1024 1=0.00001 2=1
Split split_att_17_xx 1 3 att_17_xx att_17_xx_0 att_17_xx_1 state_51_out
BinaryOp sub_34 2 1 state_51_in att_17_xx_0 att_17_sx 0=1
MemoryData data_att_17_maa 0 1 att_17_maa 0=1024 1=6 21=0
BinaryOp mul_220 2 1 att_17_sx att_17_maa att_17_maa_x 0=2
BinaryOp add_219 2 1 att_17_xx_1 att_17_maa_x att_17_maa_rwkvag 0=0
Slice slice_att_17_maa_rwkvag 1 6 att_17_maa_rwkvag att_17_mr att_17_mw att_17_mk att_17_mv att_17_ma att_17_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_17_mv 1 2 att_17_mv att_17_mv_0 att_17_mv_1
Gemm gemm_att_17_key 1 1 att_17_mk att_17_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_value 1 1 att_17_mv_0 att_17_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_receptance 1 1 att_17_mr att_17_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_ma_lora_0 1 1 att_17_ma att_17_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_17_ma_lora_1 1 1 att_17_ma_lora_0 att_17_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_17_a0 0 1 att_17_a0 0=1024 21=0
BinaryOp add_220 2 1 att_17_ma_lora_1 att_17_a0 att_17_ma_lora_2 0=0
Sigmoid sigmoid_67 1 1 att_17_ma_lora_2 att_17_a
Split split_att_17_a 1 2 att_17_a att_17_a_0 att_17_a_1
Gemm gemm_att_17_mg_lora 1 1 att_17_mg att_17_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_68 1 1 att_17_mg_lora att_17_mg_lora_sigmoid
Gemm gemm_att_17_gate 1 1 att_17_mg_lora_sigmoid att_17_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_17_k_k 0 1 att_17_k_k 0=1024 21=0
MemoryData data_att_17_k_a 0 1 att_17_k_a 0=1024 21=0
MemoryData data_att_17_r_k 0 1 att_17_r_k 0=1024 21=0
Split split_att_17_key 1 2 att_17_key att_17_key_0 att_17_key_1
BinaryOp mul_221 2 1 att_17_key_0 att_17_k_k att_17_key_k 0=2
Reshape reshape_204 1 1 att_17_key_k att_17_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_17 1 1 att_17_key_k_reshape att_17_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_205 1 1 att_17_key_k_norm att_17_key_k_norm_reshape 0=1024
BinaryOp add_221 1 1 att_17_a_0 att_17_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_222 2 1 att_17_a_0_minus att_17_k_a att_17_a_minus1_mulka 0=2
BinaryOp add_222 1 1 att_17_a_minus1_mulka att_17_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_223 2 1 att_17_a_minus1_mulka_plus1 att_17_key_1 att_17_key_a 0=2
Split split_att_17_value 1 2 att_17_value att_17_value_0 att_17_value_1
Split split_v_first_16 1 2 v_first_16 att_17_v_first v_first_17
BinaryOp minus_16 2 1 att_17_v_first att_17_value_1 att_17_vfirst_minus_value 0=1
Gemm gemm_att_17_mv_lora_0 1 1 att_17_mv_1 att_17_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_17_mv_lora_1 1 1 att_17_mv_lora_0 att_17_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_17_v0 0 1 att_17_v0 0=1024 21=0
BinaryOp add_223 2 1 att_17_mv_lora_1 att_17_v0 att_17_mv_lora_2 0=0
Sigmoid sigmoid_69 1 1 att_17_mv_lora_2 att_17_mv_lora_sigmoid
BinaryOp mul_224 2 1 att_17_mv_lora_sigmoid att_17_vfirst_minus_value att_17_vfirst_minus_value_mul 0=2
BinaryOp add_224 2 1 att_17_value_0 att_17_vfirst_minus_value_mul att_17_value_final 0=0
Gemm gemm_att_17_mw_lora 1 1 att_17_mw att_17_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_17_mw_lora 1 1 att_17_mw_lora att_17_mw_lora_tanh
Gemm gemm_att_17_mw_lora_tanh_linear 1 1 att_17_mw_lora_tanh att_17_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_17_td 0 1 att_17_td 0=1024 21=0
BinaryOp add_225 2 1 att_17_mw_lora_tanh_linear att_17_td att_17_time_decay_pre 0=0
Sigmoid sigmoid_70 1 1 att_17_time_decay_pre att_17_time_decay_sigmoid
Exp exp_17 1 1 att_17_time_decay_sigmoid att_17_time_decay 1=-0.606531
Reshape reshape_206 1 1 att_17_time_decay att_17_time_decay_reshape 0=64 1=1 2=16
Split split_att_17_key_a 1 2 att_17_key_a att_17_key_a_0 att_17_key_a_1
Reshape reshape_207 1 1 att_17_key_a_0 att_17_key_a_reshape 0=64 1=1 2=16
Split split_att_17_value_final 1 3 att_17_value_final att_17_value_final_0 att_17_value_final_1 att_17_value_final_2
Reshape reshape_208 1 1 att_17_value_final_0 att_17_value_final_0_reshape 0=64 1=16
Reshape reshape_209 1 1 att_17_value_final_1 att_17_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_17_vk 2 1 att_17_value_final_1_reshape att_17_key_a_reshape att_17_vk 0=0
Split split_att_17_key_k_norm_reshape 1 2 att_17_key_k_norm_reshape att_17_kk_0 att_17_kk_1
BinaryOp mul_225 1 1 att_17_kk_0 att_17_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_210 1 1 att_17_kk_0_neg att_17_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_226 2 1 att_17_kk_1 att_17_a_1 att_17_kk_a 0=2
Reshape reshape_211 1 1 att_17_kk_a att_17_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_17_ab 2 1 att_17_kk_0_neg_reshape att_17_kk_a_reshape att_17_ab 0=0
Split split_state_52_in 1 2 state_52_in att_17_state_prev_0 att_17_state_prev_1
BinaryOp mul_227 2 1 att_17_state_prev_0 att_17_time_decay_reshape att_17_state_td 0=2
MatMul matmul_att_17_sab 2 1 att_17_state_prev_1 att_17_ab att_17_sab 0=0
BinaryOp add_226 2 1 att_17_sab att_17_vk att_17_sab_vk 0=0
BinaryOp add_227 2 1 att_17_sab_vk att_17_state_td att_17_state_new 0=0
Split split_att_17_state_new 1 2 att_17_state_new att_17_state_new_0 state_52_out
Split split_att_17_receptance 1 2 att_17_receptance att_17_receptance_0 att_17_receptance_1
Reshape reshape_212 1 1 att_17_receptance_0 att_17_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_17_wkv_out 2 1 att_17_state_new_0 att_17_receptance_reshape att_17_wkv_out 0=0
Reshape reshape_213 1 1 att_17_wkv_out att_17_wkv_out_flatten 0=1024
GroupNorm groupnorm_17 1 1 att_17_wkv_out_flatten att_17_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_228 2 1 att_17_receptance_1 att_17_key_a_1 att_17_receptance_key_a 0=2
BinaryOp mul_229 2 1 att_17_receptance_key_a att_17_r_k att_17_rk 0=2
Reshape reshape_214 1 1 att_17_rk att_17_rk_reshape 0=64 1=16
Reduction sum_17 1 1 att_17_rk_reshape att_17_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_230 2 1 att_17_value_final_0_reshape att_17_rk_sum att_17_value_final_0_rk 0=2
Reshape reshape_215 1 1 att_17_value_final_0_rk att_17_value_final_0_rk_reshape 0=1024
BinaryOp add_228 2 1 att_17_value_final_0_rk_reshape att_17_x_gn att_17_x_gn_rkv 0=0
BinaryOp mul_231 2 1 att_17_x_gn_rkv att_17_gate att_17_x_gate 0=2
Gemm gemm_att_17_x_out 1 1 att_17_x_gate att_17_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_229 2 1 att_17_x_out att_17_x_last time_mixing_17_out 0=0
Split split_time_mixing_17_out 1 2 time_mixing_17_out ffn_17_x_last ffn_17_x
LayerNorm layernorm_35 1 1 ffn_17_x ffn_17_xx 0=1024 1=0.00001 2=1
Split split_ffn_17_xx 1 3 ffn_17_xx ffn_17_xx_0 ffn_17_xx_1 state_53_out
BinaryOp sub_35 2 1 state_53_in ffn_17_xx_0 ffn_17_sx 0=1
Split split_ffn_17_sx 1 1 ffn_17_sx ffn_17_sx_0
MemoryData data_ffn_17_x_k 0 1 ffn_17_x_k 0=1024 21=0
BinaryOp mul_232 2 1 ffn_17_sx_0 ffn_17_x_k ffn_17_xk 0=2
BinaryOp add_230 2 1 ffn_17_xk ffn_17_xx_1 ffn_17_xxk 0=0
Gemm gemm_ffn_17_key 1 1 ffn_17_xxk ffn_17_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_17 1 1 ffn_17_key ffn_17_key_relu
UnaryOp square_ffn_17_key_relu 1 1 ffn_17_key_relu ffn_17_key_relu_square 0=4
Gemm gemm_ffn_17_value 1 1 ffn_17_key_relu_square ffn_17_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_231 2 1 ffn_17_value ffn_17_x_last channel_mixing_17_out 0=0
Split split_channel_mixing_17_out 1 2 channel_mixing_17_out att_18_x_last att_18_x
LayerNorm layernorm_36 1 1 att_18_x att_18_xx 0=1024 1=0.00001 2=1
Split split_att_18_xx 1 3 att_18_xx att_18_xx_0 att_18_xx_1 state_54_out
BinaryOp sub_36 2 1 state_54_in att_18_xx_0 att_18_sx 0=1
MemoryData data_att_18_maa 0 1 att_18_maa 0=1024 1=6 21=0
BinaryOp mul_233 2 1 att_18_sx att_18_maa att_18_maa_x 0=2
BinaryOp add_232 2 1 att_18_xx_1 att_18_maa_x att_18_maa_rwkvag 0=0
Slice slice_att_18_maa_rwkvag 1 6 att_18_maa_rwkvag att_18_mr att_18_mw att_18_mk att_18_mv att_18_ma att_18_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_18_mv 1 2 att_18_mv att_18_mv_0 att_18_mv_1
Gemm gemm_att_18_key 1 1 att_18_mk att_18_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_value 1 1 att_18_mv_0 att_18_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_receptance 1 1 att_18_mr att_18_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_ma_lora_0 1 1 att_18_ma att_18_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_18_ma_lora_1 1 1 att_18_ma_lora_0 att_18_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_18_a0 0 1 att_18_a0 0=1024 21=0
BinaryOp add_233 2 1 att_18_ma_lora_1 att_18_a0 att_18_ma_lora_2 0=0
Sigmoid sigmoid_71 1 1 att_18_ma_lora_2 att_18_a
Split split_att_18_a 1 2 att_18_a att_18_a_0 att_18_a_1
Gemm gemm_att_18_mg_lora 1 1 att_18_mg att_18_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_72 1 1 att_18_mg_lora att_18_mg_lora_sigmoid
Gemm gemm_att_18_gate 1 1 att_18_mg_lora_sigmoid att_18_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_18_k_k 0 1 att_18_k_k 0=1024 21=0
MemoryData data_att_18_k_a 0 1 att_18_k_a 0=1024 21=0
MemoryData data_att_18_r_k 0 1 att_18_r_k 0=1024 21=0
Split split_att_18_key 1 2 att_18_key att_18_key_0 att_18_key_1
BinaryOp mul_234 2 1 att_18_key_0 att_18_k_k att_18_key_k 0=2
Reshape reshape_216 1 1 att_18_key_k att_18_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_18 1 1 att_18_key_k_reshape att_18_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_217 1 1 att_18_key_k_norm att_18_key_k_norm_reshape 0=1024
BinaryOp add_234 1 1 att_18_a_0 att_18_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_235 2 1 att_18_a_0_minus att_18_k_a att_18_a_minus1_mulka 0=2
BinaryOp add_235 1 1 att_18_a_minus1_mulka att_18_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_236 2 1 att_18_a_minus1_mulka_plus1 att_18_key_1 att_18_key_a 0=2
Split split_att_18_value 1 2 att_18_value att_18_value_0 att_18_value_1
Split split_v_first_17 1 2 v_first_17 att_18_v_first v_first_18
BinaryOp minus_17 2 1 att_18_v_first att_18_value_1 att_18_vfirst_minus_value 0=1
Gemm gemm_att_18_mv_lora_0 1 1 att_18_mv_1 att_18_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_18_mv_lora_1 1 1 att_18_mv_lora_0 att_18_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_18_v0 0 1 att_18_v0 0=1024 21=0
BinaryOp add_236 2 1 att_18_mv_lora_1 att_18_v0 att_18_mv_lora_2 0=0
Sigmoid sigmoid_73 1 1 att_18_mv_lora_2 att_18_mv_lora_sigmoid
BinaryOp mul_237 2 1 att_18_mv_lora_sigmoid att_18_vfirst_minus_value att_18_vfirst_minus_value_mul 0=2
BinaryOp add_237 2 1 att_18_value_0 att_18_vfirst_minus_value_mul att_18_value_final 0=0
Gemm gemm_att_18_mw_lora 1 1 att_18_mw att_18_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_18_mw_lora 1 1 att_18_mw_lora att_18_mw_lora_tanh
Gemm gemm_att_18_mw_lora_tanh_linear 1 1 att_18_mw_lora_tanh att_18_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_18_td 0 1 att_18_td 0=1024 21=0
BinaryOp add_238 2 1 att_18_mw_lora_tanh_linear att_18_td att_18_time_decay_pre 0=0
Sigmoid sigmoid_74 1 1 att_18_time_decay_pre att_18_time_decay_sigmoid
Exp exp_18 1 1 att_18_time_decay_sigmoid att_18_time_decay 1=-0.606531
Reshape reshape_218 1 1 att_18_time_decay att_18_time_decay_reshape 0=64 1=1 2=16
Split split_att_18_key_a 1 2 att_18_key_a att_18_key_a_0 att_18_key_a_1
Reshape reshape_219 1 1 att_18_key_a_0 att_18_key_a_reshape 0=64 1=1 2=16
Split split_att_18_value_final 1 3 att_18_value_final att_18_value_final_0 att_18_value_final_1 att_18_value_final_2
Reshape reshape_220 1 1 att_18_value_final_0 att_18_value_final_0_reshape 0=64 1=16
Reshape reshape_221 1 1 att_18_value_final_1 att_18_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_18_vk 2 1 att_18_value_final_1_reshape att_18_key_a_reshape att_18_vk 0=0
Split split_att_18_key_k_norm_reshape 1 2 att_18_key_k_norm_reshape att_18_kk_0 att_18_kk_1
BinaryOp mul_238 1 1 att_18_kk_0 att_18_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_222 1 1 att_18_kk_0_neg att_18_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_239 2 1 att_18_kk_1 att_18_a_1 att_18_kk_a 0=2
Reshape reshape_223 1 1 att_18_kk_a att_18_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_18_ab 2 1 att_18_kk_0_neg_reshape att_18_kk_a_reshape att_18_ab 0=0
Split split_state_55_in 1 2 state_55_in att_18_state_prev_0 att_18_state_prev_1
BinaryOp mul_240 2 1 att_18_state_prev_0 att_18_time_decay_reshape att_18_state_td 0=2
MatMul matmul_att_18_sab 2 1 att_18_state_prev_1 att_18_ab att_18_sab 0=0
BinaryOp add_239 2 1 att_18_sab att_18_vk att_18_sab_vk 0=0
BinaryOp add_240 2 1 att_18_sab_vk att_18_state_td att_18_state_new 0=0
Split split_att_18_state_new 1 2 att_18_state_new att_18_state_new_0 state_55_out
Split split_att_18_receptance 1 2 att_18_receptance att_18_receptance_0 att_18_receptance_1
Reshape reshape_224 1 1 att_18_receptance_0 att_18_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_18_wkv_out 2 1 att_18_state_new_0 att_18_receptance_reshape att_18_wkv_out 0=0
Reshape reshape_225 1 1 att_18_wkv_out att_18_wkv_out_flatten 0=1024
GroupNorm groupnorm_18 1 1 att_18_wkv_out_flatten att_18_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_241 2 1 att_18_receptance_1 att_18_key_a_1 att_18_receptance_key_a 0=2
BinaryOp mul_242 2 1 att_18_receptance_key_a att_18_r_k att_18_rk 0=2
Reshape reshape_226 1 1 att_18_rk att_18_rk_reshape 0=64 1=16
Reduction sum_18 1 1 att_18_rk_reshape att_18_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_243 2 1 att_18_value_final_0_reshape att_18_rk_sum att_18_value_final_0_rk 0=2
Reshape reshape_227 1 1 att_18_value_final_0_rk att_18_value_final_0_rk_reshape 0=1024
BinaryOp add_241 2 1 att_18_value_final_0_rk_reshape att_18_x_gn att_18_x_gn_rkv 0=0
BinaryOp mul_244 2 1 att_18_x_gn_rkv att_18_gate att_18_x_gate 0=2
Gemm gemm_att_18_x_out 1 1 att_18_x_gate att_18_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_242 2 1 att_18_x_out att_18_x_last time_mixing_18_out 0=0
Split split_time_mixing_18_out 1 2 time_mixing_18_out ffn_18_x_last ffn_18_x
LayerNorm layernorm_37 1 1 ffn_18_x ffn_18_xx 0=1024 1=0.00001 2=1
Split split_ffn_18_xx 1 3 ffn_18_xx ffn_18_xx_0 ffn_18_xx_1 state_56_out
BinaryOp sub_37 2 1 state_56_in ffn_18_xx_0 ffn_18_sx 0=1
Split split_ffn_18_sx 1 1 ffn_18_sx ffn_18_sx_0
MemoryData data_ffn_18_x_k 0 1 ffn_18_x_k 0=1024 21=0
BinaryOp mul_245 2 1 ffn_18_sx_0 ffn_18_x_k ffn_18_xk 0=2
BinaryOp add_243 2 1 ffn_18_xk ffn_18_xx_1 ffn_18_xxk 0=0
Gemm gemm_ffn_18_key 1 1 ffn_18_xxk ffn_18_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_18 1 1 ffn_18_key ffn_18_key_relu
UnaryOp square_ffn_18_key_relu 1 1 ffn_18_key_relu ffn_18_key_relu_square 0=4
Gemm gemm_ffn_18_value 1 1 ffn_18_key_relu_square ffn_18_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_244 2 1 ffn_18_value ffn_18_x_last channel_mixing_18_out 0=0
Split split_channel_mixing_18_out 1 2 channel_mixing_18_out att_19_x_last att_19_x
LayerNorm layernorm_38 1 1 att_19_x att_19_xx 0=1024 1=0.00001 2=1
Split split_att_19_xx 1 3 att_19_xx att_19_xx_0 att_19_xx_1 state_57_out
BinaryOp sub_38 2 1 state_57_in att_19_xx_0 att_19_sx 0=1
MemoryData data_att_19_maa 0 1 att_19_maa 0=1024 1=6 21=0
BinaryOp mul_246 2 1 att_19_sx att_19_maa att_19_maa_x 0=2
BinaryOp add_245 2 1 att_19_xx_1 att_19_maa_x att_19_maa_rwkvag 0=0
Slice slice_att_19_maa_rwkvag 1 6 att_19_maa_rwkvag att_19_mr att_19_mw att_19_mk att_19_mv att_19_ma att_19_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_19_mv 1 2 att_19_mv att_19_mv_0 att_19_mv_1
Gemm gemm_att_19_key 1 1 att_19_mk att_19_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_value 1 1 att_19_mv_0 att_19_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_receptance 1 1 att_19_mr att_19_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_ma_lora_0 1 1 att_19_ma att_19_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_19_ma_lora_1 1 1 att_19_ma_lora_0 att_19_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_19_a0 0 1 att_19_a0 0=1024 21=0
BinaryOp add_246 2 1 att_19_ma_lora_1 att_19_a0 att_19_ma_lora_2 0=0
Sigmoid sigmoid_75 1 1 att_19_ma_lora_2 att_19_a
Split split_att_19_a 1 2 att_19_a att_19_a_0 att_19_a_1
Gemm gemm_att_19_mg_lora 1 1 att_19_mg att_19_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_76 1 1 att_19_mg_lora att_19_mg_lora_sigmoid
Gemm gemm_att_19_gate 1 1 att_19_mg_lora_sigmoid att_19_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_19_k_k 0 1 att_19_k_k 0=1024 21=0
MemoryData data_att_19_k_a 0 1 att_19_k_a 0=1024 21=0
MemoryData data_att_19_r_k 0 1 att_19_r_k 0=1024 21=0
Split split_att_19_key 1 2 att_19_key att_19_key_0 att_19_key_1
BinaryOp mul_247 2 1 att_19_key_0 att_19_k_k att_19_key_k 0=2
Reshape reshape_228 1 1 att_19_key_k att_19_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_19 1 1 att_19_key_k_reshape att_19_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_229 1 1 att_19_key_k_norm att_19_key_k_norm_reshape 0=1024
BinaryOp add_247 1 1 att_19_a_0 att_19_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_248 2 1 att_19_a_0_minus att_19_k_a att_19_a_minus1_mulka 0=2
BinaryOp add_248 1 1 att_19_a_minus1_mulka att_19_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_249 2 1 att_19_a_minus1_mulka_plus1 att_19_key_1 att_19_key_a 0=2
Split split_att_19_value 1 2 att_19_value att_19_value_0 att_19_value_1
Split split_v_first_18 1 2 v_first_18 att_19_v_first v_first_19
BinaryOp minus_18 2 1 att_19_v_first att_19_value_1 att_19_vfirst_minus_value 0=1
Gemm gemm_att_19_mv_lora_0 1 1 att_19_mv_1 att_19_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_19_mv_lora_1 1 1 att_19_mv_lora_0 att_19_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_19_v0 0 1 att_19_v0 0=1024 21=0
BinaryOp add_249 2 1 att_19_mv_lora_1 att_19_v0 att_19_mv_lora_2 0=0
Sigmoid sigmoid_77 1 1 att_19_mv_lora_2 att_19_mv_lora_sigmoid
BinaryOp mul_250 2 1 att_19_mv_lora_sigmoid att_19_vfirst_minus_value att_19_vfirst_minus_value_mul 0=2
BinaryOp add_250 2 1 att_19_value_0 att_19_vfirst_minus_value_mul att_19_value_final 0=0
Gemm gemm_att_19_mw_lora 1 1 att_19_mw att_19_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_19_mw_lora 1 1 att_19_mw_lora att_19_mw_lora_tanh
Gemm gemm_att_19_mw_lora_tanh_linear 1 1 att_19_mw_lora_tanh att_19_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_19_td 0 1 att_19_td 0=1024 21=0
BinaryOp add_251 2 1 att_19_mw_lora_tanh_linear att_19_td att_19_time_decay_pre 0=0
Sigmoid sigmoid_78 1 1 att_19_time_decay_pre att_19_time_decay_sigmoid
Exp exp_19 1 1 att_19_time_decay_sigmoid att_19_time_decay 1=-0.606531
Reshape reshape_230 1 1 att_19_time_decay att_19_time_decay_reshape 0=64 1=1 2=16
Split split_att_19_key_a 1 2 att_19_key_a att_19_key_a_0 att_19_key_a_1
Reshape reshape_231 1 1 att_19_key_a_0 att_19_key_a_reshape 0=64 1=1 2=16
Split split_att_19_value_final 1 3 att_19_value_final att_19_value_final_0 att_19_value_final_1 att_19_value_final_2
Reshape reshape_232 1 1 att_19_value_final_0 att_19_value_final_0_reshape 0=64 1=16
Reshape reshape_233 1 1 att_19_value_final_1 att_19_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_19_vk 2 1 att_19_value_final_1_reshape att_19_key_a_reshape att_19_vk 0=0
Split split_att_19_key_k_norm_reshape 1 2 att_19_key_k_norm_reshape att_19_kk_0 att_19_kk_1
BinaryOp mul_251 1 1 att_19_kk_0 att_19_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_234 1 1 att_19_kk_0_neg att_19_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_252 2 1 att_19_kk_1 att_19_a_1 att_19_kk_a 0=2
Reshape reshape_235 1 1 att_19_kk_a att_19_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_19_ab 2 1 att_19_kk_0_neg_reshape att_19_kk_a_reshape att_19_ab 0=0
Split split_state_58_in 1 2 state_58_in att_19_state_prev_0 att_19_state_prev_1
BinaryOp mul_253 2 1 att_19_state_prev_0 att_19_time_decay_reshape att_19_state_td 0=2
MatMul matmul_att_19_sab 2 1 att_19_state_prev_1 att_19_ab att_19_sab 0=0
BinaryOp add_252 2 1 att_19_sab att_19_vk att_19_sab_vk 0=0
BinaryOp add_253 2 1 att_19_sab_vk att_19_state_td att_19_state_new 0=0
Split split_att_19_state_new 1 2 att_19_state_new att_19_state_new_0 state_58_out
Split split_att_19_receptance 1 2 att_19_receptance att_19_receptance_0 att_19_receptance_1
Reshape reshape_236 1 1 att_19_receptance_0 att_19_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_19_wkv_out 2 1 att_19_state_new_0 att_19_receptance_reshape att_19_wkv_out 0=0
Reshape reshape_237 1 1 att_19_wkv_out att_19_wkv_out_flatten 0=1024
GroupNorm groupnorm_19 1 1 att_19_wkv_out_flatten att_19_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_254 2 1 att_19_receptance_1 att_19_key_a_1 att_19_receptance_key_a 0=2
BinaryOp mul_255 2 1 att_19_receptance_key_a att_19_r_k att_19_rk 0=2
Reshape reshape_238 1 1 att_19_rk att_19_rk_reshape 0=64 1=16
Reduction sum_19 1 1 att_19_rk_reshape att_19_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_256 2 1 att_19_value_final_0_reshape att_19_rk_sum att_19_value_final_0_rk 0=2
Reshape reshape_239 1 1 att_19_value_final_0_rk att_19_value_final_0_rk_reshape 0=1024
BinaryOp add_254 2 1 att_19_value_final_0_rk_reshape att_19_x_gn att_19_x_gn_rkv 0=0
BinaryOp mul_257 2 1 att_19_x_gn_rkv att_19_gate att_19_x_gate 0=2
Gemm gemm_att_19_x_out 1 1 att_19_x_gate att_19_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_255 2 1 att_19_x_out att_19_x_last time_mixing_19_out 0=0
Split split_time_mixing_19_out 1 2 time_mixing_19_out ffn_19_x_last ffn_19_x
LayerNorm layernorm_39 1 1 ffn_19_x ffn_19_xx 0=1024 1=0.00001 2=1
Split split_ffn_19_xx 1 3 ffn_19_xx ffn_19_xx_0 ffn_19_xx_1 state_59_out
BinaryOp sub_39 2 1 state_59_in ffn_19_xx_0 ffn_19_sx 0=1
Split split_ffn_19_sx 1 1 ffn_19_sx ffn_19_sx_0
MemoryData data_ffn_19_x_k 0 1 ffn_19_x_k 0=1024 21=0
BinaryOp mul_258 2 1 ffn_19_sx_0 ffn_19_x_k ffn_19_xk 0=2
BinaryOp add_256 2 1 ffn_19_xk ffn_19_xx_1 ffn_19_xxk 0=0
Gemm gemm_ffn_19_key 1 1 ffn_19_xxk ffn_19_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_19 1 1 ffn_19_key ffn_19_key_relu
UnaryOp square_ffn_19_key_relu 1 1 ffn_19_key_relu ffn_19_key_relu_square 0=4
Gemm gemm_ffn_19_value 1 1 ffn_19_key_relu_square ffn_19_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_257 2 1 ffn_19_value ffn_19_x_last channel_mixing_19_out 0=0
Split split_channel_mixing_19_out 1 2 channel_mixing_19_out att_20_x_last att_20_x
LayerNorm layernorm_40 1 1 att_20_x att_20_xx 0=1024 1=0.00001 2=1
Split split_att_20_xx 1 3 att_20_xx att_20_xx_0 att_20_xx_1 state_60_out
BinaryOp sub_40 2 1 state_60_in att_20_xx_0 att_20_sx 0=1
MemoryData data_att_20_maa 0 1 att_20_maa 0=1024 1=6 21=0
BinaryOp mul_259 2 1 att_20_sx att_20_maa att_20_maa_x 0=2
BinaryOp add_258 2 1 att_20_xx_1 att_20_maa_x att_20_maa_rwkvag 0=0
Slice slice_att_20_maa_rwkvag 1 6 att_20_maa_rwkvag att_20_mr att_20_mw att_20_mk att_20_mv att_20_ma att_20_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_20_mv 1 2 att_20_mv att_20_mv_0 att_20_mv_1
Gemm gemm_att_20_key 1 1 att_20_mk att_20_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_value 1 1 att_20_mv_0 att_20_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_receptance 1 1 att_20_mr att_20_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_ma_lora_0 1 1 att_20_ma att_20_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_20_ma_lora_1 1 1 att_20_ma_lora_0 att_20_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_20_a0 0 1 att_20_a0 0=1024 21=0
BinaryOp add_259 2 1 att_20_ma_lora_1 att_20_a0 att_20_ma_lora_2 0=0
Sigmoid sigmoid_79 1 1 att_20_ma_lora_2 att_20_a
Split split_att_20_a 1 2 att_20_a att_20_a_0 att_20_a_1
Gemm gemm_att_20_mg_lora 1 1 att_20_mg att_20_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_80 1 1 att_20_mg_lora att_20_mg_lora_sigmoid
Gemm gemm_att_20_gate 1 1 att_20_mg_lora_sigmoid att_20_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_20_k_k 0 1 att_20_k_k 0=1024 21=0
MemoryData data_att_20_k_a 0 1 att_20_k_a 0=1024 21=0
MemoryData data_att_20_r_k 0 1 att_20_r_k 0=1024 21=0
Split split_att_20_key 1 2 att_20_key att_20_key_0 att_20_key_1
BinaryOp mul_260 2 1 att_20_key_0 att_20_k_k att_20_key_k 0=2
Reshape reshape_240 1 1 att_20_key_k att_20_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_20 1 1 att_20_key_k_reshape att_20_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_241 1 1 att_20_key_k_norm att_20_key_k_norm_reshape 0=1024
BinaryOp add_260 1 1 att_20_a_0 att_20_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_261 2 1 att_20_a_0_minus att_20_k_a att_20_a_minus1_mulka 0=2
BinaryOp add_261 1 1 att_20_a_minus1_mulka att_20_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_262 2 1 att_20_a_minus1_mulka_plus1 att_20_key_1 att_20_key_a 0=2
Split split_att_20_value 1 2 att_20_value att_20_value_0 att_20_value_1
Split split_v_first_19 1 2 v_first_19 att_20_v_first v_first_20
BinaryOp minus_19 2 1 att_20_v_first att_20_value_1 att_20_vfirst_minus_value 0=1
Gemm gemm_att_20_mv_lora_0 1 1 att_20_mv_1 att_20_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_20_mv_lora_1 1 1 att_20_mv_lora_0 att_20_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_20_v0 0 1 att_20_v0 0=1024 21=0
BinaryOp add_262 2 1 att_20_mv_lora_1 att_20_v0 att_20_mv_lora_2 0=0
Sigmoid sigmoid_81 1 1 att_20_mv_lora_2 att_20_mv_lora_sigmoid
BinaryOp mul_263 2 1 att_20_mv_lora_sigmoid att_20_vfirst_minus_value att_20_vfirst_minus_value_mul 0=2
BinaryOp add_263 2 1 att_20_value_0 att_20_vfirst_minus_value_mul att_20_value_final 0=0
Gemm gemm_att_20_mw_lora 1 1 att_20_mw att_20_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_20_mw_lora 1 1 att_20_mw_lora att_20_mw_lora_tanh
Gemm gemm_att_20_mw_lora_tanh_linear 1 1 att_20_mw_lora_tanh att_20_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_20_td 0 1 att_20_td 0=1024 21=0
BinaryOp add_264 2 1 att_20_mw_lora_tanh_linear att_20_td att_20_time_decay_pre 0=0
Sigmoid sigmoid_82 1 1 att_20_time_decay_pre att_20_time_decay_sigmoid
Exp exp_20 1 1 att_20_time_decay_sigmoid att_20_time_decay 1=-0.606531
Reshape reshape_242 1 1 att_20_time_decay att_20_time_decay_reshape 0=64 1=1 2=16
Split split_att_20_key_a 1 2 att_20_key_a att_20_key_a_0 att_20_key_a_1
Reshape reshape_243 1 1 att_20_key_a_0 att_20_key_a_reshape 0=64 1=1 2=16
Split split_att_20_value_final 1 3 att_20_value_final att_20_value_final_0 att_20_value_final_1 att_20_value_final_2
Reshape reshape_244 1 1 att_20_value_final_0 att_20_value_final_0_reshape 0=64 1=16
Reshape reshape_245 1 1 att_20_value_final_1 att_20_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_20_vk 2 1 att_20_value_final_1_reshape att_20_key_a_reshape att_20_vk 0=0
Split split_att_20_key_k_norm_reshape 1 2 att_20_key_k_norm_reshape att_20_kk_0 att_20_kk_1
BinaryOp mul_264 1 1 att_20_kk_0 att_20_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_246 1 1 att_20_kk_0_neg att_20_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_265 2 1 att_20_kk_1 att_20_a_1 att_20_kk_a 0=2
Reshape reshape_247 1 1 att_20_kk_a att_20_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_20_ab 2 1 att_20_kk_0_neg_reshape att_20_kk_a_reshape att_20_ab 0=0
Split split_state_61_in 1 2 state_61_in att_20_state_prev_0 att_20_state_prev_1
BinaryOp mul_266 2 1 att_20_state_prev_0 att_20_time_decay_reshape att_20_state_td 0=2
MatMul matmul_att_20_sab 2 1 att_20_state_prev_1 att_20_ab att_20_sab 0=0
BinaryOp add_265 2 1 att_20_sab att_20_vk att_20_sab_vk 0=0
BinaryOp add_266 2 1 att_20_sab_vk att_20_state_td att_20_state_new 0=0
Split split_att_20_state_new 1 2 att_20_state_new att_20_state_new_0 state_61_out
Split split_att_20_receptance 1 2 att_20_receptance att_20_receptance_0 att_20_receptance_1
Reshape reshape_248 1 1 att_20_receptance_0 att_20_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_20_wkv_out 2 1 att_20_state_new_0 att_20_receptance_reshape att_20_wkv_out 0=0
Reshape reshape_249 1 1 att_20_wkv_out att_20_wkv_out_flatten 0=1024
GroupNorm groupnorm_20 1 1 att_20_wkv_out_flatten att_20_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_267 2 1 att_20_receptance_1 att_20_key_a_1 att_20_receptance_key_a 0=2
BinaryOp mul_268 2 1 att_20_receptance_key_a att_20_r_k att_20_rk 0=2
Reshape reshape_250 1 1 att_20_rk att_20_rk_reshape 0=64 1=16
Reduction sum_20 1 1 att_20_rk_reshape att_20_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_269 2 1 att_20_value_final_0_reshape att_20_rk_sum att_20_value_final_0_rk 0=2
Reshape reshape_251 1 1 att_20_value_final_0_rk att_20_value_final_0_rk_reshape 0=1024
BinaryOp add_267 2 1 att_20_value_final_0_rk_reshape att_20_x_gn att_20_x_gn_rkv 0=0
BinaryOp mul_270 2 1 att_20_x_gn_rkv att_20_gate att_20_x_gate 0=2
Gemm gemm_att_20_x_out 1 1 att_20_x_gate att_20_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_268 2 1 att_20_x_out att_20_x_last time_mixing_20_out 0=0
Split split_time_mixing_20_out 1 2 time_mixing_20_out ffn_20_x_last ffn_20_x
LayerNorm layernorm_41 1 1 ffn_20_x ffn_20_xx 0=1024 1=0.00001 2=1
Split split_ffn_20_xx 1 3 ffn_20_xx ffn_20_xx_0 ffn_20_xx_1 state_62_out
BinaryOp sub_41 2 1 state_62_in ffn_20_xx_0 ffn_20_sx 0=1
Split split_ffn_20_sx 1 1 ffn_20_sx ffn_20_sx_0
MemoryData data_ffn_20_x_k 0 1 ffn_20_x_k 0=1024 21=0
BinaryOp mul_271 2 1 ffn_20_sx_0 ffn_20_x_k ffn_20_xk 0=2
BinaryOp add_269 2 1 ffn_20_xk ffn_20_xx_1 ffn_20_xxk 0=0
Gemm gemm_ffn_20_key 1 1 ffn_20_xxk ffn_20_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_20 1 1 ffn_20_key ffn_20_key_relu
UnaryOp square_ffn_20_key_relu 1 1 ffn_20_key_relu ffn_20_key_relu_square 0=4
Gemm gemm_ffn_20_value 1 1 ffn_20_key_relu_square ffn_20_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_270 2 1 ffn_20_value ffn_20_x_last channel_mixing_20_out 0=0
Split split_channel_mixing_20_out 1 2 channel_mixing_20_out att_21_x_last att_21_x
LayerNorm layernorm_42 1 1 att_21_x att_21_xx 0=1024 1=0.00001 2=1
Split split_att_21_xx 1 3 att_21_xx att_21_xx_0 att_21_xx_1 state_63_out
BinaryOp sub_42 2 1 state_63_in att_21_xx_0 att_21_sx 0=1
MemoryData data_att_21_maa 0 1 att_21_maa 0=1024 1=6 21=0
BinaryOp mul_272 2 1 att_21_sx att_21_maa att_21_maa_x 0=2
BinaryOp add_271 2 1 att_21_xx_1 att_21_maa_x att_21_maa_rwkvag 0=0
Slice slice_att_21_maa_rwkvag 1 6 att_21_maa_rwkvag att_21_mr att_21_mw att_21_mk att_21_mv att_21_ma att_21_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_21_mv 1 2 att_21_mv att_21_mv_0 att_21_mv_1
Gemm gemm_att_21_key 1 1 att_21_mk att_21_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_value 1 1 att_21_mv_0 att_21_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_receptance 1 1 att_21_mr att_21_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_ma_lora_0 1 1 att_21_ma att_21_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_21_ma_lora_1 1 1 att_21_ma_lora_0 att_21_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_21_a0 0 1 att_21_a0 0=1024 21=0
BinaryOp add_272 2 1 att_21_ma_lora_1 att_21_a0 att_21_ma_lora_2 0=0
Sigmoid sigmoid_83 1 1 att_21_ma_lora_2 att_21_a
Split split_att_21_a 1 2 att_21_a att_21_a_0 att_21_a_1
Gemm gemm_att_21_mg_lora 1 1 att_21_mg att_21_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_84 1 1 att_21_mg_lora att_21_mg_lora_sigmoid
Gemm gemm_att_21_gate 1 1 att_21_mg_lora_sigmoid att_21_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_21_k_k 0 1 att_21_k_k 0=1024 21=0
MemoryData data_att_21_k_a 0 1 att_21_k_a 0=1024 21=0
MemoryData data_att_21_r_k 0 1 att_21_r_k 0=1024 21=0
Split split_att_21_key 1 2 att_21_key att_21_key_0 att_21_key_1
BinaryOp mul_273 2 1 att_21_key_0 att_21_k_k att_21_key_k 0=2
Reshape reshape_252 1 1 att_21_key_k att_21_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_21 1 1 att_21_key_k_reshape att_21_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_253 1 1 att_21_key_k_norm att_21_key_k_norm_reshape 0=1024
BinaryOp add_273 1 1 att_21_a_0 att_21_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_274 2 1 att_21_a_0_minus att_21_k_a att_21_a_minus1_mulka 0=2
BinaryOp add_274 1 1 att_21_a_minus1_mulka att_21_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_275 2 1 att_21_a_minus1_mulka_plus1 att_21_key_1 att_21_key_a 0=2
Split split_att_21_value 1 2 att_21_value att_21_value_0 att_21_value_1
Split split_v_first_20 1 2 v_first_20 att_21_v_first v_first_21
BinaryOp minus_20 2 1 att_21_v_first att_21_value_1 att_21_vfirst_minus_value 0=1
Gemm gemm_att_21_mv_lora_0 1 1 att_21_mv_1 att_21_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_21_mv_lora_1 1 1 att_21_mv_lora_0 att_21_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_21_v0 0 1 att_21_v0 0=1024 21=0
BinaryOp add_275 2 1 att_21_mv_lora_1 att_21_v0 att_21_mv_lora_2 0=0
Sigmoid sigmoid_85 1 1 att_21_mv_lora_2 att_21_mv_lora_sigmoid
BinaryOp mul_276 2 1 att_21_mv_lora_sigmoid att_21_vfirst_minus_value att_21_vfirst_minus_value_mul 0=2
BinaryOp add_276 2 1 att_21_value_0 att_21_vfirst_minus_value_mul att_21_value_final 0=0
Gemm gemm_att_21_mw_lora 1 1 att_21_mw att_21_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_21_mw_lora 1 1 att_21_mw_lora att_21_mw_lora_tanh
Gemm gemm_att_21_mw_lora_tanh_linear 1 1 att_21_mw_lora_tanh att_21_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_21_td 0 1 att_21_td 0=1024 21=0
BinaryOp add_277 2 1 att_21_mw_lora_tanh_linear att_21_td att_21_time_decay_pre 0=0
Sigmoid sigmoid_86 1 1 att_21_time_decay_pre att_21_time_decay_sigmoid
Exp exp_21 1 1 att_21_time_decay_sigmoid att_21_time_decay 1=-0.606531
Reshape reshape_254 1 1 att_21_time_decay att_21_time_decay_reshape 0=64 1=1 2=16
Split split_att_21_key_a 1 2 att_21_key_a att_21_key_a_0 att_21_key_a_1
Reshape reshape_255 1 1 att_21_key_a_0 att_21_key_a_reshape 0=64 1=1 2=16
Split split_att_21_value_final 1 3 att_21_value_final att_21_value_final_0 att_21_value_final_1 att_21_value_final_2
Reshape reshape_256 1 1 att_21_value_final_0 att_21_value_final_0_reshape 0=64 1=16
Reshape reshape_257 1 1 att_21_value_final_1 att_21_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_21_vk 2 1 att_21_value_final_1_reshape att_21_key_a_reshape att_21_vk 0=0
Split split_att_21_key_k_norm_reshape 1 2 att_21_key_k_norm_reshape att_21_kk_0 att_21_kk_1
BinaryOp mul_277 1 1 att_21_kk_0 att_21_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_258 1 1 att_21_kk_0_neg att_21_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_278 2 1 att_21_kk_1 att_21_a_1 att_21_kk_a 0=2
Reshape reshape_259 1 1 att_21_kk_a att_21_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_21_ab 2 1 att_21_kk_0_neg_reshape att_21_kk_a_reshape att_21_ab 0=0
Split split_state_64_in 1 2 state_64_in att_21_state_prev_0 att_21_state_prev_1
BinaryOp mul_279 2 1 att_21_state_prev_0 att_21_time_decay_reshape att_21_state_td 0=2
MatMul matmul_att_21_sab 2 1 att_21_state_prev_1 att_21_ab att_21_sab 0=0
BinaryOp add_278 2 1 att_21_sab att_21_vk att_21_sab_vk 0=0
BinaryOp add_279 2 1 att_21_sab_vk att_21_state_td att_21_state_new 0=0
Split split_att_21_state_new 1 2 att_21_state_new att_21_state_new_0 state_64_out
Split split_att_21_receptance 1 2 att_21_receptance att_21_receptance_0 att_21_receptance_1
Reshape reshape_260 1 1 att_21_receptance_0 att_21_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_21_wkv_out 2 1 att_21_state_new_0 att_21_receptance_reshape att_21_wkv_out 0=0
Reshape reshape_261 1 1 att_21_wkv_out att_21_wkv_out_flatten 0=1024
GroupNorm groupnorm_21 1 1 att_21_wkv_out_flatten att_21_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_280 2 1 att_21_receptance_1 att_21_key_a_1 att_21_receptance_key_a 0=2
BinaryOp mul_281 2 1 att_21_receptance_key_a att_21_r_k att_21_rk 0=2
Reshape reshape_262 1 1 att_21_rk att_21_rk_reshape 0=64 1=16
Reduction sum_21 1 1 att_21_rk_reshape att_21_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_282 2 1 att_21_value_final_0_reshape att_21_rk_sum att_21_value_final_0_rk 0=2
Reshape reshape_263 1 1 att_21_value_final_0_rk att_21_value_final_0_rk_reshape 0=1024
BinaryOp add_280 2 1 att_21_value_final_0_rk_reshape att_21_x_gn att_21_x_gn_rkv 0=0
BinaryOp mul_283 2 1 att_21_x_gn_rkv att_21_gate att_21_x_gate 0=2
Gemm gemm_att_21_x_out 1 1 att_21_x_gate att_21_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_281 2 1 att_21_x_out att_21_x_last time_mixing_21_out 0=0
Split split_time_mixing_21_out 1 2 time_mixing_21_out ffn_21_x_last ffn_21_x
LayerNorm layernorm_43 1 1 ffn_21_x ffn_21_xx 0=1024 1=0.00001 2=1
Split split_ffn_21_xx 1 3 ffn_21_xx ffn_21_xx_0 ffn_21_xx_1 state_65_out
BinaryOp sub_43 2 1 state_65_in ffn_21_xx_0 ffn_21_sx 0=1
Split split_ffn_21_sx 1 1 ffn_21_sx ffn_21_sx_0
MemoryData data_ffn_21_x_k 0 1 ffn_21_x_k 0=1024 21=0
BinaryOp mul_284 2 1 ffn_21_sx_0 ffn_21_x_k ffn_21_xk 0=2
BinaryOp add_282 2 1 ffn_21_xk ffn_21_xx_1 ffn_21_xxk 0=0
Gemm gemm_ffn_21_key 1 1 ffn_21_xxk ffn_21_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_21 1 1 ffn_21_key ffn_21_key_relu
UnaryOp square_ffn_21_key_relu 1 1 ffn_21_key_relu ffn_21_key_relu_square 0=4
Gemm gemm_ffn_21_value 1 1 ffn_21_key_relu_square ffn_21_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_283 2 1 ffn_21_value ffn_21_x_last channel_mixing_21_out 0=0
Split split_channel_mixing_21_out 1 2 channel_mixing_21_out att_22_x_last att_22_x
LayerNorm layernorm_44 1 1 att_22_x att_22_xx 0=1024 1=0.00001 2=1
Split split_att_22_xx 1 3 att_22_xx att_22_xx_0 att_22_xx_1 state_66_out
BinaryOp sub_44 2 1 state_66_in att_22_xx_0 att_22_sx 0=1
MemoryData data_att_22_maa 0 1 att_22_maa 0=1024 1=6 21=0
BinaryOp mul_285 2 1 att_22_sx att_22_maa att_22_maa_x 0=2
BinaryOp add_284 2 1 att_22_xx_1 att_22_maa_x att_22_maa_rwkvag 0=0
Slice slice_att_22_maa_rwkvag 1 6 att_22_maa_rwkvag att_22_mr att_22_mw att_22_mk att_22_mv att_22_ma att_22_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_22_mv 1 2 att_22_mv att_22_mv_0 att_22_mv_1
Gemm gemm_att_22_key 1 1 att_22_mk att_22_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_value 1 1 att_22_mv_0 att_22_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_receptance 1 1 att_22_mr att_22_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_ma_lora_0 1 1 att_22_ma att_22_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_22_ma_lora_1 1 1 att_22_ma_lora_0 att_22_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_22_a0 0 1 att_22_a0 0=1024 21=0
BinaryOp add_285 2 1 att_22_ma_lora_1 att_22_a0 att_22_ma_lora_2 0=0
Sigmoid sigmoid_87 1 1 att_22_ma_lora_2 att_22_a
Split split_att_22_a 1 2 att_22_a att_22_a_0 att_22_a_1
Gemm gemm_att_22_mg_lora 1 1 att_22_mg att_22_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_88 1 1 att_22_mg_lora att_22_mg_lora_sigmoid
Gemm gemm_att_22_gate 1 1 att_22_mg_lora_sigmoid att_22_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_22_k_k 0 1 att_22_k_k 0=1024 21=0
MemoryData data_att_22_k_a 0 1 att_22_k_a 0=1024 21=0
MemoryData data_att_22_r_k 0 1 att_22_r_k 0=1024 21=0
Split split_att_22_key 1 2 att_22_key att_22_key_0 att_22_key_1
BinaryOp mul_286 2 1 att_22_key_0 att_22_k_k att_22_key_k 0=2
Reshape reshape_264 1 1 att_22_key_k att_22_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_22 1 1 att_22_key_k_reshape att_22_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_265 1 1 att_22_key_k_norm att_22_key_k_norm_reshape 0=1024
BinaryOp add_286 1 1 att_22_a_0 att_22_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_287 2 1 att_22_a_0_minus att_22_k_a att_22_a_minus1_mulka 0=2
BinaryOp add_287 1 1 att_22_a_minus1_mulka att_22_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_288 2 1 att_22_a_minus1_mulka_plus1 att_22_key_1 att_22_key_a 0=2
Split split_att_22_value 1 2 att_22_value att_22_value_0 att_22_value_1
Split split_v_first_21 1 2 v_first_21 att_22_v_first v_first_22
BinaryOp minus_21 2 1 att_22_v_first att_22_value_1 att_22_vfirst_minus_value 0=1
Gemm gemm_att_22_mv_lora_0 1 1 att_22_mv_1 att_22_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_22_mv_lora_1 1 1 att_22_mv_lora_0 att_22_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_22_v0 0 1 att_22_v0 0=1024 21=0
BinaryOp add_288 2 1 att_22_mv_lora_1 att_22_v0 att_22_mv_lora_2 0=0
Sigmoid sigmoid_89 1 1 att_22_mv_lora_2 att_22_mv_lora_sigmoid
BinaryOp mul_289 2 1 att_22_mv_lora_sigmoid att_22_vfirst_minus_value att_22_vfirst_minus_value_mul 0=2
BinaryOp add_289 2 1 att_22_value_0 att_22_vfirst_minus_value_mul att_22_value_final 0=0
Gemm gemm_att_22_mw_lora 1 1 att_22_mw att_22_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_22_mw_lora 1 1 att_22_mw_lora att_22_mw_lora_tanh
Gemm gemm_att_22_mw_lora_tanh_linear 1 1 att_22_mw_lora_tanh att_22_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_22_td 0 1 att_22_td 0=1024 21=0
BinaryOp add_290 2 1 att_22_mw_lora_tanh_linear att_22_td att_22_time_decay_pre 0=0
Sigmoid sigmoid_90 1 1 att_22_time_decay_pre att_22_time_decay_sigmoid
Exp exp_22 1 1 att_22_time_decay_sigmoid att_22_time_decay 1=-0.606531
Reshape reshape_266 1 1 att_22_time_decay att_22_time_decay_reshape 0=64 1=1 2=16
Split split_att_22_key_a 1 2 att_22_key_a att_22_key_a_0 att_22_key_a_1
Reshape reshape_267 1 1 att_22_key_a_0 att_22_key_a_reshape 0=64 1=1 2=16
Split split_att_22_value_final 1 3 att_22_value_final att_22_value_final_0 att_22_value_final_1 att_22_value_final_2
Reshape reshape_268 1 1 att_22_value_final_0 att_22_value_final_0_reshape 0=64 1=16
Reshape reshape_269 1 1 att_22_value_final_1 att_22_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_22_vk 2 1 att_22_value_final_1_reshape att_22_key_a_reshape att_22_vk 0=0
Split split_att_22_key_k_norm_reshape 1 2 att_22_key_k_norm_reshape att_22_kk_0 att_22_kk_1
BinaryOp mul_290 1 1 att_22_kk_0 att_22_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_270 1 1 att_22_kk_0_neg att_22_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_291 2 1 att_22_kk_1 att_22_a_1 att_22_kk_a 0=2
Reshape reshape_271 1 1 att_22_kk_a att_22_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_22_ab 2 1 att_22_kk_0_neg_reshape att_22_kk_a_reshape att_22_ab 0=0
Split split_state_67_in 1 2 state_67_in att_22_state_prev_0 att_22_state_prev_1
BinaryOp mul_292 2 1 att_22_state_prev_0 att_22_time_decay_reshape att_22_state_td 0=2
MatMul matmul_att_22_sab 2 1 att_22_state_prev_1 att_22_ab att_22_sab 0=0
BinaryOp add_291 2 1 att_22_sab att_22_vk att_22_sab_vk 0=0
BinaryOp add_292 2 1 att_22_sab_vk att_22_state_td att_22_state_new 0=0
Split split_att_22_state_new 1 2 att_22_state_new att_22_state_new_0 state_67_out
Split split_att_22_receptance 1 2 att_22_receptance att_22_receptance_0 att_22_receptance_1
Reshape reshape_272 1 1 att_22_receptance_0 att_22_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_22_wkv_out 2 1 att_22_state_new_0 att_22_receptance_reshape att_22_wkv_out 0=0
Reshape reshape_273 1 1 att_22_wkv_out att_22_wkv_out_flatten 0=1024
GroupNorm groupnorm_22 1 1 att_22_wkv_out_flatten att_22_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_293 2 1 att_22_receptance_1 att_22_key_a_1 att_22_receptance_key_a 0=2
BinaryOp mul_294 2 1 att_22_receptance_key_a att_22_r_k att_22_rk 0=2
Reshape reshape_274 1 1 att_22_rk att_22_rk_reshape 0=64 1=16
Reduction sum_22 1 1 att_22_rk_reshape att_22_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_295 2 1 att_22_value_final_0_reshape att_22_rk_sum att_22_value_final_0_rk 0=2
Reshape reshape_275 1 1 att_22_value_final_0_rk att_22_value_final_0_rk_reshape 0=1024
BinaryOp add_293 2 1 att_22_value_final_0_rk_reshape att_22_x_gn att_22_x_gn_rkv 0=0
BinaryOp mul_296 2 1 att_22_x_gn_rkv att_22_gate att_22_x_gate 0=2
Gemm gemm_att_22_x_out 1 1 att_22_x_gate att_22_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_294 2 1 att_22_x_out att_22_x_last time_mixing_22_out 0=0
Split split_time_mixing_22_out 1 2 time_mixing_22_out ffn_22_x_last ffn_22_x
LayerNorm layernorm_45 1 1 ffn_22_x ffn_22_xx 0=1024 1=0.00001 2=1
Split split_ffn_22_xx 1 3 ffn_22_xx ffn_22_xx_0 ffn_22_xx_1 state_68_out
BinaryOp sub_45 2 1 state_68_in ffn_22_xx_0 ffn_22_sx 0=1
Split split_ffn_22_sx 1 1 ffn_22_sx ffn_22_sx_0
MemoryData data_ffn_22_x_k 0 1 ffn_22_x_k 0=1024 21=0
BinaryOp mul_297 2 1 ffn_22_sx_0 ffn_22_x_k ffn_22_xk 0=2
BinaryOp add_295 2 1 ffn_22_xk ffn_22_xx_1 ffn_22_xxk 0=0
Gemm gemm_ffn_22_key 1 1 ffn_22_xxk ffn_22_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_22 1 1 ffn_22_key ffn_22_key_relu
UnaryOp square_ffn_22_key_relu 1 1 ffn_22_key_relu ffn_22_key_relu_square 0=4
Gemm gemm_ffn_22_value 1 1 ffn_22_key_relu_square ffn_22_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_296 2 1 ffn_22_value ffn_22_x_last channel_mixing_22_out 0=0
Split split_channel_mixing_22_out 1 2 channel_mixing_22_out att_23_x_last att_23_x
LayerNorm layernorm_46 1 1 att_23_x att_23_xx 0=1024 1=0.00001 2=1
Split split_att_23_xx 1 3 att_23_xx att_23_xx_0 att_23_xx_1 state_69_out
BinaryOp sub_46 2 1 state_69_in att_23_xx_0 att_23_sx 0=1
MemoryData data_att_23_maa 0 1 att_23_maa 0=1024 1=6 21=0
BinaryOp mul_298 2 1 att_23_sx att_23_maa att_23_maa_x 0=2
BinaryOp add_297 2 1 att_23_xx_1 att_23_maa_x att_23_maa_rwkvag 0=0
Slice slice_att_23_maa_rwkvag 1 6 att_23_maa_rwkvag att_23_mr att_23_mw att_23_mk att_23_mv att_23_ma att_23_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_23_mv 1 2 att_23_mv att_23_mv_0 att_23_mv_1
Gemm gemm_att_23_key 1 1 att_23_mk att_23_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_value 1 1 att_23_mv_0 att_23_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_receptance 1 1 att_23_mr att_23_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_ma_lora_0 1 1 att_23_ma att_23_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_23_ma_lora_1 1 1 att_23_ma_lora_0 att_23_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_23_a0 0 1 att_23_a0 0=1024 21=0
BinaryOp add_298 2 1 att_23_ma_lora_1 att_23_a0 att_23_ma_lora_2 0=0
Sigmoid sigmoid_91 1 1 att_23_ma_lora_2 att_23_a
Split split_att_23_a 1 2 att_23_a att_23_a_0 att_23_a_1
Gemm gemm_att_23_mg_lora 1 1 att_23_mg att_23_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_92 1 1 att_23_mg_lora att_23_mg_lora_sigmoid
Gemm gemm_att_23_gate 1 1 att_23_mg_lora_sigmoid att_23_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_23_k_k 0 1 att_23_k_k 0=1024 21=0
MemoryData data_att_23_k_a 0 1 att_23_k_a 0=1024 21=0
MemoryData data_att_23_r_k 0 1 att_23_r_k 0=1024 21=0
Split split_att_23_key 1 2 att_23_key att_23_key_0 att_23_key_1
BinaryOp mul_299 2 1 att_23_key_0 att_23_k_k att_23_key_k 0=2
Reshape reshape_276 1 1 att_23_key_k att_23_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_23 1 1 att_23_key_k_reshape att_23_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_277 1 1 att_23_key_k_norm att_23_key_k_norm_reshape 0=1024
BinaryOp add_299 1 1 att_23_a_0 att_23_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_300 2 1 att_23_a_0_minus att_23_k_a att_23_a_minus1_mulka 0=2
BinaryOp add_300 1 1 att_23_a_minus1_mulka att_23_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_301 2 1 att_23_a_minus1_mulka_plus1 att_23_key_1 att_23_key_a 0=2
Split split_att_23_value 1 2 att_23_value att_23_value_0 att_23_value_1
Split split_v_first_22 1 2 v_first_22 att_23_v_first v_first_23
BinaryOp minus_22 2 1 att_23_v_first att_23_value_1 att_23_vfirst_minus_value 0=1
Gemm gemm_att_23_mv_lora_0 1 1 att_23_mv_1 att_23_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_23_mv_lora_1 1 1 att_23_mv_lora_0 att_23_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_23_v0 0 1 att_23_v0 0=1024 21=0
BinaryOp add_301 2 1 att_23_mv_lora_1 att_23_v0 att_23_mv_lora_2 0=0
Sigmoid sigmoid_93 1 1 att_23_mv_lora_2 att_23_mv_lora_sigmoid
BinaryOp mul_302 2 1 att_23_mv_lora_sigmoid att_23_vfirst_minus_value att_23_vfirst_minus_value_mul 0=2
BinaryOp add_302 2 1 att_23_value_0 att_23_vfirst_minus_value_mul att_23_value_final 0=0
Gemm gemm_att_23_mw_lora 1 1 att_23_mw att_23_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_23_mw_lora 1 1 att_23_mw_lora att_23_mw_lora_tanh
Gemm gemm_att_23_mw_lora_tanh_linear 1 1 att_23_mw_lora_tanh att_23_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_23_td 0 1 att_23_td 0=1024 21=0
BinaryOp add_303 2 1 att_23_mw_lora_tanh_linear att_23_td att_23_time_decay_pre 0=0
Sigmoid sigmoid_94 1 1 att_23_time_decay_pre att_23_time_decay_sigmoid
Exp exp_23 1 1 att_23_time_decay_sigmoid att_23_time_decay 1=-0.606531
Reshape reshape_278 1 1 att_23_time_decay att_23_time_decay_reshape 0=64 1=1 2=16
Split split_att_23_key_a 1 2 att_23_key_a att_23_key_a_0 att_23_key_a_1
Reshape reshape_279 1 1 att_23_key_a_0 att_23_key_a_reshape 0=64 1=1 2=16
Split split_att_23_value_final 1 3 att_23_value_final att_23_value_final_0 att_23_value_final_1 att_23_value_final_2
Reshape reshape_280 1 1 att_23_value_final_0 att_23_value_final_0_reshape 0=64 1=16
Reshape reshape_281 1 1 att_23_value_final_1 att_23_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_23_vk 2 1 att_23_value_final_1_reshape att_23_key_a_reshape att_23_vk 0=0
Split split_att_23_key_k_norm_reshape 1 2 att_23_key_k_norm_reshape att_23_kk_0 att_23_kk_1
BinaryOp mul_303 1 1 att_23_kk_0 att_23_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_282 1 1 att_23_kk_0_neg att_23_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_304 2 1 att_23_kk_1 att_23_a_1 att_23_kk_a 0=2
Reshape reshape_283 1 1 att_23_kk_a att_23_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_23_ab 2 1 att_23_kk_0_neg_reshape att_23_kk_a_reshape att_23_ab 0=0
Split split_state_70_in 1 2 state_70_in att_23_state_prev_0 att_23_state_prev_1
BinaryOp mul_305 2 1 att_23_state_prev_0 att_23_time_decay_reshape att_23_state_td 0=2
MatMul matmul_att_23_sab 2 1 att_23_state_prev_1 att_23_ab att_23_sab 0=0
BinaryOp add_304 2 1 att_23_sab att_23_vk att_23_sab_vk 0=0
BinaryOp add_305 2 1 att_23_sab_vk att_23_state_td att_23_state_new 0=0
Split split_att_23_state_new 1 2 att_23_state_new att_23_state_new_0 state_70_out
Split split_att_23_receptance 1 2 att_23_receptance att_23_receptance_0 att_23_receptance_1
Reshape reshape_284 1 1 att_23_receptance_0 att_23_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_23_wkv_out 2 1 att_23_state_new_0 att_23_receptance_reshape att_23_wkv_out 0=0
Reshape reshape_285 1 1 att_23_wkv_out att_23_wkv_out_flatten 0=1024
GroupNorm groupnorm_23 1 1 att_23_wkv_out_flatten att_23_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_306 2 1 att_23_receptance_1 att_23_key_a_1 att_23_receptance_key_a 0=2
BinaryOp mul_307 2 1 att_23_receptance_key_a att_23_r_k att_23_rk 0=2
Reshape reshape_286 1 1 att_23_rk att_23_rk_reshape 0=64 1=16
Reduction sum_23 1 1 att_23_rk_reshape att_23_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_308 2 1 att_23_value_final_0_reshape att_23_rk_sum att_23_value_final_0_rk 0=2
Reshape reshape_287 1 1 att_23_value_final_0_rk att_23_value_final_0_rk_reshape 0=1024
BinaryOp add_306 2 1 att_23_value_final_0_rk_reshape att_23_x_gn att_23_x_gn_rkv 0=0
BinaryOp mul_309 2 1 att_23_x_gn_rkv att_23_gate att_23_x_gate 0=2
Gemm gemm_att_23_x_out 1 1 att_23_x_gate att_23_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_307 2 1 att_23_x_out att_23_x_last time_mixing_23_out 0=0
Split split_time_mixing_23_out 1 2 time_mixing_23_out ffn_23_x_last ffn_23_x
LayerNorm layernorm_47 1 1 ffn_23_x ffn_23_xx 0=1024 1=0.00001 2=1
Split split_ffn_23_xx 1 3 ffn_23_xx ffn_23_xx_0 ffn_23_xx_1 state_71_out
BinaryOp sub_47 2 1 state_71_in ffn_23_xx_0 ffn_23_sx 0=1
Split split_ffn_23_sx 1 1 ffn_23_sx ffn_23_sx_0
MemoryData data_ffn_23_x_k 0 1 ffn_23_x_k 0=1024 21=0
BinaryOp mul_310 2 1 ffn_23_sx_0 ffn_23_x_k ffn_23_xk 0=2
BinaryOp add_308 2 1 ffn_23_xk ffn_23_xx_1 ffn_23_xxk 0=0
Gemm gemm_ffn_23_key 1 1 ffn_23_xxk ffn_23_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_23 1 1 ffn_23_key ffn_23_key_relu
UnaryOp square_ffn_23_key_relu 1 1 ffn_23_key_relu ffn_23_key_relu_square 0=4
Gemm gemm_ffn_23_value 1 1 ffn_23_key_relu_square ffn_23_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_309 2 1 ffn_23_value ffn_23_x_last channel_mixing_23_out 0=0
LayerNorm layernorm_48 1 1 channel_mixing_23_out norm_head 0=1024 1=0.00001 2=1
Gemm gemm_logits 1 1 norm_head logits 4=0 5=1 6=0 7=0 8=65536 9=1024 10=-1