| {"max_batch_size": 8, "max_beam_width": 1, "max_input_len": 14, "max_output_len": 448, "world_size": 1, "dtype": "float16", "quantize_dir": "quantize/1-gpu", "use_gpt_attention_plugin": "float16", "use_bert_attention_plugin": "float16", "use_context_fmha_enc": false, "use_context_fmha_dec": false, "use_gemm_plugin": "float16", "use_layernorm_plugin": false, "remove_input_padding": false, "use_weight_only_enc": false, "use_weight_only_dec": false, "weight_only_precision": "int8", "int8_kv_cache": false, "debug_mode": false, "cuda_compute_capability": [8, 9], "cuda_device_name": "NVIDIA GeForce RTX 4090", "model_path": "models/large-v3/pt_ckpt.pt", "output_dir": "models/e19300457c5c10b1afb8832c3b56157f"} |