Hi,
I am trying to quantize a BERT but the size is not reducing, I wonder why, Can someone help me?
Here is the code snippet:-
import torch as torch
import os
from transformers.modeling_bert import BertConfig, BertForPreTraining, load_tf_weights_in_bert, BertModel
tf_checkpoint_path="./distlang/"
bert_config_file = “./config.json”
pytorch_dump_path="./distlangpytorch/"
device = “cpu”
torch.backends.quantized.engine = ‘qnnpack’
qconfig = torch.quantization.get_default_qconfig(‘fbgemm’)
print(qconfig)
config = BertConfig.from_json_file(bert_config_file)
print(“Building PyTorch model from configuration: {}”.format(str(config)))
model = BertModel.from_pretrained("./distlangpytorch/")
model.to(device)
torch.quantization.prepare(model)
quantized_model=torch.quantization.convert(model)
def print_size_of_model(model):
torch.save(model.state_dict(), “temp.p”)
print(‘Size (MB):’, os.path.getsize(“temp.p”)/1e6)
os.remove(‘temp.p’)
print_size_of_model(model)
print_size_of_model(quantized_model)
quantized_output_dir = “./quantized_model”
if not os.path.exists(quantized_output_dir):
os.makedirs(quantized_output_dir)