app.py

# -*- coding: utf-8 -*-

import operator
import copy
import re

from transformers import BertTokenizer, BertForMaskedLM
import gradio as gr
import opencc
import torch


pretrained_model_name_or_path = "Macropodus/macbert4mdcspell_v2"
tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path)
model = BertForMaskedLM.from_pretrained(pretrained_model_name_or_path)
vocab = tokenizer.vocab


# from modelscope import AutoTokenizer, AutoModelForMaskedLM
# pretrained_model_name_or_path = "Macadam/macbert4mdcspell_v2"
# tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path)
# model = AutoModelForMaskedLM.from_pretrained(pretrained_model_name_or_path)
# vocab = tokenizer.vocab
converter_t2s = opencc.OpenCC("t2s.json")
context = converter_t2s.convert("汉字")  # 漢字
PUN_EN2ZH_DICT = {",": "，", ";": "；", "!": "！", "?": "？", ":": "：", "(": "（", ")": "）", "_": "—"}
PUN_BERT_DICT = {"“":'"', "”":'"', "‘":'"', "’":'"', "—": "_", "——": "__"}


def func_macro_correct(text):
    with torch.no_grad():
        outputs = model(**tokenizer([text], padding=True, return_tensors='pt'))

    def flag_total_chinese(text):
        """
        judge is total chinese or not, 判断是不是全是中文
        Args:
            text: str, eg. "macadam, 碎石路"
        Returns:
            bool, True or False
        """
        for word in text:
            if not "\u4e00" <= word <= "\u9fa5":
                return False
        return True

    def get_errors_from_diff_length(corrected_text, origin_text, unk_tokens=[], know_tokens=[]):
        """Get errors between corrected text and origin text
        code from:  https://github.com/shibing624/pycorrector
        """
        new_corrected_text = ""
        errors = []
        i, j = 0, 0
        unk_tokens = unk_tokens or [' ', '“', '”', '‘', '’', '琊', '\n', '…', '擤', '\t', '玕', '']
        while i < len(origin_text) and j < len(corrected_text):
            if origin_text[i] in unk_tokens or origin_text[i] not in know_tokens:
                new_corrected_text += origin_text[i]
                i += 1
            elif corrected_text[j] in unk_tokens:
                new_corrected_text += corrected_text[j]
                j += 1
            # Deal with Chinese characters
            elif flag_total_chinese(origin_text[i]) and flag_total_chinese(corrected_text[j]):
                # If the two characters are the same, then the two pointers move forward together
                if origin_text[i] == corrected_text[j]:
                    new_corrected_text += corrected_text[j]
                    i += 1
                    j += 1
                else:
                    # Check for insertion errors
                    if j + 1 < len(corrected_text) and origin_text[i] == corrected_text[j + 1]:
                        errors.append(('', corrected_text[j], j))
                        new_corrected_text += corrected_text[j]
                        j += 1
                    # Check for deletion errors
                    elif i + 1 < len(origin_text) and origin_text[i + 1] == corrected_text[j]:
                        errors.append((origin_text[i], '', i))
                        i += 1
                    # Check for replacement errors
                    else:
                        errors.append((origin_text[i], corrected_text[j], i))
                        new_corrected_text += corrected_text[j]
                        i += 1
                        j += 1
            else:
                new_corrected_text += origin_text[i]
                if origin_text[i] == corrected_text[j]:
                    j += 1
                i += 1
        errors = sorted(errors, key=operator.itemgetter(2))
        return new_corrected_text, errors

    def get_errors_from_same_length(corrected_text, origin_text, unk_tokens=[], know_tokens=[]):
        """Get new corrected text and errors between corrected text and origin text
        code from:  https://github.com/shibing624/pycorrector
        """
        errors = []
        unk_tokens = unk_tokens or [' ', '“', '”', '‘', '’', '琊', '\n', '…', '擤', '\t', '玕', '', '，']

        for i, ori_char in enumerate(origin_text):
            if i >= len(corrected_text):
                continue
            if ori_char in unk_tokens or ori_char not in know_tokens:
                # deal with unk word
                corrected_text = corrected_text[:i] + ori_char + corrected_text[i + 1:]
                continue
            if ori_char != corrected_text[i]:
                if not flag_total_chinese(ori_char):
                    # pass not chinese char
                    corrected_text = corrected_text[:i] + ori_char + corrected_text[i + 1:]
                    continue
                if not flag_total_chinese(corrected_text[i]):
                    corrected_text = corrected_text[:i] + corrected_text[i + 1:]
                    continue
                errors.append([ori_char, corrected_text[i], i])
        errors = sorted(errors, key=operator.itemgetter(2))
        return corrected_text, errors

    _text = tokenizer.decode(torch.argmax(outputs.logits[0], dim=-1), skip_special_tokens=True).replace(' ', '')
    corrected_text = _text[:len(text)]
    print("#" * 128)
    print(text)
    print(corrected_text)
    print(len(text), len(corrected_text))
    if len(corrected_text) == len(text):
        corrected_text, details = get_errors_from_same_length(corrected_text, text, know_tokens=vocab)
    else:
        corrected_text, details = get_errors_from_diff_length(corrected_text, text, know_tokens=vocab)
    print(text, ' => ', corrected_text, details)
    # return corrected_text + ' ' + str(details)
    line_dict = {"source": text, "target": corrected_text, "errors": details}
    return line_dict


def transfor_english_symbol_to_chinese(text, kv_dict=PUN_EN2ZH_DICT):
    """   将英文标点符号转化为中文标点符号, 位数不能变防止pos_id变化   """
    for k, v in kv_dict.items():  # 英文替换
        text = text.replace(k, v)
    if text and text[-1] == ".":  # 最后一个字符是英文.
        text = text[:-1] + "。"

    if text and "\"" in text:  # 双引号
        index_list = [i.start() for i in re.finditer("\"", text)]
        if index_list:
            for idx, index in enumerate(index_list):
                symbol = "“" if idx % 2 == 0 else "”"
                text = text[:index] + symbol + text[index + 1:]

    if text and "'" in text:  # 单引号
        index_list = [i.start() for i in re.finditer("'", text)]
        if index_list:
            for idx, index in enumerate(index_list):
                symbol = "‘" if idx % 2 == 0 else "’"
                text = text[:index] + symbol + text[index + 1:]
    return text
def cut_sent_by_stay(text, return_length=True, add_semicolon=False):
    """  分句但是保存原标点符号  """
    if add_semicolon:
        text_sp = re.split(r"！”|？”|。”|……”|”！|”？|”。|”……|》。|）。|；|！|？|。|…|\!|\?", text)
        conn_symbol = "；！？。…”;!?》）\n"
    else:
        text_sp = re.split(r"！”|？”|。”|……”|”！|”？|”。|”……|》。|）。|！|？|。|…|\!|\?", text)
        conn_symbol = "！？。…”!?》）\n"
    text_length_s = []
    text_cut = []
    len_text = len(text) - 1
    # signal_symbol = "—”>；？…）‘《’（·》“~，、！。：<"
    len_global = 0
    for idx, text_sp_i in enumerate(text_sp):
        text_cut_idx = text_sp[idx]
        len_global_before = copy.deepcopy(len_global)
        len_global += len(text_sp_i)
        while True:
            if len_global <= len_text and text[len_global] in conn_symbol:
                text_cut_idx += text[len_global]
            else:
                # len_global += 1
                if text_cut_idx:
                    text_length_s.append([len_global_before, len_global])
                    text_cut.append(text_cut_idx)
                break
            len_global += 1
    if return_length:
        return text_cut, text_length_s
    return text_cut
def transfor_bert_unk_pun_to_know(text, kv_dict=PUN_BERT_DICT):
    """   将英文标点符号转化为中文标点符号, 位数不能变防止pos_id变化   """
    for k, v in kv_dict.items():  # 英文替换
        text = text.replace(k, v)
    return text
def tradition_to_simple(text):
    """  繁体到简体  """
    return converter_t2s.convert(text)
def string_q2b(ustring):
    """把字符串全角转半角"""
    return "".join([q2b(uchar) for uchar in ustring])
def q2b(uchar):
    """全角转半角"""
    inside_code = ord(uchar)
    if inside_code == 0x3000:
        inside_code = 0x0020
    else:
        inside_code -= 0xfee0
    if inside_code < 0x0020 or inside_code > 0x7e:  # 转完之后不是半角字符返回原来的字符
        return uchar
    return chr(inside_code)


def func_macro_correct_long(text):
    """   长句   """
    texts, length = cut_sent_by_stay(text, return_length=True, add_semicolon=True)
    text_correct = ""
    errors_new = []
    for idx, text in enumerate(texts):
        # 前处理
        text = transfor_english_symbol_to_chinese(text)
        text = string_q2b(text)
        text = tradition_to_simple(text)
        text = transfor_bert_unk_pun_to_know(text)

        text_out = func_macro_correct(text)
        source = text_out.get("source")
        target = text_out.get("target")
        errors = text_out.get("errors")
        text_correct += target
        for error in errors:
            pos = length[idx][0] + error[-1]
            error_1 = [error[0], error[1], pos]
            errors_new.append(error_1)
    return text_correct + '\n' + str(errors_new)


if __name__ == '__main__':
    text = """网购的烦脑
emer 发布于 2025-7-3 18:20 阅读：73

最近网购遇到件恼火的事。我在网店看中件羽戎服，店家保正是正品，还承诺七天无里由退换。收到货后却发现袖口有开线，更糟的是拉链老是卡住。

联系客服时，对方态度敷衔，先说让我自行缝补，后又说要扣除运废才给退。我在评沦区如实描述经历，结果发现好多消废者都有类似遭遇。

这次购物让我明白，不能光看店家的宣全，要多查考真实评价。现在我已经学精了，下单前总会反复合对商品信息。
网购的烦恼发布于2025-7-310期阅读：最近网购遇到件恼火的事。我在网店看中件羽绒服，店家保证是正品，还承诺七天无理由退换。收到货后却发现袖口有开线，更糟的是拉链老是卡住。联系客服时，对方态度敷衍，先说让我自行缝补，后又说要扣除运废才给退。我在评论区如实描述经历，结果发现好多消废者都有类似遭遇。这次购物让我明白，不能光看店家的宣全，要多查考真实评价。现在我已经学精了，下单前总会反复核对商品信息。
网购的烦恼e发布于2025-7-3期期阅读：最近网购遇到件恼火的事。我在网店看中件羽绒服，店家保证是正品，还承诺七天无理由退换。收到货后却发现袖口有开线，更糟的是拉链老是卡住。联系客服时，对方态度敷衍，先说让我自行缝补，后又说要扣除运废才给退。我在评论区如实描述经历，结果发现好多消废者都有类似遭遇。这次购物让我明白，不能光看店家的宣全，要多查考真实评价。现在我已经学精了，下单前总会反复核对商品信息。网购的烦恼发布于2025-7-310期阅读：最近网购遇到件恼火的事。我在网店看中件羽绒服，店家保证是正品，还承诺七天无理由退换。收到货后却发现袖口有开线，更糟的是拉链老是卡住。联系客服时，对方态度敷衍，先说让我自行缝补，后又说要扣除运废才给退。我在评论区如实描述经历，结果发现好多消废者都有类似遭遇。这次购物让我明白，不能光看店家的宣全，要多查考真实评价。现在我已经学精了，下单前总会反复核对商品信息。"""
    print(func_macro_correct_long(text))

    examples = [
        "夫谷之雨，犹复云之亦从的起，因与疾风俱飘，参于天，集于的。",
        "机七学习是人工智能领遇最能体现智能的一个分知",
        '他们的吵翻很不错，再说他们做的咖喱鸡也好吃',
        "抗疫路上，除了提心吊胆也有难的得欢笑。",
        "我是练习时长两念半的鸽仁练习生蔡徐坤",
        "清晨，如纱一般地薄雾笼罩着世界。",
        "得府许我立庙于此，故请君移去尔。",
        "他法语说的很好，的语也不错",
        "遇到一位很棒的奴生跟我疗天",
        "五年级得数学，我考的很差。",
        "我们为这个目标努力不解",
        '今天兴情很好',
    ]

    gr.Interface(
        func_macro_correct_long,
        inputs='text',
        outputs='text',
        title="Chinese Spelling Correction Model Macropodus/macbert4mdcspell_v2",
        description="Copy or input error Chinese text. Submit and the machine will correct text.",
        article="Link to <a href='https://github.com/yongzhuo/macro-correct' style='color:blue;' target='_blank\'>Github REPO: macro-correct</a>",
        examples=examples
    ).launch()