File size: 5,032 Bytes
751936e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 |
# coding=utf-8
# author: xusong
# time: 2022/8/23 16:06
"""
plots
table
## related demo
http://text-processing.com/demo/tokenize/
## 可视化
[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
"""
import json
import pandas as pd
import gradio as gr
from vocab import all_tokenizers, load_tokener
# 显示空格：https://blog.csdn.net/liuxiao723846/article/details/118994673
# 隐藏legend：
css = """
.space-show {white-space: pre-wrap;}
.cell-wrap {white-space: pre-wrap;}
.category-legend {display: none !important}
"""
example_text = """中文测试：华为智能音箱发布：华为Sound X。維基百科由非營利組織──維基媒體基金會負責維持
标点测试：，。！？；
空格测试： 2个空格 8个空格
数字测试：(10086 + 98) = 100184"""
def tokenize(text, tokenizer_type):
print(text, tokenizer_type)
pos_tokens = []
tokenizer = load_tokener(tokenizer_type)
encoding = tokenizer.encode(text)
table = []
for idx, token_id in enumerate(encoding):
decode_text = tokenizer.decode([token_id]) # 特殊字符解码后会统一变成 �，对应 "\ufffd"
pos_tokens.extend([(decode_text, str(idx % 3))])
# token "Byte": # 这是 utf-8编码吧？
token = tokenizer.convert_ids_to_tokens([token_id])[0]
if isinstance(token, bytes):
token_str = token.decode("utf-8")
token_bytes = token
json_dumps = json.dumps(token_str)
elif isinstance(token, str):
token_str = token
token_bytes = bytes(token_str, "utf-8")
json_dumps = json.dumps(token_str)
else:
return
table.append(
{"TokenID": token_id,
"Token": token_str, # utf-8解码后的字符串，为什么有些是 <0xE7>，表示什么？比如llama
"Text": decode_text, #
# "Bytes": token_bytes, # bytes类型在gradio前端页面被解码成字符串，比如 b'\xe4\xb8\xad' 仍然显示成 "中"。因此 str(token_bytes)
"Bytes": str(token_bytes),
# "Unicode": json_dumps # unicode, 如果是ascii码，就直接显示。如果不是ascii码，就显示unicode
}
)
table_df = pd.DataFrame(table)
print(table)
print(table_df)
return pos_tokens, table_df
def test_coding():
bytes1 = b'\xe4\xb8\xad'
print(bytes1) # b'\xe4\xb8\xad'
with gr.Blocks(css=css) as demo:
gr.HTML("""<h1 align="center">Tokenizer Arena</h1>""")
# links: https://www.coderstool.com/utf8-encoding-decoding
#
user_input = gr.Textbox(
value=example_text,
lines=5
) # placeholder="Enter sentence here..."
# submitBtn = gr.Button("生成回复", variant="primary")
# TODO: 图 表 压缩率
# llama chatglm gpt_nexo_20b baichuan baichuan_7b
with gr.Row():
with gr.Column():
tokenizer_type_1 = gr.Dropdown(
all_tokenizers, value="llama", label="tokenizer"
)
token_counter_1 = None # 计数器
output_text_1 = gr.Highlightedtext(
label="Tokenization",
show_legend=True,
elem_classes="space-show"
)
output_table_1 = gr.Dataframe(
headers=["TokenID", "Byte", "Text"],
datatype=["str", "str", "str"],
#elem_classes="space-show", # 给整个Dataframe加这个css不起作用，因此直接修改cell-wrap
)
with gr.Column():
tokenizer_type_2 = gr.Dropdown(
all_tokenizers, value="baichuan_7b", label="tokenizer"
)
token_counter_2 = None # 计数器
output_text_2 = gr.Highlightedtext(
label="Tokenization",
show_legend=True,
elem_classes="space-show"
)
output_table_2 = gr.Dataframe(
headers=["TokenID", "Token", "Text"],
datatype=["str", "str", "str"],
)
user_input.change(tokenize,
[user_input, tokenizer_type_1],
[output_text_1, output_table_1])
tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1])
user_input.change(tokenize,
[user_input, tokenizer_type_2],
[output_text_2, output_table_2])
tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2])
# submitBtn.click(tokenize, [user_input, tokenizer_type], outputs,
# show_progress=True)
# examples=[
# ["What a beautiful morning for a walk!"],
# ["It was the best of times, it was the worst of times."],
# ["多个空格 It ss was the best of times, it was the worst of times."],
# ]
if __name__ == "__main__":
demo.launch()
|