add deepseek

chat_template_app.py

@@ -0,0 +1,11 @@
+"""
+
+## chat template
+
+- special_tokens
+- default_system
+- tools
+- tool_call
+
+
+"""

client.py

@@ -0,0 +1,14 @@
+from gradio_client import Client
+
+
+def self_chat_demo(system_message, num_turn=4):
+    client = Client("xu-song/tokenizer-arena")
+    result = client.predict(
+        text="Hello!!",
+        tokenizer_name="01-ai/Yi-1.5-34B",
+        api_name="/tokenize"
+    )
+    print(result)
+
+if __name__ == "__main__":
+    self_chat_demo(system_message="你是一个小说家，擅长写武侠小说")

consistency_app.py

@@ -0,0 +1,9 @@
+"""
+On the consistency of LLM tokenizer.
+
+
+
+
+"""
+
+

setup.md

@@ -0,0 +1,5 @@
+
+
+```sh
+python  compression_util.py
+```

utils/voting_util.py

@@ -0,0 +1,15 @@
+"""
+https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard/blob/main/src/voting/vote_system.py
+
+
+## 原理
+
+https://huggingface.co/docs/huggingface_hub/guides/upload
+
+## TODO
+投票需要增加哪些 tokenizer。
+"""
+
+
+class VoteManager:
+    pass