update

tokens.txt

@@ -19,7 +19,6 @@ G 10
 h 11
 H 11
 i 12
-I 12
 j 13
 J 13
 k 14
@@ -128,7 +127,6 @@ Z 29
 ī 67
 Ī 67
 ı 68
-I 68
 ķ 69
 Ķ 69
 ļ 70

vits-coqui.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import collections
 import os
 from typing import Any, Dict
 
@@ -105,6 +106,15 @@ def main():
     add_meta_data(filename="model.onnx", meta_data=meta_data)
 
     # Now generate tokens.txt
+    all_upper_tokens = [i.upper() for i in vits.tokenizer.characters._char_to_id.keys()]
+    duplicate = set(
+        [
+            item
+            for item, count in collections.Counter(all_upper_tokens).items()
+            if count > 1
+        ]
+    )
+
     with open("tokens.txt", "w", encoding="utf-8") as f:
         for token, idx in vits.tokenizer.characters._char_to_id.items():
             f.write(f"{token} {idx}\n")
@@ -114,11 +124,9 @@ def main():
                 token not in ("<PAD>", "<EOS>", "BOS", "<BLNK>")
                 and token.lower() != token.upper()
                 and len(token.upper()) == 1
+                and token.upper() not in duplicate
             ):
-                if token == "i" and "I" in vits.tokenizer.characters._char_to_id:
-                    pass
-                else:
-                    f.write(f"{token.upper()} {idx}\n")
+                f.write(f"{token.upper()} {idx}\n")
 
 
 if __name__ == "__main__":