Upload 10 files

ganti nama folder

g2pid/.gitignore

@@ -0,0 +1,164 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.DS_Store
+.backup/
+.data/

g2pid/__init__.py

@@ -0,0 +1,3 @@
+from .g2p import G2P
+
+__version__ = "0.0.5"

g2pid/g2p.py

@@ -0,0 +1,220 @@
+import json
+import os
+import re
+
+import numpy as np
+import onnxruntime
+from nltk.tokenize import TweetTokenizer
+from sacremoses import MosesDetokenizer
+
+from .syllable_splitter import SyllableSplitter
+
+ABJAD_MAPPING = {
+    "a": "a",
+    "b": "bé",
+    "c": "cé",
+    "d": "dé",
+    "e": "é",
+    "f": "èf",
+    "g": "gé",
+    "h": "ha",
+    "i": "i",
+    "j": "jé",
+    "k": "ka",
+    "l": "èl",
+    "m": "èm",
+    "n": "èn",
+    "o": "o",
+    "p": "pé",
+    "q": "ki",
+    "r": "èr",
+    "s": "ès",
+    "t": "té",
+    "u": "u",
+    "v": "vé",
+    "w": "wé",
+    "x": "èks",
+    "y": "yé",
+    "z": "zèt",
+}
+
+PHONETIC_MAPPING = {
+    "sy": "ʃ",
+    "ny": "ɲ",
+    "ng": "ŋ",
+    "dj": "dʒ",
+    "'": "ʔ",
+    "c": "tʃ",
+    "é": "e",
+    "è": "ɛ",
+    "ê": "ə",
+    "g": "ɡ",
+    "I": "ɪ",
+    "j": "dʒ",
+    "ô": "ɔ",
+    "q": "k",
+    "U": "ʊ",
+    "v": "f",
+    "x": "ks",
+    "y": "j",
+}
+
+
+dirname = os.path.dirname(__file__)
+
+# Predict pronounciation with BERT Masking
+# Read more: https://w11wo.github.io/posts/2022/04/predicting-phonemes-with-bert/
+class Predictor:
+    def __init__(self, model_path):
+        # fmt: off
+        self.vocab = ['', '[UNK]', 'a', 'n', 'ê', 'e', 'i', 'r', 'k', 's', 't', 'g', 'm', 'u', 'l', 'p', 'o', 'd', 'b', 'h', 'c', 'j', 'y', 'f', 'w', 'v', 'z', 'x', 'q', '[mask]']
+        self.mask_token_id = self.vocab.index("[mask]")
+        # fmt: on
+        self.session = onnxruntime.InferenceSession(model_path)
+
+    def predict(self, word: str) -> str:
+        """
+        Predict the phonetic representation of a word.
+
+        Args:
+            word (str): The word to predict.
+
+        Returns:
+            str: The predicted phonetic representation of the word.
+        """
+        text = [self.vocab.index(c) if c != "e" else self.mask_token_id for c in word]
+        text.extend([0] * (32 - len(text)))  # Pad to 32 tokens
+        inputs = np.array([text], dtype=np.int64)
+        (predictions,) = self.session.run(None, {"input_4": inputs})
+
+        # find masked idx token
+        _, masked_index = np.where(inputs == self.mask_token_id)
+
+        # get prediction at those masked index only
+        mask_prediction = predictions[0][masked_index]
+        predicted_ids = np.argmax(mask_prediction, axis=1)
+
+        # replace mask with predicted token
+        for i, idx in enumerate(masked_index):
+            text[idx] = predicted_ids[i]
+
+        return "".join([self.vocab[i] for i in text if i != 0])
+
+
+class G2P:
+    def __init__(self):
+        self.tokenizer = TweetTokenizer()
+        self.detokenizer = MosesDetokenizer(lang="id")
+
+        dict_path = os.path.join(dirname, "data/dict.json")
+        with open(dict_path) as f:
+            self.dict = json.load(f)
+
+        model_path = os.path.join(dirname, "model/bert_pron.onnx")
+        self.predictor = Predictor(model_path)
+
+        self.syllable_splitter = SyllableSplitter()
+
+    def __call__(self, text: str) -> str:
+        """
+        Convert text to phonetic representation.
+
+        Args:
+            text (str): The text to convert.
+
+        Returns:
+            str: The phonetic representation of the text.
+        """
+        text = text.lower()
+        text = re.sub(r"[^ a-z0-9'\.,?!-]", "", text)
+        text = text.replace("-", " ")
+
+        prons = []
+        words = self.tokenizer.tokenize(text)
+        for word in words:
+            # PUEBI pronunciation
+            if word in self.dict:
+                pron = self.dict[word]
+            elif len(word) == 1 and word in ABJAD_MAPPING:
+                pron = ABJAD_MAPPING[word]
+            elif "e" not in word or not word.isalpha():
+                pron = word
+            elif "e" in word:
+                pron = self.predictor.predict(word)
+
+            # Replace alofon /e/ with e (temporary)
+            pron = pron.replace("é", "e")
+            pron = pron.replace("è", "e")
+
+            # Replace /x/ with /s/
+            if pron.startswith("x"):
+                pron = "s" + pron[1:]
+
+            sylls = self.syllable_splitter.split_syllables(pron)
+            # Decide where to put the stress
+            stress_loc = len(sylls) - 1
+            if len(sylls) > 1 and "ê" in sylls[-2]:
+                if "ê" in sylls[-1]:
+                    stress_loc = len(sylls) - 2
+                else:
+                    stress_loc = len(sylls)
+
+            # Apply rules on syllable basis
+            # All alophone are set to tense by default
+            # and will be changed to lax if needed
+            alophone = {"e": "é", "o": "o"}
+            alophone_map = {"i": "I", "u": "U", "e": "è", "o": "ô"}
+            for i, syll in enumerate(sylls, start=1):
+                # Put Syllable stress
+                if i == stress_loc:
+                    syll = "ˈ" + syll
+
+                # Alophone syllable rules
+                for v in ["e", "o"]:
+                    # Replace with lax allphone [��, ɔ] if
+                    # in closed final syllables
+                    if v in syll and not syll.endswith(v) and i == len(sylls):
+                        alophone[v] = alophone_map[v]
+
+                # Alophone syllable stress rules
+                for v in ["i", "u"]:
+                    # Replace with lax allphone [ɪ, ʊ] if
+                    # in the middle of syllable without stress
+                    # and not ends with coda nasal [m, n, ng] (except for final syllable)
+                    if (
+                        v in syll
+                        and not syll.startswith("ˈ")
+                        and not syll.endswith(v)
+                        and (
+                            not any(syll.endswith(x) for x in ["m", "n", "ng"])
+                            or i == len(sylls)
+                        )
+                    ):
+                        syll = syll.replace(v, alophone_map[v])
+
+                if syll.endswith("nk"):
+                    syll = syll[:-2] + "ng"
+                elif syll.endswith("d"):
+                    syll = syll[:-1] + "t"
+                elif syll.endswith("b"):
+                    syll = syll[:-1] + "p"
+                elif syll.endswith("k") or (
+                    syll.endswith("g") and not syll.endswith("ng")
+                ):
+                    syll = syll[:-1] + "'"
+                sylls[i - 1] = syll
+
+            pron = "".join(sylls)
+            # Apply phonetic and alophone mapping
+            for v in alophone:
+                if v == "o" and pron.count("o") == 1:
+                    continue
+                pron = pron.replace(v, alophone[v])
+            for g, p in PHONETIC_MAPPING.items():
+                pron = pron.replace(g, p)
+            pron = pron.replace("kh", "x")
+
+            prons.append(pron)
+            prons.append(" ")
+
+        return self.detokenizer.detokenize(prons)

g2pid/model/bert_pron.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bc9b45f1cdeff4dc473f722627e94db4e3ff0ba7a2b066e542a0fa46f49d330
+size 1295867

g2pid/syllable_splitter.py

@@ -0,0 +1,127 @@
+# Copied from https://github.com/fahadh4ilyas/syllable_splitter
+# MIT License
+import re
+
+
+class SyllableSplitter:
+    def __init__(self):
+        self.consonant = set(
+            [
+                "b",
+                "c",
+                "d",
+                "f",
+                "g",
+                "h",
+                "j",
+                "k",
+                "l",
+                "m",
+                "n",
+                "p",
+                "q",
+                "r",
+                "s",
+                "t",
+                "v",
+                "w",
+                "x",
+                "y",
+                "z",
+                "ng",
+                "ny",
+                "sy",
+                "ch",
+                "dh",
+                "gh",
+                "kh",
+                "ph",
+                "sh",
+                "th",
+            ]
+        )
+        self.double_consonant = set(["ll", "ks", "rs", "rt", "nk", "nd"])
+        self.vocal = set(["a", "e", "ê", "é", "è", "i", "o", "u"])
+
+    def split_letters(self, string):
+        letters = []
+        arrange = []
+
+        while string != "":
+            letter = string[:2]
+
+            if letter in self.double_consonant:
+                if string[2:] != "" and string[2] in self.vocal:
+                    letters += [letter[0]]
+                    arrange += ["c"]
+                    string = string[1:]
+                else:
+                    letters += [letter]
+                    arrange += ["c"]
+                    string = string[2:]
+            elif letter in self.consonant:
+                letters += [letter]
+                arrange += ["c"]
+                string = string[2:]
+            elif letter in self.vocal:
+                letters += [letter]
+                arrange += ["v"]
+                string = string[2:]
+            else:
+                letter = string[0]
+
+                if letter in self.consonant:
+                    letters += [letter]
+                    arrange += ["c"]
+                    string = string[1:]
+                elif letter in self.vocal:
+                    letters += [letter]
+                    arrange += ["v"]
+                    string = string[1:]
+                else:
+                    letters += [letter]
+                    arrange += ["s"]
+                    string = string[1:]
+
+        return letters, "".join(arrange)
+
+    def split_syllables_from_letters(self, letters, arrange):
+        consonant_index = re.search(r"vc{2,}", arrange)
+        while consonant_index:
+            i = consonant_index.start() + 1
+            letters = letters[: i + 1] + ["|"] + letters[i + 1 :]
+            arrange = arrange[: i + 1] + "|" + arrange[i + 1 :]
+            consonant_index = re.search(r"vc{2,}", arrange)
+
+        vocal_index = re.search(r"v{2,}", arrange)
+        while vocal_index:
+            i = vocal_index.start()
+            letters = letters[: i + 1] + ["|"] + letters[i + 1 :]
+            arrange = arrange[: i + 1] + "|" + arrange[i + 1 :]
+            vocal_index = re.search(r"v{2,}", arrange)
+
+        vcv_index = re.search(r"vcv", arrange)
+        while vcv_index:
+            i = vcv_index.start()
+            letters = letters[: i + 1] + ["|"] + letters[i + 1 :]
+            arrange = arrange[: i + 1] + "|" + arrange[i + 1 :]
+            vcv_index = re.search(r"vcv", arrange)
+
+        sep_index = re.search(r"[cvs]s", arrange)
+        while sep_index:
+            i = sep_index.start()
+            letters = letters[: i + 1] + ["|"] + letters[i + 1 :]
+            arrange = arrange[: i + 1] + "|" + arrange[i + 1 :]
+            sep_index = re.search(r"[cvs]s", arrange)
+
+        sep_index = re.search(r"s[cvs]", arrange)
+        while sep_index:
+            i = sep_index.start()
+            letters = letters[: i + 1] + ["|"] + letters[i + 1 :]
+            arrange = arrange[: i + 1] + "|" + arrange[i + 1 :]
+            sep_index = re.search(r"s[cvs]", arrange)
+        return "".join(letters).split("|")
+
+    def split_syllables(self, string):
+        letters, arrange = self.split_letters(string)
+        return self.split_syllables_from_letters(letters, arrange)