SanskritBPETokenizer / src /__init__.py
PriyePrabhakar's picture
Added files for sanskritBPE tokenizer
232c9b6
raw
history blame contribute delete
154 Bytes
from .tokenizer import SanskritBPETokenizer
from .utils import save_merges, load_merges
__all__ = ['SanskritBPETokenizer', 'save_merges', 'load_merges']