Upload processor

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

processing_meralion.py

@@ -29,7 +29,7 @@ class MERaLiONProcessor(ProcessorMixin):
 
     attributes = ["feature_extractor", "tokenizer"]
     feature_extractor_class = "WhisperFeatureExtractor"
-    tokenizer_class = "GemmaTokenizer"
+    tokenizer_class = "AutoTokenizer"
     valid_kwargs = [
         "fixed_speech_embeds_length", 
         "speech_token_index", 

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
+size 34362873

tokenizer_config.json

@@ -1987,7 +1987,7 @@
       "special": false
     },
     "255999": {
-      "content": "<SpeechHere>",
+      "content": "<unused99>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,