Mdels and code

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.vocab filter=lfs diff=lfs merge=lfs -text
+texts/*.txt filter=lfs diff=lfs merge=lfs -text
+*.arpa* filter=lfs diff=lfs merge=lfs -text
+kenlm/*.bin filter=lfs diff=lfs merge=lfs -text
+kenlm/*.arpa filter=lfs diff=lfs merge=lfs -text
+samples/*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,17 @@
+tmp/
+__pycache__/
+*.pyc
+.ipynb_checkpoints
+
+
+samples/restricted*
+samples/*.json*
+kenlm/wikipedia/*
+!kenlm/wikipedia/.keep
+kenlm/harmful/*
+!kenlm/harmful/.keep
+spm/wikipedia/*
+!spm/wikipedia/.keep
+spm/*.txt
+texts/*
+!texts/.keep

README.md

@@ -1,3 +1,39 @@
 ---
 license: apache-2.0
 ---
+
+# Perplexity tools
+
+## 1. Create samples from `clean_json_3` sources
+
+Between 1k and 1M documents. Read [samples/README.md](./samples/README.md). Output files must be prefixed by `doc_type` and suffixed by language code (2 letters). For example:
+
+```bash
+$ cat /nfsmounts/datastore/ncc_corpus/mimir/jsonl_2/nrk/nrk-articles.jsonl | shuf -n 100000 > samples/restricted-newspapers_nrk_no.json
+```
+
+## 2. Create the perplexity scores for each file
+
+Example of how to create scores only for `doc_type` `restricted-newspapers_*` samples:
+
+```bash
+$ ls samples/restricted-newspapers_* | parallel --lb --jobs 5 python samples_scores.py {} --output_path scores/ --jobs 15
+```
+
+## 3. Create the quartiles CSV needed for segmenting and downsamplig
+
+The different `doc_type`s will be grouped together. By passing the flag `--group_by_prefix_lang`, the grouping will happen on the pair `doc_type` prefix and language code, e.g., `wikipedia_en`.
+
+Different downsampling ratios can be specified by using the `--sampling_ratio_per_lang` flag. For `mimir-base`, the downsampling by language is defined as follows: `"da:0.23,en:0.21,sv:0.08,is:0.50"`.
+
+```bash
+$ python samples_quartiles.py scores/ --group_by_prefix_lang --sampling_ratio_per_lang "da:0.23,en:0.21,sv:0.08,is:0.50" --output_file csv/base-perplexity_quartiles_sampling.csv
+```
+
+For `mimir-extended`, the downsampling by language is defined as follows: `"da:0.43,en:0.81,sv:0.15,code:0.62"`.
+
+```bash
+$ python samples_quartiles.py scores/ --group_by_prefix_lang --sampling_ratio_per_lang "da:0.43,en:0.81,sv:0.15,code:0.62" --output_file csv/extended-perplexity_quartiles_sampling.csv  --overwrite_prefix_lang "starcoder_en:starcode_code"
+```
+
+More information in the [spreadsheet](https://docs.google.com/spreadsheets/d/108oGVVN-Ml-TDN59UXR96oeBBt2FbgT81zt8_1y9PUw/edit?usp=sharing).

csv/base-perplexity_quartiles_sampling.csv

@@ -0,0 +1,33 @@
+doc_type,model,language,reject,bad,medium,good,norm,mean,std
+books,books,no,542.15,301.25,219.3,165.12,0.0032422660862847633,208.18464621605895,68.02897458931068
+culturax,wikipedia,nn,1113.2,753.4,559.9,387.7,0.001172357337862289,487.27059437715525,185.90322713836343
+culturax,wikipedia,sv,1118.6,772.2,606.9,479.8,0.01968171485234145,580.0945047395821,142.99911605358275
+culturax,wikipedia,da,1012.9,648.2,503.3,397.98,0.007997295965244292,488.615463864415,124.17368632962524
+digimanus,wikipedia,no,1991.88,1226.65,989.1,830.35,0.0011146154086008851,974.7133669943673,209.08555530030617
+culturax,wikipedia,no,1073.1,691.1,538.2,430.0,0.0017538216816248486,523.6960713940705,130.62730440702228
+culturax,wikipedia,is,1420.0,884.5,720.2,594.5,0.0030935154995326906,693.7606785221377,147.6241796866134
+evalueringsrapport,maalfrid,no,268.25,163.5,127.8,98.3,0.006540788722418088,117.29318501940242,34.47568292096079
+hplt,wikipedia,nn,1539.1,980.6,772.7,627.5,0.0012826369023540814,752.0725635933572,179.13196906762977
+lovdata,maalfrid,no,457.9,162.9,84.6,41.6,0.0038894207845140477,96.06375056993284,58.30277337274196
+maalfrid,maalfrid,no,686.5,286.9,164.8,87.3,0.0022814356724527207,164.0258389923656,101.07016579025363
+hplt,wikipedia,da,1445.5,829.3,616.3,493.5,0.00597386636355673,630.7049612170936,168.77191092534918
+book,books,no,636.48,302.58,187.4,67.0,0.002034229155801576,158.1210630456195,109.45691866057511
+hplt,wikipedia,sv,1398.0,910.9,715.8,578.5,0.0173199443667263,698.8065459625257,165.03293101814995
+hplt,wikipedia,no,1589.0,880.7,668.5,532.6,0.0013206924407238364,671.3073940020074,174.52833317000255
+newspapers,newspapers,nn,1685.4,1221.9,1005.4,825.2,0.0011282397163826917,951.0683339330576,197.39448542294457
+newspaper,newspapers,no,2308.6,792.3,475.2,307.9,0.0009454270671058767,526.1389696700705,244.38110894007406
+parlamint,maalfrid,no,129.23,104.0,93.8,84.6,0.02105587174354365,89.24246433500929,10.099393230392014
+newspapers,newspapers,no,782.3,466.7,336.0,243.5,0.002096701749929567,326.7656126928853,108.47162732564873
+wikipedia,wikipedia,da,1226.31,470.7,278.7,127.0,0.006116042100206995,272.5462428872027,159.55477781562297
+wikipedia,wikipedia,is,1893.3,740.7,449.1,174.5,0.001640993616891793,429.6854374438017,283.9768832443661
+wikipedia,wikipedia,nn,1159.86,494.45,283.1,123.6,0.0013200962342698906,280.91195392289364,167.82742834163992
+wikipedia,wikipedia,no,2058.62,612.2,324.6,139.3,0.0009966961122328344,363.387061861549,229.2323512781706
+slimpajama,wikipedia,en,2259.2,756.5,534.4,418.5,0.006212514831977225,569.5492667529695,179.9279253054439
+wikipedia,wikipedia,sv,1586.56,521.5,304.0,165.4,0.016951427796527165,325.8191384990417,163.13795554088844
+wikipedia,wikipedia,en,1815.4,671.6,455.7,331.2,0.006112968939492834,470.655891042871,184.96531992400435
+hplt,wikipedia,is,2310.06,1484.7,1160.3,921.3,0.001632796440658896,1119.008609637535,278.2396677657607
+pg19,wikipedia,en,865.84,540.3,473.2,419.1,0.017132607020012576,460.9763713901977,63.76307180686858
+starcoder,wikipedia,en,6898.5,2724.5,1603.4,972.4,0.0012712203723443047,1734.1527299358695,858.6110807589087
+slimpajama,wikipedia,no,2259.2,756.5,534.4,418.5,0.006212514831977225,569.5492667529695,179.9279253054439
+starcoder,wikipedia,no,6898.5,2724.5,1603.4,972.4,0.0012712203723443047,1734.1527299358695,858.6110807589087
+pg19,wikipedia,no,865.84,540.3,473.2,419.1,0.017132607020012576,460.9763713901977,63.76307180686858

csv/extended-perplexity_quartiles_sampling.csv

@@ -0,0 +1,37 @@
+doc_type,model,language,reject,bad,medium,good,norm,mean,std
+books,books,no,542.15,301.25,219.3,165.12,0.0032422660862847633,208.18464621605895,68.02897458931068
+culturax,wikipedia,nn,1113.2,753.4,559.9,387.7,0.001172357337862289,487.27059437715525,185.90322713836343
+culturax,wikipedia,sv,1118.6,772.2,606.9,479.8,0.01049691458791544,580.0945047395821,142.99911605358275
+culturax,wikipedia,da,1012.9,648.2,503.3,397.98,0.004277623423270203,488.615463864415,124.17368632962524
+digimanus,wikipedia,no,1991.88,1226.65,989.1,830.35,0.0011146154086008851,974.7133669943673,209.08555530030617
+culturax,wikipedia,no,1073.1,691.1,538.2,430.0,0.0017538216816248486,523.6960713940705,130.62730440702228
+culturax,wikipedia,is,1420.0,884.5,720.2,594.5,0.0015467577497663453,693.7606785221377,147.6241796866134
+evalueringsrapport,maalfrid,no,268.25,163.5,127.8,98.3,0.006540788722418088,117.29318501940242,34.47568292096079
+hplt,wikipedia,nn,1539.1,980.6,772.7,627.5,0.0012826369023540814,752.0725635933572,179.13196906762977
+lovdata,maalfrid,no,457.9,162.9,84.6,41.6,0.0038894207845140477,96.06375056993284,58.30277337274196
+maalfrid,maalfrid,no,686.5,286.9,164.8,87.3,0.0022814356724527207,164.0258389923656,101.07016579025363
+hplt,wikipedia,da,1445.5,829.3,616.3,493.5,0.0031953238688791816,630.7049612170936,168.77191092534918
+book,books,no,636.48,302.58,187.4,67.0,0.002034229155801576,158.1210630456195,109.45691866057511
+hplt,wikipedia,sv,1398.0,910.9,715.8,578.5,0.009237303662254026,698.8065459625257,165.03293101814995
+hplt,wikipedia,no,1589.0,880.7,668.5,532.6,0.0013206924407238364,671.3073940020074,174.52833317000255
+newspapers,newspapers,nn,1685.4,1221.9,1005.4,825.2,0.0011282397163826917,951.0683339330576,197.39448542294457
+newspaper,newspapers,no,2308.6,792.3,475.2,307.9,0.0009454270671058767,526.1389696700705,244.38110894007406
+parlamint,maalfrid,no,129.23,104.0,93.8,84.6,0.02105587174354365,89.24246433500929,10.099393230392014
+newspapers,newspapers,no,782.3,466.7,336.0,243.5,0.002096701749929567,326.7656126928853,108.47162732564873
+wikipedia,wikipedia,da,1226.31,470.7,278.7,127.0,0.003271371355924672,272.5462428872027,159.55477781562297
+wikipedia,wikipedia,is,1893.3,740.7,449.1,174.5,0.0008204968084458965,429.6854374438017,283.9768832443661
+wikipedia,wikipedia,nn,1159.86,494.45,283.1,123.6,0.0013200962342698906,280.91195392289364,167.82742834163992
+wikipedia,wikipedia,no,2058.62,612.2,324.6,139.3,0.0009966961122328344,363.387061861549,229.2323512781706
+slimpajama,wikipedia,en,2259.2,756.5,534.4,418.5,0.0016106519934755766,569.5492667529695,179.9279253054439
+wikipedia,wikipedia,sv,1586.56,521.5,304.0,165.4,0.009040761491481156,325.8191384990417,163.13795554088844
+wikipedia,wikipedia,en,1815.4,671.6,455.7,331.2,0.0015848437991277716,470.655891042871,184.96531992400435
+hplt,wikipedia,is,2310.06,1484.7,1160.3,921.3,0.000816398220329448,1119.008609637535,278.2396677657607
+pg19,wikipedia,en,865.84,540.3,473.2,419.1,0.004441787005188445,460.9763713901977,63.76307180686858
+starcoder,wikipedia,code,6898.5,2724.5,1603.4,972.4,0.0004305746422456516,1734.1527299358695,858.6110807589087
+restricted-newspapers,newspapers,no,847.7,451.7,328.5,246.5,0.002248478883149024,325.7155732204811,102.50329419364242
+restricted-books,books,no,636.88,375.5,282.8,216.8,0.0028282201514638694,272.19155841413874,81.36986186892527
+restricted-book,books,no,569.8,365.9,281.7,218.6,0.0030429861768025046,267.8089800338991,74.79791679414626
+slimpajama,wikipedia,no,2259.2,756.5,534.4,418.5,0.0016106519934755766,569.5492667529695,179.9279253054439
+starcoder,wikipedia,no,6898.5,2724.5,1603.4,972.4,0.0004305746422456516,1734.1527299358695,858.6110807589087
+starcoder,wikipedia,code,6898.5,2724.5,1603.4,972.4,0.0004305746422456516,1734.1527299358695,858.6110807589087
+pg19,wikipedia,no,865.84,540.3,473.2,419.1,0.004441787005188445,460.9763713901977,63.76307180686858

download_all.sh

@@ -0,0 +1,40 @@
+mkdir kenlm
+mv *arpa* kenlm/
+
+mkdir spm
+mv *.model spm/
+mv *.vocab spm/
+
+mkdir kenlm/harmful
+wget -O kenlm/harmful/da.arpa https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/da.arpa
+wget -O kenlm/harmful/da.bin https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/da.binary
+wget -O kenlm/harmful/sv.arpa https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/sv.arpa
+wget -O kenlm/harmful/sv.bin https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/sv.binary
+wget -O kenlm/harmful/is.arpa https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/is.arpa
+wget -O kenlm/harmful/is.bin https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/is.binary
+wget -O kenlm/harmful/no.arpa https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/no.arpa
+wget -O kenlm/harmful/no.bin https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/no.binary
+wget -O kenlm/harmful/en.arpa https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/en.arpa
+wget -O kenlm/harmful/en.bin https://huggingface.co/oscar-corpus/harmful-kenlms/resolve/main/en.binary
+
+mkdir kenlm/wikipedia
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/da.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/da.arpa.bin
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/sv.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/sv.arpa.bin
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/is.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/is.arpa.bin
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/no.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/no.arpa.bin
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/nn.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/nn.arpa.bin
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O kenlm/wikipedia/en.arpa.bin https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/en.arpa.bin
+
+mkdir spm/wikipedia
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/da.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/da.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/sv.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/sv.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/is.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/is.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/no.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/no.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/nn.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/nn.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/en.sp.model https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/en.sp.model
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/da.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/da.sp.vocab
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/sv.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/sv.sp.vocab
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/is.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/is.sp.vocab
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/no.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/no.sp.vocab
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/nn.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/nn.sp.vocab
+wget --header="Authorization: Bearer $(cat $HOME/.cache/huggingface/token)" -O spm/wikipedia/en.sp.vocab https://huggingface.co/uonlp/kenlm/resolve/main/wikipedia_20230501/en.sp.vocab

histograms.py

@@ -0,0 +1,104 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import json
+import argparse
+import os
+from scipy.stats import gaussian_kde
+import numpy as np
+
+def get_model_for(doc_type: str, override_model: str) -> str:
+    """Returns model type or the override model if specified"""
+    if override_model:
+        return override_model
+    doc_type = doc_type.split("_", 1)[0]
+    if doc_type in ("book", "books", "pg19"):
+        return "books_pp"
+    elif doc_type in ("culturax", "slimpajama", "wikipedia", "digimanus"):
+        return "wikipedia_pp"
+    elif doc_type in ("newspaper", "newspapers"):
+        return "newspapers_pp"
+    elif doc_type in ("evalueringsrapport", "lovdata", "maalfrid", "parlamint"):
+        return "maalfrid_pp"
+    else:
+        return "wikipedia_pp"
+
+def load_data(files):
+    all_data = []
+    for file_path in files:
+        with open(file_path, 'r') as file:
+            lines = file.readlines()
+            data = [json.loads(line) for line in lines]
+            all_data.extend(data)
+    return pd.DataFrame(all_data)
+
+def plot_histograms(files, output_folder, xlim, override_model):
+    df = load_data(files)
+    doc_types = df['doctype'].unique()
+    fig, axes = plt.subplots(len(doc_types), 1, figsize=(12, 4 * len(doc_types)), squeeze=False)
+    
+    # Set up a color palette
+    palette = sns.color_palette("husl", len(doc_types))
+    
+    for i, doc_type in enumerate(doc_types):
+        ax = axes[i, 0]
+        group = df[df['doctype'] == doc_type]
+        languages = group['lang'].unique()
+        
+        # Prepare a unique color for each language within the document type
+        colors = sns.color_palette("husl", len(languages))
+        
+        for j, lang in enumerate(languages):
+            lang_group = group[group['lang'] == lang]
+            perplexity_model = get_model_for(doc_type, override_model)
+            perplexity_values = lang_group['perplexities'].apply(lambda x: x[perplexity_model]).values
+            
+            series_color = colors[j]
+            
+            # Plot histogram with lighter color
+            sns.histplot(perplexity_values, ax=ax, color=series_color, alpha=0.3, element="step", fill=True, stat="density", binwidth=30)
+
+            # Plot KDE without filling
+            sns.kdeplot(perplexity_values, ax=ax, bw_adjust=2, color=series_color, label=f"{lang} - {doc_type} ({perplexity_model})", linewidth=1.5)
+            
+            
+            kde = gaussian_kde(perplexity_values)
+            x_range = np.linspace(0, xlim, 1000)
+            y_values = kde.evaluate(x_range)
+            
+            quartiles = np.quantile(perplexity_values, [0.25, 0.5, 0.75])
+            quartile_labels = ["Q1", "Q2", "Q3"]
+            for q, quartile in enumerate(quartiles):
+                idx = (np.abs(x_range-quartile)).argmin()
+                y_quartile = y_values[idx]
+                ax.plot([quartile, quartile], [0, y_quartile], color=series_color, linestyle='--', linewidth=1)
+                ax.text(quartile, y_quartile, f'{quartile_labels[q]}: {quartile:.2f}', verticalalignment='bottom', horizontalalignment='right', color=series_color, fontsize=6)
+            
+            ax.set_title(f'Document Type: {doc_type} ({perplexity_model})')
+            ax.set_xlabel('Perplexity Value')
+            ax.set_ylabel('Density')
+            ax.legend()
+            ax.set_xlim(left=0, right=xlim)
+        
+    plt.tight_layout()
+    output_filename = os.path.join(output_folder, "all_doc_types_plots.png")
+    plt.savefig(output_filename, dpi=300)
+    plt.close(fig)
+    print(f"All document type plots saved to {output_filename}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Plot histograms from JSON lines files.")
+    parser.add_argument('files', nargs='+', help="Path to the JSON lines files")
+    parser.add_argument('-o', '--output_folder', default=".", help="Output folder for the plots")
+    parser.add_argument('--xlim', type=int, default=2500, help="Maximum x-axis limit for the plots")
+    parser.add_argument('--model', default="", help="Override the perplexity model for all plots")
+    
+    args = parser.parse_args()
+    
+    if not os.path.exists(args.output_folder):
+        os.makedirs(args.output_folder, exist_ok=True)
+        
+    plot_histograms(args.files, args.output_folder, args.xlim, args.model)
+
+if __name__ == "__main__":
+    main()

kenlm/books.norm.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc2058f3fe709dcdc9e02c3094d9dba6e1d9e2846e3064fd597b632bdda7424f
+size 26787259332

kenlm/books.norm.arpa.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16934f2f95c19d22bf681552c5b667483e80915a72ed559e954914f492513604
+size 14951532895

kenlm/books.norm.sp.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582210ccef9a44feb2dde5029e3b02986ba3bb50d06152e2850a863fee8df16d
+size 27269792294

kenlm/books.norm.sp.arpa.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c67bb924e8d2e0515037b1aca7c381267e7363d95ae4c5a773ae8517f9c34f81
+size 14081165146

kenlm/maalfrid.norm.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9964b5a0a25e8d8f352bd85ee3de5cea80cd56cb033f4831c83e450ef42ee9b2
+size 14095675125

kenlm/maalfrid.norm.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4468f452cd224c25a7ab125f930692d415ba9a44564b6d8590ae60a697021ff8
+size 6334870758

kenlm/maalfrid.norm.sp.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12acfaf2360adec24e0456c0c9ab2a3199eda397dddb8c6b194ac7376d0811d5
+size 15096276243

kenlm/maalfrid.norm.sp.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05f2b5ee9ad6f953bcfb6ed31584706225d8390275fb78b4848b1dd697fbedb6
+size 5938309481

kenlm/newspapers.norm.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d87f6044f5f3b58b94c23e556ef2fef1f2f5cee4f27f0bd81293e6d6bb2579ff
+size 2151432996

kenlm/newspapers.norm.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e63eef20ccd2a4977f1cd314e3d42ec3c04fe68ec5fb3a5ff37e2af64d966c9a
+size 1095860943

kenlm/newspapers.norm.sp.arpa

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65bb2007e807efcb548f51c18b9c7791606bd11807e292d250051efd4529ee7b
+size 2660277943

kenlm/newspapers.norm.sp.arpa.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a79b25fc03c34278dc2cbb0b91119dfe3ba3d1e6c671b9a81127edf3746a67
+size 1217336194

normalization.py

@@ -0,0 +1,154 @@
+import argparse
+import unicodedata
+import re
+from tqdm import tqdm
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import re
+import unicodedata
+
+PUNCTS = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~«»'
+UNICODE_PUNCT = {
+    "，": ",",
+    "。": ".",
+    "、": ",",
+    "„": '"',
+    "”": '"',
+    "“": '"',
+    "«": '"',
+    "»": '"',
+    "１": '"',
+    "」": '"',
+    "「": '"',
+    "《": '"',
+    "》": '"',
+    "´": "'",
+    "∶": ":",
+    "：": ":",
+    "？": "?",
+    "！": "!",
+    "（": "(",
+    "）": ")",
+    "；": ";",
+    "–": "-",
+    "—": " - ",
+    "．": ". ",
+    "～": "~",
+    "’": "'",
+    "…": "...",
+    "━": "-",
+    "〈": "<",
+    "〉": ">",
+    "【": "[",
+    "】": "]",
+    "％": "%",
+    "►": "-",
+    "■": " ",  # added for Mimir
+}
+
+UNICODE_PUNCT_RE = re.compile(f"[{''.join(UNICODE_PUNCT.keys())}]")
+
+
+def replace_unicode_punct(text: str) -> str:
+    return "".join(UNICODE_PUNCT.get(c, c) for c in text)
+
+
+def remove_unicode_punct(text: str) -> str:
+    """More aggressive version of replace_unicode_punct but also faster."""
+    return UNICODE_PUNCT_RE.sub("", text)
+
+
+def strip_accents(line: str) -> str:
+    """Strips accents from a piece of text."""
+    nfd = unicodedata.normalize("NFD", line)
+    output = [c for c in nfd if unicodedata.category(c) != "Mn"]
+    if len(output) == line:
+        return line
+    return "".join(output)
+
+
+# Build a regex matching all control characters.
+NON_PRINTING_CHARS_RE = re.compile(
+    f"[{''.join(map(chr, list(range(0,32)) + list(range(127,160))))}]"
+)
+DIGIT_RE = re.compile(r"\d")
+PUNCT_OR_NON_PRINTING_CHARS_RE = re.compile(
+    (UNICODE_PUNCT_RE.pattern + NON_PRINTING_CHARS_RE.pattern).replace("][", "")
+)
+
+
+def remove_non_printing_char(text: str) -> str:
+    return NON_PRINTING_CHARS_RE.sub("", text)
+
+
+def normalize(line: str, accent=True, case=True, numbers=True, punct=1) -> str:
+    line = line.strip()
+    if not line:
+        return line
+    if case:
+        line = line.lower()
+    if accent:
+        line = strip_accents(line)
+    if numbers:
+        line = DIGIT_RE.sub("0", line)
+    if punct == 1:
+        line = replace_unicode_punct(line)
+    elif punct == 2:
+        line = remove_unicode_punct(line)
+    line = remove_non_printing_char(line)
+    return line
+
+
+def slow_normalize_for_dedup(line: str) -> str:
+    return normalize(line, accent=False, case=True, numbers=True, punct=2)
+
+
+def normalize_for_dedup(line: str) -> str:
+    line = line.strip()
+    if not line:
+        return line
+    # case
+    line = line.lower()
+    # numbers
+    line = DIGIT_RE.sub("0", line)
+    line = PUNCT_OR_NON_PRINTING_CHARS_RE.sub("", line)
+    return line
+
+## START OF MIMIR CODE
+def normalize_text(line):
+    normalized_line = unicodedata.normalize('NFKC', line).lower()
+
+    # Add a trailing dot if the line does not end with a punctuation mark
+    normalized_line = normalized_line.rstrip()
+    if  normalized_line and normalized_line[-1] not in PUNCTS:
+        normalized_line += '.'
+
+    # Replace newline characters with spaces (if any remain)
+    # normalized_line = re.sub(r'\r\n|\r|\n', ' ', normalized_line)
+    normalized_line = normalize(normalized_line, accent=False, case=True, numbers=True, punct=1)
+    return normalized_line
+
+
+def normalize_file(input_file, output_file, cutoff=None):
+    with (open(output_file, 'w', encoding='utf-8') as f,
+          open(input_file, 'r', encoding='utf-8') as lines):
+        for line_count, line in tqdm(enumerate(lines), desc="Processing"):
+            f.write(normalize_text(line) + "\n")
+            if cutoff and line_count >= cutoff:
+                break
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Normalize text file line by line, ensure trailing punctuation, replace newlines with spaces, and show progress.')
+    parser.add_argument('input_file', type=str, help='Input file path')
+    parser.add_argument('output_file', type=str, help='Output file path')
+    parser.add_argument('--cutoff', required=False, type=int, help='Max number of lines to process')
+
+    args = parser.parse_args()
+
+    normalize_file(args.input_file, args.output_file, args.cutoff)

perplexity.py

@@ -0,0 +1,449 @@
+import argparse
+import json
+import re
+import os
+from functools import cache
+from pathlib import Path
+from typing import Iterator, List, NoReturn, Optional, Tuple, Union
+
+import kenlm
+import msgspec
+import sentencepiece
+from numpy.random import default_rng
+from scipy.stats import norm
+from tqdm import tqdm
+
+from normalization import normalize_text
+
+
+RNG = default_rng()
+LANGS = ("no", "nn", "nob", "nno", "da", "sv", "is", "en")
+DEFAULT_LANG = "no"
+BASEPATH = Path(os.environ.get("PERPLEXITY_BASEPATH", "/nfsmounts/datastore/mimir/perplexity"))
+CONFIG = {
+    "harmful": {
+        "no": {"model": BASEPATH / "kenlm" / "harmful" / "no.bin", "normalize": True},
+        "nn": {"model": BASEPATH / "kenlm" / "harmful" / "no.bin", "normalize": True},
+        "nob": {"model": BASEPATH / "kenlm" / "harmful" / "no.bin", "normalize": True},
+        "nno": {"model": BASEPATH / "kenlm" / "harmful" / "no.bin", "normalize": True},
+        "da": {"model": BASEPATH / "kenlm" / "harmful" / "da.bin", "normalize": True},
+        "sv": {"model": BASEPATH / "kenlm" / "harmful" / "sv.bin", "normalize": True},
+        "is": {"model": BASEPATH / "kenlm" / "harmful" / "is.bin", "normalize": True},
+        "en": {"model": BASEPATH / "kenlm" / "harmful" / "en.bin", "normalize": True},
+    },
+    "wikipedia": {
+        "no": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "no.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "no.sp.model", 
+            "normalize": True
+        },
+        "nn": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "nn.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "nn.sp.model", 
+            "normalize": True
+        },
+        "nob": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "no.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "no.sp.model", 
+            "normalize": True
+        },
+        "nno": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "nn.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "nn.sp.model", 
+            "normalize": True
+        },
+        "da": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "da.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "da.sp.model", 
+            "normalize": True
+        },
+        "en": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "en.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "en.sp.model", 
+            "normalize": True
+        },
+        "is": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "is.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "is.sp.model", 
+            "normalize": True
+        },
+        "sv": {
+            "model": BASEPATH / "kenlm" / "wikipedia" / "sv.arpa.bin",
+            "tokenizer": BASEPATH / "spm" / "wikipedia" / "sv.sp.model", 
+            "normalize": True
+        },
+    },
+    "books": {
+        "model": BASEPATH / "kenlm" / "books.norm.sp.arpa.bin",
+        "tokenizer": BASEPATH / "spm" / "books.norm.sp.model", 
+        "normalize": True
+    },
+    "newspapers": {
+        "model": BASEPATH / "kenlm" / "newspapers.norm.sp.arpa.bin",
+        "tokenizer": BASEPATH / "spm" / "newspapers.norm.sp.model", 
+        "normalize": True
+    },
+    "maalfrid": {
+        "model": BASEPATH / "kenlm" / "maalfrid.norm.sp.arpa.bin",
+        "tokenizer": BASEPATH / "spm" / "maalfrid.norm.sp.model", 
+        "normalize": True
+    }
+}
+
+# Not used anymore, speed is almost same as naive algorithm
+# class PerplexityDoc(msgspec.Struct):
+#     id: str
+#     doc_type: str
+#     publish_year: int
+#     lang_fasttext: str
+#     lang_fasttext_conf: Union[str, float]
+#     text: str
+#     perplexity: float | None = -1.0
+#     perplexity_model: str | None = None
+#     harmful_pp: float | None = None
+#     # wikipedia_pp: float | None = None
+#     # books_pp: float | None = None
+#     # newspapers_pp: float | None = None
+#     # maalfrid_pp: float | None = None
+
+
+def should_keep(
+    perp: float, dist_norm: float, dist_mean: float, dist_std: float
+) -> bool:
+    """
+    Decide if a doc is to be retained based on its perplexity value
+    Note: set() must have been called previously
+    """
+    p = norm.pdf(perp, loc=dist_mean, scale=dist_std) / dist_norm
+    return RNG.uniform() < p
+
+
+def fix_language(language: str) -> str:
+    if language not in LANGS:
+        return DEFAULT_LANG
+    else:
+        return language
+
+
+def pp(log_score, length):
+    return 10.0 ** (-log_score / length)
+
+
+@cache
+def load_kenlm(model: str) -> kenlm.Model:
+    lm_config = kenlm.Config()
+    lm_config.load_method = 2
+    return kenlm.Model(str(model), lm_config)
+
+
+@cache
+def load_sentencepiece(model: str) -> sentencepiece.SentencePieceProcessor:
+    sp = sentencepiece.SentencePieceProcessor()
+    sp.load(str(model))
+    return sp
+
+
+def get_perplexity(
+    document: str,
+    model: str,
+    tokenizer: str=None,
+    normalize: bool=False
+) -> float:
+    lines = document.split("\n")
+    model = load_kenlm(model)
+    if not lines or not model:
+        return 0.0
+    if tokenizer:
+        sp = load_sentencepiece(tokenizer)
+    doc_log_score, doc_length = 0, 0
+    for line in lines:
+        if not line:
+            continue
+        if normalize:
+            line = normalize_text(line)
+        if tokenizer:
+            line = " ".join(sp.encode_as_pieces(line))
+        log_score = model.score(line)
+        length = len(line.split()) + 1
+        doc_log_score += log_score
+        doc_length += length
+
+    return round(pp(doc_log_score, doc_length), 1)
+
+
+def get_perplexity_local(
+    document: str,
+    model: kenlm.Model,
+    tokenizer: sentencepiece.SentencePieceProcessor=None,
+    normalize: bool=False
+) -> float:
+    lines = document.split("\n")
+    if not lines or not model:
+        return 0.0
+    doc_log_score, doc_length = 0, 0
+    for line in lines:
+        if normalize:
+            line = normalize_text(line)
+        if tokenizer is not None:
+            line = " ".join(tokenizer.encode_as_pieces(line))
+        log_score = model.score(line)
+        length = len(line.split()) + 1
+        doc_log_score += log_score
+        doc_length += length
+
+    return round(pp(doc_log_score, doc_length), 1)
+
+
+def harmful_perplexity(document: str, language: str) -> float:
+    params = CONFIG["harmful"][fix_lang(language)]
+    return get_perplexity(document=document, **params)
+
+
+def wikipedia_perplexity(document: str, language: str) -> float:
+    params = CONFIG["wikipedia"][fix_lang(language)]
+    return get_perplexity(document=document, **params)
+
+
+def books_perplexity(document: str) -> float:
+    params = CONFIG["books"]
+    return get_perplexity(document=document, **params)
+
+
+def newspapers_perplexity(document: str) -> float:
+    params = CONFIG["newspapers"]
+    return get_perplexity(document=document, **params)
+
+
+def maalfrid_perplexity(document: str) -> float:
+    params = CONFIG["maalfrid"]
+    return get_perplexity(document=document, **params)
+
+
+def source_perplexities(
+    document: str,
+    language: str,
+    model: str | None = None,
+    include_harmful: bool=True) -> float:
+    """Calculates all models perplexities at once"""
+    # Since normalization is applied to all, we normalize first and set it to False
+    normalized_document = "\n".join(normalize_text(line) for line in document.split("\n"))
+    language = fix_language(language)
+    
+    if model is not None:
+        params = CONFIG[model]
+        if model == "wikipedia":
+            params = params[language]
+        params.update({"normalize": False})
+        perplexity = get_perplexity(document=normalized_document, **params)
+        perplexities = {
+            f"{model}_pp": perplexity,
+        }
+    else:
+        params = CONFIG["wikipedia"][language]
+        params.update({"normalize": False})
+        wikipedia_perplexity = get_perplexity(document=normalized_document, **params)
+
+        params = CONFIG["books"]
+        params.update({"normalize": False})
+        books_perplexity = get_perplexity(document=normalized_document, **params)
+        
+        params = CONFIG["newspapers"]
+        params.update({"normalize": False})
+        newspapers_perplexity = get_perplexity(document=normalized_document, **params)
+
+        params = CONFIG["maalfrid"]
+        params.update({"normalize": False})
+        maalfrid_perplexity = get_perplexity(document=normalized_document, **params)
+        perplexities = {
+            "wikipedia_pp": wikipedia_perplexity,
+            "books_pp": books_perplexity,
+            "newspapers_pp": newspapers_perplexity,
+            "maalfrid_pp": maalfrid_perplexity,
+        }
+    if include_harmful:
+        params = CONFIG["harmful"][language]
+        params.update({"normalize": False})
+        harmful_perplexity = get_perplexity(document=normalized_document, **params)
+        perplexities.update({
+            "harmful_pp": harmful_perplexity,
+        })
+    return perplexities        
+
+
+def get_model_for(doc_type: str) -> (str, bool):
+    """Returns model type and if it needs a language variant"""
+    doc_type = doc_type.split("_", 1)[0]
+    if "-" in doc_type:
+        doc_type = doc_type.split("-", 1)[-1]
+    if doc_type in ("book", "books"):
+        return "books", False
+    elif doc_type in ("culturax", "slimpajama", "wikipedia", "digimanus", "pg19", "hplt", "starcoder"):
+        return "wikipedia", True
+    elif doc_type in ("newspaper", "newspapers"):
+        return "newspapers", False
+    elif doc_type in ("evalueringsrapport", "lovdata", "maalfrid", "parlamint"):
+        return "maalfrid", False
+    else:
+        return "wikipedia", True
+
+
+def preload_models_tokenizers() -> List:
+    print("Preloading models...", end=" ")
+    models = {
+        "books": (
+            load_kenlm(BASEPATH / "kenlm" / "books.norm.arpa.bin"),
+            load_sentencepiece(BASEPATH / "spm" / "books.norm.sp.model")
+        ),
+        "newspapers": (
+            load_kenlm(BASEPATH / "kenlm" / "newspapers.norm.arpa.bin"),
+            load_sentencepiece(BASEPATH / "spm" / "newspapers.norm.sp.model")
+        ),
+        "maalfrid": (
+            load_kenlm(BASEPATH / "kenlm" / "maalfrid.norm.arpa.bin"),
+            load_sentencepiece(BASEPATH / "spm" / "maalfrid.norm.sp.model")
+        ),
+    }
+    for lang, params in CONFIG["harmful"].items():
+        model = load_kenlm(params["model"])
+        models[f"harmful-{lang}"] = model, None
+
+    for lang, params in CONFIG["wikipedia"].items():
+        model = load_kenlm(params["model"])
+        tokenizer = load_sentencepiece(params["tokenizer"])
+        models[f"wikipedia-{lang}"] = model, tokenizer
+    print("Done")
+    return models
+
+
+# Not used anymore, speed is almost same as naive algorithm
+# def process_file_binary(input_file, output_path, cutoff=None, overwrite_output=True):
+#     input_file = Path(input_file)
+#     output_file = Path(output_path) / input_file.name
+#     if not overwrite_output and output_file.exists():
+#         print(f"Skipping {output_file} as it already exists")
+#         return
+#     models = preload_models_tokenizers()
+#     encoder = msgspec.json.Encoder()
+#     decoder = msgspec.json.Decoder(PerplexityDoc)
+#     buffer = bytearray(64)
+#     with (open(output_file, 'wb') as f,
+#           open(input_file, 'r', encoding='utf-8') as lines):
+#         for line_count, line in tqdm(enumerate(lines), desc=f"Processing {input_file.name}"):
+#             doc = decoder.decode(line)
+#             if "code" not in doc.doc_type:
+#                 # Perplexity
+#                 model_type, needs_lang = get_model_for(doc.doc_type)
+#                 if needs_lang:
+#                     model_key = f"{model_type}-{fix_language(doc.lang_fasttext)}"
+#                 else:
+#                     model_key = model_type
+#                 model, tokenizer = models[model_key]
+#                 text = "\n".join(normalize_text(line) for line in doc.text.split("\n"))
+#                 score = get_perplexity_local(
+#                     text, model=model, tokenizer=tokenizer, normalize=False
+#                 )
+#                 doc.perplexity = score
+#                 doc.perplexity_model = model_type
+#                 # Harmfulness
+#                 harmful_key =  f"harmful-{fix_language(doc.lang_fasttext)}"
+#                 harmful_model, harmful_tokenizer = models[harmful_key]
+#                 harmful_pp = get_perplexity_local(
+#                     text, model=harmful_model, tokenizer=harmful_tokenizer, normalize=False
+#                 )
+#                 doc.harmful_pp = harmful_pp
+
+#             encoder.encode_into(doc, buffer)
+#             buffer.extend(b"\n")            
+#             f.write(buffer)
+#             if cutoff is not None and line_count >= cutoff:
+#                 break
+
+
+def process_file(input_file, output_path, cutoff=None, model=None, overwrite_output=True):
+    """
+    Processes a file by reading its contents, analyzing each line for language and document type,
+    computing perplexities using specified models, and writing the modified content to a new file.
+
+    This function performs several steps:
+    1. Determines the output file path and checks for its existence if overwrite is not desired.
+    2. Reads the input file line by line, processing each line as a separate JSON document.
+    3. For each document, identifies its language using a fastText model. If the document type is "starcoder", 
+       it defaults the language to English.
+    4. Depending on the model parameter, computes perplexities for the document text either using a 
+       single document type model or a specified general model.
+    5. Updates the document with computed perplexities and writes it to the output file in JSON format.
+    6. Optionally stops processing after a specified number of lines determined by the cutoff parameter.
+
+    Parameters:
+    - input_file (str or Path): Path to the input file to be processed.
+    - output_path (str or Path): Directory path where the output file will be saved. The output file
+                                 will have the same name as the input file.
+    - cutoff (int, optional): If provided, processing will stop after this number of lines. Defaults to None.
+    - model (str, optional): Specifies the model to use for computing perplexities. If 'single', uses a
+                             model specific to the document's type. Otherwise, uses the model specified. 
+                             Defaults to None.
+    - overwrite_output (bool): If True, will overwrite the output file if it already exists. If False, 
+                               will skip processing if the output file exists. Defaults to True.
+
+    Returns:
+    None. Writes processed documents to an output file in the specified output path.
+    """
+    input_file = Path(input_file)
+    output_file = Path(output_path) / input_file.name
+    if not overwrite_output and output_file.exists():
+        print(f"Skipping {output_file} as it already exists")
+        return
+    with (open(output_file, 'w', encoding='utf-8') as f,
+          open(input_file, 'r', encoding='utf-8') as lines):
+        for line_count, line in tqdm(enumerate(lines), desc=f"Processing {input_file.name}"):
+            doc = json.loads(line)
+            language = doc["lang_fasttext"]
+            if doc["doc_type"] == "starcoder":
+                language = "en"
+            if model == "single":
+                doc_type_model, _ = get_model_for(doc["doc_type"])
+                perplexities = source_perplexities(doc["text"], language, model=doc_type_model)
+                perplexities["perplexity"] = perplexities.pop(f"{doc_type_model}_pp")
+                perplexities["perplexity_model"] = doc_type_model
+            else:
+                perplexities = source_perplexities(doc["text"], language, model=model)
+            doc.update(perplexities)
+            f.write(json.dumps(doc) + "\n")
+            if cutoff is not None and line_count >= cutoff:
+                break
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Calculate perplexity values for a given JSON Lines file and output the result to a new file.')
+    parser.add_argument('-i', '--input_file', type=str,
+                        help='Input file path')
+    parser.add_argument('-o', '--output_path', type=str,
+                        help='Output path to write enriched file')
+    parser.add_argument('-c', '--cutoff', required=False, type=int,
+                        help='Max number of lines to process')
+    parser.add_argument('-m', '--model', required=False, type=str,
+                        help='Run "single" model per doc type, "all" the models, '
+                             'or a specific model to choose from '
+                             '"books", "wikipedia", "newspapers" or "maalfrid". '
+                             'Defaults to "single"')
+    parser.add_argument('--overwrite_output',
+                        action=argparse.BooleanOptionalAction, default=True,
+                        help="Whether to overwrite the output file if exists.")
+
+    args = parser.parse_args()
+
+    if args.model == "single":
+        process_file(
+            args.input_file, args.output_path, args.cutoff,
+            model="single", overwrite_output=args.overwrite_output,
+        )
+    elif args.model in ("books", "wikipedia", "newspapers", "maalfrid"):
+        process_file(
+            args.input_file, args.output_path, args.cutoff,
+            model=args.model, overwrite_output=args.overwrite_output,
+        )
+    else:
+        process_file(
+            args.input_file, args.output_path, args.cutoff,
+            overwrite_output=args.overwrite_output,
+        )