add notebooks from Kaggle

kaggle-notebooks/cmu-us-awb-arctic-fairseq-files.ipynb

@@ -0,0 +1,393 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "bddfd111",
+   "metadata": {
+    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
+    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.094601Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.094098Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.103907Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.103002Z"
+    },
+    "papermill": {
+     "duration": 0.026388,
+     "end_time": "2022-05-07T08:23:49.105961",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.079573",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "RAWTEXT = \"../input/cmu-us-awb-arctic-tts-dataset/cmu_us_awb_arctic/etc/txt.done.data\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "146a024b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.130710Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.130417Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.135862Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.134985Z"
+    },
+    "papermill": {
+     "duration": 0.020182,
+     "end_time": "2022-05-07T08:23:49.138041",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.117859",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "NORMS = {\n",
+    "    \"0.75\": \"zero point seven five\",\n",
+    "    \"t.h\": \"t h\",\n",
+    "    \"1880\": \"eighteen eighty\",\n",
+    "    \"16\": \"sixteenth\",\n",
+    "    \"1908\": \"nineteen oh eight\",\n",
+    "    \"18\": \"eighteenth\",\n",
+    "    \"17\": \"seventeenth\",\n",
+    "    \"29th\": \"twenty ninth\",\n",
+    "    \"mrs\": \"misses\",\n",
+    "    \"etc\": \"etcetera\",\n",
+    "    \"etc.\": \"etcetera\",\n",
+    "    \"to-day\": \"today\",\n",
+    "    \"to-day's\": \"today's\",\n",
+    "    \"to-morrow\": \"tomorrow\"\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "46c0c682",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.163049Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.162461Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.170619Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.169730Z"
+    },
+    "papermill": {
+     "duration": 0.022915,
+     "end_time": "2022-05-07T08:23:49.172592",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.149677",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def _check_apos(word):\n",
+    "    if word.endswith(\"'s\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"s'\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"'d\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"'ve\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"'re\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"'ll\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"n't\"):\n",
+    "        return word\n",
+    "    elif word.endswith(\"'ve\"):\n",
+    "        return word\n",
+    "    elif word in [\"i'm\", \"'em\", \"o'brien\"]:\n",
+    "        return word\n",
+    "    else:\n",
+    "        return word.replace(\"'\", \"\")\n",
+    "\n",
+    "def fix_apos(text):\n",
+    "    words = [_check_apos(w) for w in text.split(\" \")]\n",
+    "    return \" \".join(words)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "ad9ec20c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.197464Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.197091Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.206427Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.205485Z"
+    },
+    "papermill": {
+     "duration": 0.024203,
+     "end_time": "2022-05-07T08:23:49.208449",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.184246",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def normalise(text):\n",
+    "    if text[-1] == \".\":\n",
+    "        text = text[:-1]\n",
+    "    text = text.lower()\n",
+    "    words = []\n",
+    "    text = text.replace(\",\", \"\")\n",
+    "    for word in text.split(\" \"):\n",
+    "        if word in NORMS:\n",
+    "            words.append(NORMS[word])\n",
+    "        else:\n",
+    "            words.append(word)\n",
+    "    text = \" \".join(words)\n",
+    "    text = text.replace(\".\", \"\")\n",
+    "    text = text.replace(\"?\", \"\")\n",
+    "    text = text.replace(\"!\", \"\")\n",
+    "    text = text.replace(\":\", \"\")\n",
+    "    text = text.replace(\";\", \"\")\n",
+    "    text = text.replace(\"--\", \" \")\n",
+    "    text = text.replace(\"  \", \" \")\n",
+    "    text = text.replace(\" - \", \" \")\n",
+    "    text = text.replace(\"to- morrow\", \"tomorrow\")\n",
+    "    text = fix_apos(text)\n",
+    "    text = text.replace(\"-\", \" \")\n",
+    "    return text.strip().upper()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "aec22c8a",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.233246Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.232952Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.275141Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.274337Z"
+    },
+    "papermill": {
+     "duration": 0.057432,
+     "end_time": "2022-05-07T08:23:49.277432",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.220000",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "data = {}\n",
+    "with open(RAWTEXT) as inf:\n",
+    "    for line in inf.readlines():\n",
+    "        first_space = line.find(' ')\n",
+    "        first_quote = line.find('\"')\n",
+    "        last_quote = line.rfind('\"')\n",
+    "        id = line[first_space+1:first_quote].strip()\n",
+    "        text = line[first_quote+1:last_quote]\n",
+    "        data[id] = normalise(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "c4d12ef6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.302618Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.302155Z",
+     "iopub.status.idle": "2022-05-07T08:23:49.306841Z",
+     "shell.execute_reply": "2022-05-07T08:23:49.306240Z"
+    },
+    "papermill": {
+     "duration": 0.019827,
+     "end_time": "2022-05-07T08:23:49.308829",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.289002",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "with open(\"text.tsv\", \"w\") as of:\n",
+    "    for id in data.keys():\n",
+    "        of.write(f\"{id}\\t{data[id]}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "62dc9132",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:49.333170Z",
+     "iopub.status.busy": "2022-05-07T08:23:49.332878Z",
+     "iopub.status.idle": "2022-05-07T08:23:58.857125Z",
+     "shell.execute_reply": "2022-05-07T08:23:58.855696Z"
+    },
+    "papermill": {
+     "duration": 9.539562,
+     "end_time": "2022-05-07T08:23:58.859914",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:49.320352",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total: 4777.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import soundfile as sf\n",
+    "\n",
+    "total = 0\n",
+    "WAVPATH = Path(\"../input/cmu-us-awb-arctic-tts-dataset/cmu_us_awb_arctic/wav/\")\n",
+    "with open(\"frames.tsv\", \"w\") as of:\n",
+    "    for wav in WAVPATH.glob(\"*.wav\"):\n",
+    "        frames, sr = sf.read(str(wav))\n",
+    "        assert sr == 16000\n",
+    "        total += len(frames)\n",
+    "        of.write(f\"{wav.stem}.wav\\t{len(frames)}\\n\")\n",
+    "print(\"Total:\", total / 16000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "71a8a65f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:23:58.896455Z",
+     "iopub.status.busy": "2022-05-07T08:23:58.895715Z",
+     "iopub.status.idle": "2022-05-07T08:24:01.181186Z",
+     "shell.execute_reply": "2022-05-07T08:24:01.179981Z"
+    },
+    "papermill": {
+     "duration": 2.308171,
+     "end_time": "2022-05-07T08:24:01.183774",
+     "exception": false,
+     "start_time": "2022-05-07T08:23:58.875603",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "lines=!wc -l frames.tsv|awk '{print $1}'\n",
+    "!tail -n 114 frames.tsv |head -n 57 > test.tsv\n",
+    "!tail -n 114 frames.tsv |tail -n 57 > dev.tsv\n",
+    "!head -n $((1138-114)) frames.tsv > train.tsv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "6c7413f1",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:24:01.210537Z",
+     "iopub.status.busy": "2022-05-07T08:24:01.209625Z",
+     "iopub.status.idle": "2022-05-07T08:24:01.215252Z",
+     "shell.execute_reply": "2022-05-07T08:24:01.214668Z"
+    },
+    "papermill": {
+     "duration": 0.021169,
+     "end_time": "2022-05-07T08:24:01.217174",
+     "exception": false,
+     "start_time": "2022-05-07T08:24:01.196005",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def do_fairseq(text):\n",
+    "    words = text.split(\" \")\n",
+    "    owords = [\" \".join(w) for w in words]\n",
+    "    return \" | \".join(owords) + \" |\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "9342e47e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T08:24:01.242716Z",
+     "iopub.status.busy": "2022-05-07T08:24:01.242299Z",
+     "iopub.status.idle": "2022-05-07T08:24:01.257797Z",
+     "shell.execute_reply": "2022-05-07T08:24:01.257054Z"
+    },
+    "papermill": {
+     "duration": 0.031049,
+     "end_time": "2022-05-07T08:24:01.260092",
+     "exception": false,
+     "start_time": "2022-05-07T08:24:01.229043",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for part in [\"test\", \"train\", \"dev\"]:\n",
+    "    ids = []\n",
+    "    with open(f\"{part}.ltr\", \"w\") as of, open(f\"{part}.tsv\") as inf:\n",
+    "        for line in inf.readlines():\n",
+    "            if \"\\t\" in line:\n",
+    "                parts = line.strip().split(\"\\t\")\n",
+    "                id = parts[0].replace(\".wav\", \"\")\n",
+    "                of.write(do_fairseq(data[id]) + \"\\n\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": 23.168986,
+   "end_time": "2022-05-07T08:24:01.994036",
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "__notebook__.ipynb",
+   "output_path": "__notebook__.ipynb",
+   "parameters": {},
+   "start_time": "2022-05-07T08:23:38.825050",
+   "version": "2.3.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

kaggle-notebooks/create-awb-splits.ipynb

@@ -0,0 +1,507 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d250a6ca",
+   "metadata": {
+    "papermill": {
+     "duration": 0.015615,
+     "end_time": "2022-05-07T20:21:51.320467",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.304852",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Load frame lengths"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c67326d6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.351926Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.351146Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.375423Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.374683Z"
+    },
+    "papermill": {
+     "duration": 0.042741,
+     "end_time": "2022-05-07T20:21:51.377820",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.335079",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "train_frames = {}\n",
+    "\n",
+    "total = 0\n",
+    "with open(\"../input/cmu-us-awb-arctic-fairseq-files/train.tsv\") as f:\n",
+    "    for line in f.readlines():\n",
+    "        if not \"\\t\" in line:\n",
+    "            continue\n",
+    "        pieces = line.strip().split(\"\\t\")\n",
+    "        total += int(pieces[1])\n",
+    "        assert len(pieces) == 2\n",
+    "        id = pieces[0].replace(\".wav\", \"\")\n",
+    "        train_frames[id] = int(pieces[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "b7994029",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.408607Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.408333Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.415606Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.414918Z"
+    },
+    "papermill": {
+     "duration": 0.025832,
+     "end_time": "2022-05-07T20:21:51.418472",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.392640",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4295.0"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "total / 16000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "3f849d42",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.453347Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.452190Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.458730Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.457854Z"
+    },
+    "papermill": {
+     "duration": 0.025537,
+     "end_time": "2022-05-07T20:21:51.461052",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.435515",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "71.58333333333333"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "4295 / 60"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9f9a6faf",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.493604Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.492779Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.497692Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.497031Z"
+    },
+    "papermill": {
+     "duration": 0.023598,
+     "end_time": "2022-05-07T20:21:51.499886",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.476288",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "MINS = [i * 5 for i in range(1, 13)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "48ffe86d",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.532980Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.532352Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.539573Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.538702Z"
+    },
+    "papermill": {
+     "duration": 0.026147,
+     "end_time": "2022-05-07T20:21:51.541893",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.515746",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "MINS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "c2190118",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.575902Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.575081Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.579726Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.578698Z"
+    },
+    "papermill": {
+     "duration": 0.02397,
+     "end_time": "2022-05-07T20:21:51.581997",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.558027",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "WAVDIR = \"/kaggle/input/ljspeech-for-asr/wav16\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0dd1b778",
+   "metadata": {
+    "papermill": {
+     "duration": 0.016062,
+     "end_time": "2022-05-07T20:21:51.613958",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.597896",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Minute splits"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b39e450f",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.647074Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.646766Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.665170Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.664195Z"
+    },
+    "papermill": {
+     "duration": 0.03809,
+     "end_time": "2022-05-07T20:21:51.667729",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.629639",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for min in MINS:\n",
+    "    frames = min * 60 * 16000\n",
+    "    idlist = [k for k in train_frames.keys()]\n",
+    "    outtsv = f\"{min}mins.tsv\"\n",
+    "    with open(outtsv, \"w\") as of:\n",
+    "        current = 0\n",
+    "        of.write(f\"{WAVDIR}\\n\")\n",
+    "        while frames > 0 and frames > current:\n",
+    "            id = idlist.pop(0)\n",
+    "            current = train_frames[id]\n",
+    "            of.write(f\"{id}.wav\\t{current}\\n\")\n",
+    "            frames = frames - current\n",
+    "        max = 0\n",
+    "        maxid = \"\"\n",
+    "        for id in idlist:\n",
+    "            time = train_frames[id]\n",
+    "            if time > current:\n",
+    "                continue\n",
+    "            if time > max:\n",
+    "                max = time\n",
+    "                maxid = id\n",
+    "        of.write(f\"{maxid}.wav\\t{max}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f4127c0d",
+   "metadata": {
+    "papermill": {
+     "duration": 0.016089,
+     "end_time": "2022-05-07T20:21:51.699865",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.683776",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Generate `ltr` files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "df62fef6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.734361Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.734065Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.739428Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.738516Z"
+    },
+    "papermill": {
+     "duration": 0.024729,
+     "end_time": "2022-05-07T20:21:51.741510",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.716781",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def fairseqify(text):\n",
+    "    text = text.strip().replace(\"  \", \" \")\n",
+    "    words = text.split(\" \")\n",
+    "    spread = [\" \".join(a) for a in words]\n",
+    "    return \" | \".join(spread) + \" |\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1f1f0ede",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.775253Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.774938Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.796186Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.795461Z"
+    },
+    "papermill": {
+     "duration": 0.040472,
+     "end_time": "2022-05-07T20:21:51.798476",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.758004",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "transcripts = {}\n",
+    "with open(\"../input/cmu-us-awb-arctic-fairseq-files/text.tsv\") as tf:\n",
+    "    for line in tf.readlines():\n",
+    "        line = line.strip()\n",
+    "        if not \"\\t\" in line:\n",
+    "            pass\n",
+    "        parts = line.split(\"\\t\")\n",
+    "        assert len(parts) == 2\n",
+    "        transcripts[parts[0]] = fairseqify(parts[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "b39c56c7",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.832082Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.831792Z",
+     "iopub.status.idle": "2022-05-07T20:21:51.848438Z",
+     "shell.execute_reply": "2022-05-07T20:21:51.847533Z"
+    },
+    "papermill": {
+     "duration": 0.036232,
+     "end_time": "2022-05-07T20:21:51.850791",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.814559",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "for tsv in glob.glob(\"*.tsv\"):\n",
+    "    out = tsv.replace(\".tsv\", \".ltr\")\n",
+    "    with open(tsv) as inf, open(out, \"w\") as outf:\n",
+    "        for line in inf.readlines()[1:]:\n",
+    "            id, _ = line.split(\"\\t\")\n",
+    "            id = id.replace(\".wav\", \"\")\n",
+    "            outf.write(f\"{transcripts[id]}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b5d1f3dc",
+   "metadata": {
+    "papermill": {
+     "duration": 0.015826,
+     "end_time": "2022-05-07T20:21:51.883384",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.867558",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Tidy up"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "8bf75958",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:51.917484Z",
+     "iopub.status.busy": "2022-05-07T20:21:51.917020Z",
+     "iopub.status.idle": "2022-05-07T20:21:53.699362Z",
+     "shell.execute_reply": "2022-05-07T20:21:53.698383Z"
+    },
+    "papermill": {
+     "duration": 1.802712,
+     "end_time": "2022-05-07T20:21:53.702205",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:51.899493",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2022-05-07 20:21:52--  https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt\r\n",
+      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\r\n",
+      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\r\n",
+      "HTTP request sent, awaiting response... 200 OK\r\n",
+      "Length: 207 [text/plain]\r\n",
+      "Saving to: ‘dict.ltr.txt’\r\n",
+      "\r\n",
+      "dict.ltr.txt        100%[===================>]     207  --.-KB/s    in 0s      \r\n",
+      "\r\n",
+      "2022-05-07 20:21:53 (40.9 MB/s) - ‘dict.ltr.txt’ saved [207/207]\r\n",
+      "\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "!wget https://dl.fbaipublicfiles.com/fairseq/wav2vec/dict.ltr.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "cd0e9762",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2022-05-07T20:21:53.741484Z",
+     "iopub.status.busy": "2022-05-07T20:21:53.741123Z",
+     "iopub.status.idle": "2022-05-07T20:21:54.715272Z",
+     "shell.execute_reply": "2022-05-07T20:21:54.714013Z"
+    },
+    "papermill": {
+     "duration": 0.997334,
+     "end_time": "2022-05-07T20:21:54.717846",
+     "exception": false,
+     "start_time": "2022-05-07T20:21:53.720512",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!for i in *mins.tsv;do b=$(basename $i \".tsv\");mkdir $b; mv $b.tsv $b/train.tsv; mv $b.ltr $b/train.ltr; cp dict.ltr.txt ../input/cmu-us-awb-arctic-fairseq-files/test.* $b/;cp ../input/cmu-us-awb-arctic-fairseq-files/dev.tsv $b/valid.tsv; cp ../input/cmu-us-awb-arctic-fairseq-files/dev.ltr $b/valid.ltr;done"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": 14.656502,
+   "end_time": "2022-05-07T20:21:55.457936",
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "__notebook__.ipynb",
+   "output_path": "__notebook__.ipynb",
+   "parameters": {},
+   "start_time": "2022-05-07T20:21:40.801434",
+   "version": "2.3.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}