update

.gitattributes

@@ -0,0 +1 @@
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,21 @@
+# Python
+__pycache__
+*.py[cod]
+*.so
+.Python
+env/
+venv/
+*.egg-info/
+dist/
+build/
+*.egg
+.idea/
+.vscode/
+*.swp
+.DS_Store
+# Node
+node_modules/
+*.log
+*.env
+*.cache
+

Dockerfile

@@ -0,0 +1,40 @@
+# Use an official Node runtime as the base image for building the application
+FROM node:20 AS build
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy package.json and package-lock.json
+COPY app/package*.json ./
+
+# Install dependencies
+RUN npm install
+
+# Copy the rest of the application code
+COPY app/ .
+COPY analysis/data ../analysis/data
+
+# Build the application
+RUN npm run build
+
+# Use an official Nginx runtime as the base image for serving the application
+FROM nginx:alpine
+
+# Copy the built application from the build stage
+COPY --from=build /app/dist /usr/share/nginx/html
+
+# Copy a custom Nginx configuration file
+COPY nginx.conf /etc/nginx/nginx.conf
+
+# Create necessary directories and set permissions
+RUN mkdir -p /var/cache/nginx /var/run /var/log/nginx && \
+    chmod -R 777 /var/cache/nginx /var/run /var/log/nginx /etc/nginx/nginx.conf
+
+# Switch to non-root user
+USER nginx
+
+# Expose port 8080
+EXPOSE 8080
+
+# Command to run the application
+CMD ["nginx", "-g", "daemon off;"]

README.md

@@ -0,0 +1,11 @@
+---
+title: 'Scaling FineWeb to 1000+ languages: Step 1: finding signal in 100s of evaluation tasks'
+emoji: 📝
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+pinned: false
+header: mini
+app_port: 8080
+thumbnail: https://huggingface.co/spaces/HuggingFaceFW/blogpost-fine-tasks/resolve/main/app/assets/images/banner.png
+---

app/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+*.log
+*.env
+*.cache

app/README.md

@@ -0,0 +1,3 @@
+## How to run
+1. npm install
+2. npm run dev

app/src/bibliography.bib

@@ -0,0 +1,398 @@
+@misc{penedo2024finewebdatasetsdecantingweb,
+  title={The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale},
+  author={Guilherme Penedo and Hynek Kydlíček and Loubna Ben allal and Anton Lozhkov and Margaret Mitchell and Colin Raffel and Leandro Von Werra and Thomas Wolf},
+  year={2024},
+  eprint={2406.17557},
+  archivePrefix={arXiv},
+  primaryClass={cs.CL},
+  url={https://arxiv.org/abs/2406.17557},
+}
+@article{hendryckstest2021,
+  title={Measuring Massive Multitask Language Understanding},
+  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
+  year={2021}
+}
+@inproceedings{zellers2019hellaswag,
+    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
+    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
+    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
+    year={2019}
+}
+@misc{madaan2024quantifyingvarianceevaluationbenchmarks,
+      title={Quantifying Variance in Evaluation Benchmarks},
+      author={Lovish Madaan and Aaditya K. Singh and Rylan Schaeffer and Andrew Poulton and Sanmi Koyejo and Pontus Stenetorp and Sharan Narang and Dieuwke Hupkes},
+      year={2024},
+      eprint={2406.10229},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2406.10229},
+}
+@misc{open-llm-leaderboard-v2,
+  author = {Clémentine Fourrier and Nathan Habib and Alina Lozovskaya and Konrad Szafer and Thomas Wolf},
+  title = {Open LLM Leaderboard v2},
+  year = {2024},
+  publisher = {Hugging Face},
+  howpublished = "\url{https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard}",
+}
+@misc{biderman2024lessonstrenchesreproducibleevaluation,
+      title={Lessons from the Trenches on Reproducible Evaluation of Language Models},
+      author={Stella Biderman and Hailey Schoelkopf and Lintang Sutawika and Leo Gao and Jonathan Tow and Baber Abbasi and Alham Fikri Aji and Pawan Sasanka Ammanamanchi and Sidney Black and Jordan Clive and Anthony DiPofi and Julen Etxaniz and Benjamin Fattori and Jessica Zosa Forde and Charles Foster and Jeffrey Hsu and Mimansa Jaiswal and Wilson Y. Lee and Haonan Li and Charles Lovering and Niklas Muennighoff and Ellie Pavlick and Jason Phang and Aviya Skowron and Samson Tan and Xiangru Tang and Kevin A. Wang and Genta Indra Winata and François Yvon and Andy Zou},
+      year={2024},
+      eprint={2405.14782},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2405.14782},
+}
+@misc{li2024datacomplmsearchgenerationtraining,
+      title={DataComp-LM: In search of the next generation of training sets for language models},
+      author={Jeffrey Li and Alex Fang and Georgios Smyrnis and Maor Ivgi and Matt Jordan and Samir Gadre and Hritik Bansal and Etash Guha and Sedrick Keh and Kushal Arora and Saurabh Garg and Rui Xin and Niklas Muennighoff and Reinhard Heckel and Jean Mercat and Mayee Chen and Suchin Gururangan and Mitchell Wortsman and Alon Albalak and Yonatan Bitton and Marianna Nezhurina and Amro Abbas and Cheng-Yu Hsieh and Dhruba Ghosh and Josh Gardner and Maciej Kilian and Hanlin Zhang and Rulin Shao and Sarah Pratt and Sunny Sanyal and Gabriel Ilharco and Giannis Daras and Kalyani Marathe and Aaron Gokaslan and Jieyu Zhang and Khyathi Chandu and Thao Nguyen and Igor Vasiljevic and Sham Kakade and Shuran Song and Sujay Sanghavi and Fartash Faghri and Sewoong Oh and Luke Zettlemoyer and Kyle Lo and Alaaeldin El-Nouby and Hadi Pouransari and Alexander Toshev and Stephanie Wang and Dirk Groeneveld and Luca Soldaini and Pang Wei Koh and Jenia Jitsev and Thomas Kollar and Alexandros G. Dimakis and Yair Carmon and Achal Dave and Ludwig Schmidt and Vaishaal Shankar},
+      year={2024},
+      eprint={2406.11794},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2406.11794},
+}
+@misc{gu2024olmesstandardlanguagemodel,
+      title={OLMES: A Standard for Language Model Evaluations},
+      author={Yuling Gu and Oyvind Tafjord and Bailey Kuehl and Dany Haddad and Jesse Dodge and Hannaneh Hajishirzi},
+      year={2024},
+      eprint={2406.08446},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2406.08446},
+}
+@article{radford2019language,
+  title={Language Models are Unsupervised Multitask Learners},
+  author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
+  year={2019}
+}
+@inproceedings{barbaresi-2021-trafilatura,
+  title = {Trafilatura: A Web Scraping Library and Command-Line Tool for Text Discovery and Extraction},
+  author = "Barbaresi, Adrien",
+  booktitle = "Proceedings of the Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing: System Demonstrations",
+  pages = "122--131",
+  publisher = "Association for Computational Linguistics",
+  url = "https://aclanthology.org/2021.acl-demo.15",
+  year = 2021,
+}
+@misc{penedo2023refinedweb,
+      title={The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only},
+      author={Guilherme Penedo and Quentin Malartic and Daniel Hesslow and Ruxandra Cojocaru and Alessandro Cappelli and Hamza Alobeidli and Baptiste Pannier and Ebtesam Almazrouei and Julien Launay},
+      year={2023},
+      eprint={2306.01116},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@article{joulin2016fasttext,
+  title={FastText.zip: Compressing text classification models},
+  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\'e}gou, H{\'e}rve and Mikolov, Tomas},
+  journal={arXiv preprint arXiv:1612.03651},
+  year={2016}
+}
+@article{joulin2016bag,
+  title={Bag of Tricks for Efficient Text Classification},
+  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},
+  journal={arXiv preprint arXiv:1607.01759},
+  year={2016}
+}
+@misc{penedo2024datatrove,
+  author = {Penedo, Guilherme and Kydlíček, Hynek and Cappelli, Alessandro and Sasko, Mario and Wolf, Thomas},
+  title = {DataTrove: large scale data processing},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+  url = {https://github.com/huggingface/datatrove}
+}
+@misc{chiang2024chatbot,
+      title={Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference},
+      author={Wei-Lin Chiang and Lianmin Zheng and Ying Sheng and Anastasios Nikolas Angelopoulos and Tianle Li and Dacheng Li and Hao Zhang and Banghua Zhu and Michael Jordan and Joseph E. Gonzalez and Ion Stoica},
+      year={2024},
+      eprint={2403.04132},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{rae2022scaling,
+      title={Scaling Language Models: Methods, Analysis & Insights from Training Gopher},
+      author={Jack W. Rae and Sebastian Borgeaud and Trevor Cai and Katie Millican and Jordan Hoffmann and Francis Song and John Aslanides and Sarah Henderson and Roman Ring and Susannah Young and Eliza Rutherford and Tom Hennigan and Jacob Menick and Albin Cassirer and Richard Powell and George van den Driessche and Lisa Anne Hendricks and Maribeth Rauh and Po-Sen Huang and Amelia Glaese and Johannes Welbl and Sumanth Dathathri and Saffron Huang and Jonathan Uesato and John Mellor and Irina Higgins and Antonia Creswell and Nat McAleese and Amy Wu and Erich Elsen and Siddhant Jayakumar and Elena Buchatskaya and David Budden and Esme Sutherland and Karen Simonyan and Michela Paganini and Laurent Sifre and Lena Martens and Xiang Lorraine Li and Adhiguna Kuncoro and Aida Nematzadeh and Elena Gribovskaya and Domenic Donato and Angeliki Lazaridou and Arthur Mensch and Jean-Baptiste Lespiau and Maria Tsimpoukelli and Nikolai Grigorev and Doug Fritz and Thibault Sottiaux and Mantas Pajarskas and Toby Pohlen and Zhitao Gong and Daniel Toyama and Cyprien de Masson d'Autume and Yujia Li and Tayfun Terzi and Vladimir Mikulik and Igor Babuschkin and Aidan Clark and Diego de Las Casas and Aurelia Guy and Chris Jones and James Bradbury and Matthew Johnson and Blake Hechtman and Laura Weidinger and Iason Gabriel and William Isaac and Ed Lockhart and Simon Osindero and Laura Rimell and Chris Dyer and Oriol Vinyals and Kareem Ayoub and Jeff Stanway and Lorrayne Bennett and Demis Hassabis and Koray Kavukcuoglu and Geoffrey Irving},
+      year={2022},
+      eprint={2112.11446},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{lee2022deduplicating,
+      title={Deduplicating Training Data Makes Language Models Better},
+      author={Katherine Lee and Daphne Ippolito and Andrew Nystrom and Chiyuan Zhang and Douglas Eck and Chris Callison-Burch and Nicholas Carlini},
+      year={2022},
+      eprint={2107.06499},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{carlini2023quantifying,
+      title={Quantifying Memorization Across Neural Language Models},
+      author={Nicholas Carlini and Daphne Ippolito and Matthew Jagielski and Katherine Lee and Florian Tramer and Chiyuan Zhang},
+      year={2023},
+      eprint={2202.07646},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{raffel2023exploring,
+      title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
+      author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
+      year={2023},
+      eprint={1910.10683},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@misc{touvron2023llama,
+      title={LLaMA: Open and Efficient Foundation Language Models},
+      author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample},
+      year={2023},
+      eprint={2302.13971},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@article{dolma,
+  title = {Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research},
+  author={
+    Luca Soldaini and Rodney Kinney and Akshita Bhagia and Dustin Schwenk and David Atkinson and
+    Russell Authur and Ben Bogin and Khyathi Chandu and Jennifer Dumas and Yanai Elazar and
+    Valentin Hofmann and Ananya Harsh Jha and Sachin Kumar and Li Lucy and Xinxi Lyu and
+    Nathan Lambert and Ian Magnusson and Jacob Morrison and Niklas Muennighoff and Aakanksha Naik and
+    Crystal Nam and Matthew E. Peters and Abhilasha Ravichander and Kyle Richardson and Zejiang Shen and
+    Emma Strubell and Nishant Subramani and Oyvind Tafjord and Pete Walsh and Luke Zettlemoyer and
+    Noah A. Smith and Hannaneh Hajishirzi and Iz Beltagy and Dirk Groeneveld and Jesse Dodge and Kyle Lo
+  },
+  year = {2024},
+  journal={arXiv preprint},
+}
+@article{gao2020pile,
+  title={The {P}ile: An 800{GB} dataset of diverse text for language modeling},
+  author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and others},
+  journal={arXiv preprint arXiv:2101.00027},
+  year={2020}
+}
+@misc{cerebras2023slimpajama,
+  author = {Soboleva, Daria and Al-Khateeb, Faisal and Myers, Robert and Steeves, Jacob R and Hestness, Joel and Dey, Nolan},
+  title = {SlimPajama: A 627B token cleaned and deduplicated version of RedPajama},
+  month = {June},
+  year = 2023,
+  url = {https://huggingface.co/datasets/cerebras/SlimPajama-627B},
+}
+@software{together2023redpajama,
+  author = {Together Computer},
+  title = {RedPajama: an Open Dataset for Training Large Language Models},
+  month = {October},
+  year = 2023,
+  url = {https://github.com/togethercomputer/RedPajama-Data}
+}
+@article{jaccard1912distribution,
+  title={The distribution of the flora in the alpine zone. 1},
+  author={Jaccard, Paul},
+  journal={New phytologist},
+  volume={11},
+  number={2},
+  pages={37--50},
+  year={1912},
+  publisher={Wiley Online Library}
+}
+@misc{albalak2024survey,
+      title={A Survey on Data Selection for Language Models},
+      author={Alon Albalak and Yanai Elazar and Sang Michael Xie and Shayne Longpre and Nathan Lambert and Xinyi Wang and Niklas Muennighoff and Bairu Hou and Liangming Pan and Haewon Jeong and Colin Raffel and Shiyu Chang and Tatsunori Hashimoto and William Yang Wang},
+      year={2024},
+      eprint={2402.16827},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{longpre2023pretrainers,
+      title={A Pretrainer's Guide to Training Data: Measuring the Effects of Data Age, Domain Coverage, Quality, & Toxicity},
+      author={Shayne Longpre and Gregory Yauney and Emily Reif and Katherine Lee and Adam Roberts and Barret Zoph and Denny Zhou and Jason Wei and Kevin Robinson and David Mimno and Daphne Ippolito},
+      year={2023},
+      eprint={2305.13169},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{wenzek2019ccnet,
+      title={CCNet: Extracting High Quality Monolingual Datasets from Web Crawl Data},
+      author={Guillaume Wenzek and Marie-Anne Lachaux and Alexis Conneau and Vishrav Chaudhary and Francisco Guzmán and Armand Joulin and Edouard Grave},
+      year={2019},
+      eprint={1911.00359},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{soldaini2024dolma,
+      title={Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research},
+      author={Luca Soldaini and Rodney Kinney and Akshita Bhagia and Dustin Schwenk and David Atkinson and Russell Authur and Ben Bogin and Khyathi Chandu and Jennifer Dumas and Yanai Elazar and Valentin Hofmann and Ananya Harsh Jha and Sachin Kumar and Li Lucy and Xinxi Lyu and Nathan Lambert and Ian Magnusson and Jacob Morrison and Niklas Muennighoff and Aakanksha Naik and Crystal Nam and Matthew E. Peters and Abhilasha Ravichander and Kyle Richardson and Zejiang Shen and Emma Strubell and Nishant Subramani and Oyvind Tafjord and Pete Walsh and Luke Zettlemoyer and Noah A. Smith and Hannaneh Hajishirzi and Iz Beltagy and Dirk Groeneveld and Jesse Dodge and Kyle Lo},
+      year={2024},
+      eprint={2402.00159},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{ouyang2022training,
+      title={Training language models to follow instructions with human feedback},
+      author={Long Ouyang and Jeff Wu and Xu Jiang and Diogo Almeida and Carroll L. Wainwright and Pamela Mishkin and Chong Zhang and Sandhini Agarwal and Katarina Slama and Alex Ray and John Schulman and Jacob Hilton and Fraser Kelton and Luke Miller and Maddie Simens and Amanda Askell and Peter Welinder and Paul Christiano and Jan Leike and Ryan Lowe},
+      year={2022},
+      eprint={2203.02155},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hoffmann2022training,
+      title={Training Compute-Optimal Large Language Models},
+      author={Jordan Hoffmann and Sebastian Borgeaud and Arthur Mensch and Elena Buchatskaya and Trevor Cai and Eliza Rutherford and Diego de Las Casas and Lisa Anne Hendricks and Johannes Welbl and Aidan Clark and Tom Hennigan and Eric Noland and Katie Millican and George van den Driessche and Bogdan Damoc and Aurelia Guy and Simon Osindero and Karen Simonyan and Erich Elsen and Jack W. Rae and Oriol Vinyals and Laurent Sifre},
+      year={2022},
+      eprint={2203.15556},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{muennighoff2023scaling,
+      title={Scaling Data-Constrained Language Models},
+      author={Niklas Muennighoff and Alexander M. Rush and Boaz Barak and Teven Le Scao and Aleksandra Piktus and Nouamane Tazi and Sampo Pyysalo and Thomas Wolf and Colin Raffel},
+      year={2023},
+      eprint={2305.16264},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{hernandez2022scaling,
+      title={Scaling Laws and Interpretability of Learning from Repeated Data},
+      author={Danny Hernandez and Tom Brown and Tom Conerly and Nova DasSarma and Dawn Drain and Sheer El-Showk and Nelson Elhage and Zac Hatfield-Dodds and Tom Henighan and Tristan Hume and Scott Johnston and Ben Mann and Chris Olah and Catherine Olsson and Dario Amodei and Nicholas Joseph and Jared Kaplan and Sam McCandlish},
+      year={2022},
+      eprint={2205.10487},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@article{llama3modelcard,
+
+title={Llama 3 Model Card},
+
+author={AI@Meta},
+
+year={2024},
+
+url = {https://github.com/meta-llama/llama3/blob/main/MODEL_CARD.md}
+
+}
+@misc{jiang2024mixtral,
+      title={Mixtral of Experts},
+      author={Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and Arthur Mensch and Blanche Savary and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Emma Bou Hanna and Florian Bressand and Gianna Lengyel and Guillaume Bour and Guillaume Lample and Lélio Renard Lavaud and Lucile Saulnier and Marie-Anne Lachaux and Pierre Stock and Sandeep Subramanian and Sophia Yang and Szymon Antoniak and Teven Le Scao and Théophile Gervet and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
+      year={2024},
+      eprint={2401.04088},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+@article{yuan2024self,
+  title={Self-rewarding language models},
+  author={Yuan, Weizhe and Pang, Richard Yuanzhe and Cho, Kyunghyun and Sukhbaatar, Sainbayar and Xu, Jing and Weston, Jason},
+  journal={arXiv preprint arXiv:2401.10020},
+  year={2024}
+}
+@article{verga2024replacing,
+  title={Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models},
+  author={Verga, Pat and Hofstatter, Sebastian and Althammer, Sophia and Su, Yixuan and Piktus, Aleksandra and Arkhangorodsky, Arkady and Xu, Minjie and White, Naomi and Lewis, Patrick},
+  journal={arXiv preprint arXiv:2404.18796},
+  year={2024}
+}
+@article{abdin2024phi,
+  title={Phi-3 technical report: A highly capable language model locally on your phone},
+  author={Abdin, Marah and Jacobs, Sam Ade and Awan, Ammar Ahmad and Aneja, Jyoti and Awadallah, Ahmed and Awadalla, Hany and Bach, Nguyen and Bahree, Amit and Bakhtiari, Arash and Behl, Harkirat and others},
+  journal={arXiv preprint arXiv:2404.14219},
+  year={2024}
+}
+@misc{meta2024responsible,
+  title = {Our responsible approach to Meta AI and Meta Llama 3},
+  author = {Meta},
+  year = {2024},
+  url = {https://ai.meta.com/blog/meta-llama-3-meta-ai-responsibility/},
+  note = {Accessed: 2024-05-31}
+}
+@inproceedings{talmor-etal-2019-commonsenseqa,
+    title = "CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge",
+    author = "Talmor, Alon  and
+      Herzig, Jonathan  and
+      Lourie, Nicholas  and
+      Berant, Jonathan",
+    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
+    month = jun,
+    year = "2019",
+    address = "Minneapolis, Minnesota",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/N19-1421",
+    doi = "10.18653/v1/N19-1421",
+    pages = "4149--4158",
+    archivePrefix = "arXiv",
+    eprint        = "1811.00937",
+    primaryClass  = "cs",
+}
+@inproceedings{zellers-etal-2019-hellaswag,
+    title = "HellaSwag: Can a Machine Really Finish Your Sentence?",
+    author = "Zellers, Rowan  and
+      Holtzman, Ari  and
+      Bisk, Yonatan  and
+      Farhadi, Ali  and
+      Choi, Yejin",
+    editor = "Korhonen, Anna  and
+      Traum, David  and
+      M{\`a}rquez, Llu{\'\i}s",
+    booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics",
+    month = jul,
+    year = "2019",
+    address = "Florence, Italy",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/P19-1472",
+    doi = "10.18653/v1/P19-1472",
+    pages = "4791--4800",
+    abstract = "Recent work by Zellers et al. (2018) introduced a new task of commonsense natural language inference: given an event description such as {``}A woman sits at a piano,{''} a machine must select the most likely followup: {``}She sets her fingers on the keys.{''} With the introduction of BERT, near human-level performance was reached. Does this mean that machines can perform human level commonsense inference? In this paper, we show that commonsense inference still proves difficult for even state-of-the-art models, by presenting HellaSwag, a new challenge dataset. Though its questions are trivial for humans ({\textgreater}95{\%} accuracy), state-of-the-art models struggle ({\textless}48{\%}). We achieve this via Adversarial Filtering (AF), a data collection paradigm wherein a series of discriminators iteratively select an adversarial set of machine-generated wrong answers. AF proves to be surprisingly robust. The key insight is to scale up the length and complexity of the dataset examples towards a critical {`}Goldilocks{'} zone wherein generated text is ridiculous to humans, yet often misclassified by state-of-the-art models. Our construction of HellaSwag, and its resulting difficulty, sheds light on the inner workings of deep pretrained models. More broadly, it suggests a new path forward for NLP research, in which benchmarks co-evolve with the evolving state-of-the-art in an adversarial way, so as to present ever-harder challenges.",
+}
+@inproceedings{OpenBookQA2018,
+ title={Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering},
+ author={Todor Mihaylov and Peter Clark and Tushar Khot and Ashish Sabharwal},
+ booktitle={EMNLP},
+ year={2018}
+}
+@misc{bisk2019piqa,
+      title={PIQA: Reasoning about Physical Commonsense in Natural Language},
+      author={Yonatan Bisk and Rowan Zellers and Ronan Le Bras and Jianfeng Gao and Yejin Choi},
+      year={2019},
+      eprint={1911.11641},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{sap2019socialiqa,
+      title={SocialIQA: Commonsense Reasoning about Social Interactions},
+      author={Maarten Sap and Hannah Rashkin and Derek Chen and Ronan LeBras and Yejin Choi},
+      year={2019},
+      eprint={1904.09728},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{sakaguchi2019winogrande,
+      title={WinoGrande: An Adversarial Winograd Schema Challenge at Scale},
+      author={Keisuke Sakaguchi and Ronan Le Bras and Chandra Bhagavatula and Yejin Choi},
+      year={2019},
+      eprint={1907.10641},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+@misc{clark2018think,
+      title={Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
+      author={Peter Clark and Isaac Cowhey and Oren Etzioni and Tushar Khot and Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+      year={2018},
+      eprint={1803.05457},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+@misc{hendrycks2021measuring,
+      title={Measuring Massive Multitask Language Understanding},
+      author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+      year={2021},
+      eprint={2009.03300},
+      archivePrefix={arXiv},
+      primaryClass={cs.CY}
+}
+@misc{mitchell2023measuring,
+      title={Measuring Data},
+      author={Margaret Mitchell and Alexandra Sasha Luccioni and Nathan Lambert and Marissa Gerchick and Angelina McMillan-Major and Ezinwanne Ozoani and Nazneen Rajani and Tristan Thrush and Yacine Jernite and Douwe Kiela},
+      year={2023},
+      eprint={2212.05129},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}

app/src/colors.mjs

@@ -0,0 +1,8 @@
+export const COLORS = [
+  "#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
+  "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"
+];
+
+export function getColor(index) {
+  return COLORS[index % COLORS.length];
+}

app/src/fine_tasks.js

@@ -0,0 +1,245 @@
+import Papa from 'papaparse';
+import { DataTable } from 'simple-datatables';
+
+const languageMap = {
+  'Arabic': 'ar',
+  'Turkish': 'tr',
+  'Swahili': 'sw',
+  'Russian': 'ru',
+  'Telugu': 'te',
+  'Thai': 'th',
+  'Chinese': 'zh',
+  'French': 'fr',
+  'Hindi': 'hi',
+};
+
+const metricTypes = [
+  { value: 'max_score', label: 'Max Score' },
+  { value: 'avg_snr', label: 'Low Noise' },
+  { value: 'avg_spearman', label: 'Monotonicity' },
+  { value: 'max_n_std', label: 'Non-Randomness' },
+  { value: 'avg_kendall_tau_a', label: 'Ordering Consistency' }
+];
+
+const tableTypes = [
+  { value: 'gen', label: 'Generative' },
+  { value: 'mc', label: 'Multichoice' }
+];
+
+const taskFolders = [
+  { value: 'selected', label: 'FineTasks' },
+  { value: 'non_selected', label: 'Non-Selected' }
+];
+
+function createDropdown(options, onChange) {
+  const select = document.createElement('select');
+  options.forEach(option => {
+    const optionElement = document.createElement('option');
+    if (typeof option === 'object' && option.value && option.label) {
+      optionElement.value = option.value;
+      optionElement.textContent = option.label;
+    } else {
+      optionElement.value = option;
+      optionElement.textContent = option;
+    }
+    select.appendChild(optionElement);
+  });
+  select.addEventListener('change', onChange);
+  return select;
+}
+
+function createPerTaskResultsTable(data, tableType, metric) {
+  const tableWrapper = document.createElement('div');
+  tableWrapper.className = 'table-wrapper fine-tasks-table-wrapper';
+
+  const table = document.createElement('table');
+  table.className = 'results-table fine-tasks-results-table';
+
+  const columns = ['Task', 'Type', ...(tableType === 'gen' ? ['f1', 'prefix_match'] : ['acc', 'acc_norm', 'acc_norm_token', 'acc_norm_pmi'])];
+  
+  const columnNameMap = {
+    // 'Task': 'Task',
+    // 'Type': 'Type',
+    // 'f1': 'f1',
+    // 'prefix_match': 'prefix_match',
+    // 'acc': 'acc',
+    'acc_norm': 'acc_char',
+    'acc_norm_token': 'acc_token',
+    'acc_norm_pmi': 'acc_pmi',
+    'prefix_match': 'prefix'
+  };
+
+  const taskMetricMap = {
+    'max_score': 'score',
+    'avg_snr': 'snr',
+    'avg_spearman': 'monotonicity',
+    'max_n_std': 'non-randomness',
+    'avg_kendall_tau_a': 'ordering'
+    // 'avg_spearman': 'monotonicity',
+  }
+
+  const header = table.createTHead();
+  const headerRow = header.insertRow();
+  columns.forEach(column => {
+    const th = document.createElement('th');
+    th.textContent = columnNameMap[column] || column;
+
+    if (th.textContent !== "Task" && th.textContent !== "Type") {
+        th.textContent += " " + (taskMetricMap[metric] || metric);
+    }
+    th.title = th.textContent;
+    if (column === 'Type')
+      th.style.width = '40px';
+    headerRow.appendChild(th);
+  });
+
+  const body = table.createTBody();
+  data.forEach(row => {
+    if (Object.values(row).every(value => value === '' || value === undefined || value === null)) {
+      return;
+    }
+    
+    const tr = body.insertRow();
+    columns.forEach(column => {
+      const td = tr.insertCell();
+      let value = row[column];
+      if (column === 'Task') {
+        const fullTaskName = value; // Store the full task name
+        const parts = value.split('|');
+        value = parts.length > 1 ? parts[1] : value;
+        value = value.split('_mcf')[0].split('_cf')[0];
+        td.title = fullTaskName; // Set the title attribute to show the full name on hover
+      } else if (column === 'Type') {
+        // Keep the task type as is
+      } else if (typeof value === 'number') {
+        value = value.toFixed(2);
+      } else if (value && !isNaN(parseFloat(value))) {
+        value = parseFloat(value).toFixed(2);
+      } else {
+        value = '';
+      }
+      td.textContent = value;
+    });
+  });
+
+  tableWrapper.appendChild(table);
+  return tableWrapper;
+}
+
+export function initFineTasks(containerId) {
+  const container = document.getElementById(containerId);
+  if (!container) return;
+
+  const perTaskTitleElement = document.createElement('h3');
+  perTaskTitleElement.textContent = 'Task Results';
+  perTaskTitleElement.className = 'fine-tasks-title';
+
+  const perTaskTableContainer = document.createElement('div');
+  perTaskTableContainer.className = 'table-container';
+
+  let perTaskDataTable;
+
+  function updatePerTaskResults() {
+    const language = languageDropdownPerTask.value;
+    const metric = metricDropdownPerTask.value;
+    const tableType = tableTypeDropdownPerTask.value;
+    const taskFolder = taskFolderDropdownPerTask.value;
+
+    const languageCode = languageMap[language];
+
+    if (!languageCode) {
+      console.error(`Language code not found for ${language}`);
+      perTaskTableContainer.innerHTML = `<p>Error: Language code not found for ${language}</p>`;
+      return;
+    }
+
+    let url = `data/tasks/${taskFolder}/${languageCode}/${metric}/${tableType}_stats.csv`;
+
+    fetch(url)
+      .then(response => {
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        return response.text();
+      })
+      .then(csvText => {
+        const results = Papa.parse(csvText, { header: true }).data;
+        perTaskTableContainer.innerHTML = '';
+        const tableWrapper = createPerTaskResultsTable(results, tableType, metric);
+        perTaskTableContainer.appendChild(tableWrapper);
+
+        if (perTaskDataTable) {
+          perTaskDataTable.destroy();
+        }
+
+        perTaskDataTable = new DataTable('.fine-tasks-results-table', {
+          perPage: 10,
+          perPageSelect: false,
+          searchable: false,
+          sortable: true,
+          fixedHeight: true,
+          labels: {
+            info: ''  // This removes the "Showing 1 to X of Y entries" text
+          }
+        });
+
+      })
+      .catch(error => {
+        console.error('Error fetching CSV:', error);
+        perTaskTableContainer.innerHTML = `<p>Error loading data: ${error.message}</p>`;
+      });
+  }
+
+  const perTaskControls = document.createElement('div');
+  perTaskControls.className = 'controls fine-tasks-controls';
+
+  // Task folder control group
+  const taskFolderControlGroup = document.createElement('div');
+  taskFolderControlGroup.className = 'control-group';
+  const taskFolderLabelPerTask = document.createElement('label');
+  taskFolderLabelPerTask.textContent = 'Task Set: ';
+  const taskFolderDropdownPerTask = createDropdown(taskFolders, updatePerTaskResults);
+  taskFolderDropdownPerTask.value = 'selected'; // Set default to FineTasks
+  taskFolderControlGroup.appendChild(taskFolderLabelPerTask);
+  taskFolderControlGroup.appendChild(taskFolderDropdownPerTask);
+
+  // Language control group
+  const languageControlGroup = document.createElement('div');
+  languageControlGroup.className = 'control-group';
+  const languageLabelPerTask = document.createElement('label');
+  languageLabelPerTask.textContent = 'Language: ';
+  const languageDropdownPerTask = createDropdown(Object.keys(languageMap), updatePerTaskResults);
+  languageControlGroup.appendChild(languageLabelPerTask);
+  languageControlGroup.appendChild(languageDropdownPerTask);
+
+  // Table type control group
+  const tableTypeControlGroup = document.createElement('div');
+  tableTypeControlGroup.className = 'control-group';
+  const tableTypeLabelPerTask = document.createElement('label');
+  tableTypeLabelPerTask.textContent = 'Type: ';
+  const tableTypeDropdownPerTask = createDropdown(tableTypes, updatePerTaskResults);
+  tableTypeControlGroup.appendChild(tableTypeLabelPerTask);
+  tableTypeControlGroup.appendChild(tableTypeDropdownPerTask);
+
+  // Metric control group
+  const metricControlGroup = document.createElement('div');
+  metricControlGroup.className = 'control-group';
+  const metricLabelPerTask = document.createElement('label');
+  metricLabelPerTask.textContent = 'Criteria: ';
+  const metricDropdownPerTask = createDropdown(metricTypes, updatePerTaskResults);
+  metricDropdownPerTask.value = 'max_score'; // Set default to Max Score
+  metricControlGroup.appendChild(metricLabelPerTask);
+  metricControlGroup.appendChild(metricDropdownPerTask);
+
+  perTaskControls.appendChild(taskFolderControlGroup);
+  perTaskControls.appendChild(languageControlGroup);
+  perTaskControls.appendChild(tableTypeControlGroup);
+  perTaskControls.appendChild(metricControlGroup);
+
+  container.appendChild(perTaskControls);
+  // container.appendChild(perTaskTitleElement);
+  container.appendChild(perTaskTableContainer);
+
+  // Initialize with default values
+  updatePerTaskResults();
+}

app/src/index.html

@@ -0,0 +1,388 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="distill.bundle.js" type="module" fetchpriority="high" blocking></script>
+    <script src="main.bundle.js" type="module" fetchpriority="low" defer></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <meta charset="utf8">
+    <base target="_blank">
+    <title>Scaling FineWeb to 1000+ languages: Step 1: finding signal in 100s of evaluation tasks</title>
+    <link rel="stylesheet" href="style.css">
+</head>
+
+<body>
+<d-front-matter>
+    <script id='distill-front-matter' type="text/json">{
+    "title": "📝 Scaling FineWeb to 1000+ languages: Step 1: finding signal in 100s of evaluation tasks",
+    "description": "This blog covers a discussion on multilingual evaluation and task signal, the processes for selecting existing evaluation tasks based on signal resulting in FineTasks, and comparisson of open and closed sourced on the FineTasks.",
+    "published": "Oct 23, 2024",
+    "affiliation": {"name": "HuggingFace"},
+    "authors": [
+      {
+        "author":"Hynek Kydlíček",
+        "authorURL":"https://huggingface.co/hynky"
+      },
+      {
+        "author":"Guilherme Penedo",
+        "authorURL":"https://huggingface.co/guipenedo"
+      },
+      {
+        "author":"Clémentine Fourier",
+        "authorURL":"https://huggingface.co/clefourrier"
+      },
+      {
+        "author":"Nathan Habib",
+        "authorURL":"https://huggingface.co/SaylorTwift"
+      },
+      {
+        "author":"Thomas Wolf",
+        "authorURL":"https://huggingface.co/thomwolf"
+      }
+    ]
+    }</script>
+</d-front-matter>
+
+<d-title>
+    <h1 class="l-page" style="text-align: center; display: none;">📝 Scaling FineWeb to 1000+ languages: Step 1: finding signal in 100s of evaluation tasks</h1>
+    <div id="title-plot" class="main-plot-container l-page">
+        <figure>
+            <img src="assets/images/banner.png" alt="FineTasks">
+        </figure>
+    </div>
+</d-title>
+<d-byline></d-byline>
+<d-article>
+    <d-contents>
+    </d-contents>
+
+    <p>Following the strong community reception of our FineWeb English dataset<d-cite key="penedo2024finewebdatasetsdecantingweb"></d-cite>, we have been hard at work on a <b>multilingual version</b>, which will cover 1000+ languages (that we hope to release <em>soon</em>!).</p>
+
+    <p>However, we quickly encountered a significant challenge: how can one effectively evaluate models across different languages during training?</p>
+
+    <p>For English, it's straightforward: we can utilize well-established benchmarks like <b>MMLU</b><d-cite key="hendryckstest2021"></d-cite> or <b>HellaSwag</b><d-cite key="zellers2019hellaswag"></d-cite>, widely used by most labs and implemented in all the major evaluation frameworks. Unfortunately, non-English tasks are often scarce and lack broader community validation and, when available, are frequently of questionable quality: many are machine-translated and may even include English words in their formulations. Additionally, they are often unsuitable for early pre-training evaluation due to suboptimal task formulations and/or too high difficulty resulting in random scores.</p>
+
+    <p>To address these challenges, we developed a <b>scalable and data-driven framework</b> for evaluation task selection, which allows anyone to choose strong model evaluations for their language from existing tasks! We then applied this framework to a set of <b>9 diverse languages</b>, resulting in the creation of <b>FineTasks</b> - a comprehensive and diverse multilingual evaluation suite.</p>
+
+    <p>In this blog post, we discuss:</p>
+    <ol>
+        <li>Our <b>data-driven process</b> to create a multilingual evaluation suite: <b>FineTasks</b></li>
+        <li>Results of evaluating <b>35 major open and closed-source models</b> on FineTasks</li>
+        <li>A guide for extending FineTasks to your <b>target language</b></li>
+    </ol>
+
+
+    <h2>What Makes a Task "Fine"?</h2>
+    <p>Covering all 7000+ languages spoken over the world would be monumental endeavor, so we settled on using <b>9 languages</b> that offered diversity in script, language family and resource availability: <b>Chinese, French, Arabic, Russian, Thai, Hindi, Turkish, Swahili, and Telugu</b>.</p>
+            <p>For these languages, we collected all available tasks that we could find, implementing a total of <b>185 tasks across languages</b> in <a href="https://github.com/huggingface/lighteval">LightEval</a>, HuggingFace's model evaluation library.</p>
+
+    <p>Then, we began task selection with two primary goals: ensuring <b>evaluation diversity</b>, and making sure each task provided a <b>reliable signal</b> during pre-training.</p>
+
+    <p>For evaluation diversity, we aimed to assess a broad range of model capabilities, including:</p>
+    <ul>
+        <li><b>Reading comprehension (RC)</b>: Understanding provided context and answering questions based on it.</li>
+        <li><b>General knowledge (GK)</b>: Answering questions about facts from various fields without added context.</li>
+        <li><b>Natural Language Understanding (NLU)</b>: Comprehending the semantics of provided input.</li>
+        <li><b>Common-sense reasoning (RES)</b>: Demonstrating the ability to perform simple reasoning requiring embodied knowledge.</li>
+        <li><b>Generative tasks</b>: Ability to generate text in the target language without the "help" of multiple choice options.</li>
+    </ul>
+
+    <p>We consider that tasks provide a reliable signal if they provide a dependable score. This means the score should be above the random baseline, increase as training progresses, show low variability across different seeds, and provide consistent model ranking at each training step<d-footnote>For similar sized models trained with the same hyperparameters on the same amount of data.</d-footnote>.</p>
+
+    <h3>Finding how much signal our tasks give during pre-training</h3>
+    <p>To thoroughly examine the signal our tasks provide, we trained many 1.5B parameter models for each language, using 30B tokens from subsets of the supported languages of the five largest openly available multilingual web datasets. These models were trained with the same hyperparameters and tokenizer. We then evaluated them at regular checkpoint intervals on the collected tasks (with no instruction and no system prompt in a 0-shot setting).</p>
+
+    <p>This process required multiple evaluation runs for each task due to iterations on its implementation, resulting in a total of <b>73 000 GPU hours consumed</b> 🔥!</p>
+
+    <p>With <b>49 models trained</b> we could finally define what a <b>reliable signal</b> means to us!</p>
+
+    <h4>Monotonicity</h4>
+    <p>One of our core requirements for a task is that it can be learned from training data and this <b>learning can be gradually observed as the training progresses</b>. Without this improvement through time, it's uncertain whether there will ever be an improvement in the future.</p>
+
+    <p>To measure this, we used the <b>Spearman rank correlation</b> to quantify the correlation between steps and score. Spearman rank correlation can capture monotonicity even when scores don't evolve linearly with the number of steps. We required each task to have at least an average correlation of 0.5 over all model training runs.</p>
+
+
+    <div style="display: flex; grid-column: middle">
+        <div class="task-signal-plot" data-language="French" data-task="mlmm_hellaswag_fra_cf" data-show-controls="false" data-task-metrics="monotonicity" data-metric="acc_norm_token" data-group-seeds="true" data-title="✅ Good monotonicity: mlmm_hellaswag_fra_cf [fr]"></div>
+        <div class="task-signal-plot" data-language="Arabic" data-task="mlmm_truthfulqa_ara_cf:mc1" data-show-controls="false" data-task-metrics="monotonicity" data-metric="acc_norm_token" data-group-seeds="true" data-title="❌ Bad monotonicity: mlmm_truthfulqa_ara_cf:mc1 [ar]"></div>
+    </div>
+
+    <h4>Low noise</h4>
+    <p>When comparing model performance on tasks, we need to consider whether differences are due to <b>evaluation noise or genuine performance variations</b>.</p>
+
+    <p>Noise can arise from the stochastic processes involved in model training, such as random token sampling, data shuffling, or model initialization.<d-cite key="madaan2024quantifyingvarianceevaluationbenchmarks"></d-cite> To measure how sensitive each task is to this noise, we trained four additional models on our own monolingual corpora (unfiltered CommonCrawl data in each language) using different seeds.</p>
+
+    <p>For each task, we computed:</p>
+    <ol>
+        <li>First, a <b></b>standard deviation of model scores for every step (approximately every 1B tokens), which we call the <b>per-step-std</b>.</li>
+        <li>Then, to obtain a global variability measurement, we averaged all the per-step-std values to get the <b>avg-std</b> over the full training. We assume this value is an upper-bound across model architectures and training datasets (as it was approximated by models trained on a "dirtier" dataset, therefore with higher variability).</li>
+        <li>Finally, we computed the <b>signal-to-noise ratio</b> (SNR) as the main metric for task variability. We calculate SNR as the mean score at 30B tokens of all runs divided by the avg-std. This metric measures how significant the overall score is relative to the score variations (noise).</li>
+    </ol>
+
+    <p>We aimed for each task to have an SNR > 20. The only exception to this rule are generative tasks, which typically have relatively low SNR, but are still worth including as they provide insights into how the model behaves when prompted to generate unconstrained (without answer options). In a multilingual setting, this is particularly relevant as some models trained on multiple languages can exhibit high task scores but then suddenly reply in the wrong language for generative tasks!</p>
+
+    <div style="display: flex; grid-column: middle">
+        <div class="task-signal-plot" data-language="Telugu" data-task="xstory_cloze_tel_cf" data-show-controls="false" data-task-metrics="snr" data-metric="acc_norm_token" data-group-seeds="false" data-title="✅ Good SNR: xstory_cloze_tel_cf [te]"></div>
+        <div class="task-signal-plot" data-language="Telugu" data-task="tydiqa_tel" data-show-controls="false" data-task-metrics="snr" data-metric="acc_norm_token" data-group-seeds="false" data-title="❌ Bad SNR: tydiqa_tel [te]"></div>
+    </div>
+
+    <aside>Assuming model performance is normally distributed across different seeds, we want the benchmark-run performance to be at least 3 final-stds above the benchmark random baseline. This would mean that 99.85% of seed scores are above the random baseline (formally, benchmark-run performance - benchmark random baseline > 3 * final-std).</aside>
+    <h4>Non-Random Performance</h4>
+    <p>Many model capabilities are acquired later in training, thus <b>many tasks</b> (especially harder ones, such as math-related ones) <b>show baseline-level performance for an extended period</b>. While these tasks are useful, they're not ideal for early pre-training evaluation, and <b>we did not want to keep them</b> for this setting.</p>
+
+    <p>We first computed the baseline random performance of the task (as the sum of 1/n_choices for all samples for multiple choice questions, and as zero for generative evaluations). Then we calculated the task's distance from the baseline as the maximum score across all models minus the baseline.</p>
+
+
+    <div style="display: flex; grid-column: middle">
+        <div class="task-signal-plot" data-language="Chinese" data-task="agieval_zho_cf:_average" data-show-controls="false" data-task-metrics="randomness" data-metric="acc_norm_pmi" data-group-seeds="true" data-title="✅ Non-random: agieval_zho_cf/acc_pmi [zh]"></div>
+        <div class="task-signal-plot" data-language="Chinese" data-task="agieval_zho_cf:_average" data-show-controls="false" data-task-metrics="randomness" data-metric="acc" data-group-seeds="true" data-title="❌ Random perf: agieval_zho_cf/acc [zh]"></div>
+    </div>
+
+    <h4>Model Ordering Consistency</h4>
+    <p>Let's not forget that the main goal of these evaluations is to compare models and datasets!</p>
+
+    <p>In the future, we want to use these evaluations to select the best datasets for full model pretraining. This means <b>our tasks should rank datasets trained using very few tokens (we typically run data ablations on 30B tokens), in the same order as they would when trained for longer, after significantly more steps.</b></p>
+
+    <p>In other words, we would like tasks to have <b>predictive capability regarding future performance during pre-training</b>: if pre-training dataset A outperforms pre-training dataset B at 30 billion tokens, we would like this trend to continue at 300 billion tokens.</p>
+
+    <p>Proving this is inherently impossible, but there is a necessary preliminary condition that we can test for: for the results to be consistent at large scales, they must also first show consistency at smaller scales!</p>
+
+    <p>To measure this consistency in task ordering, we computed the average <b>Kendall's Tau</b> of models ranking between every two consecutive steps. We only considered steps starting after 15B tokens of pre-training, as we found orderings before the range incredibly noisy. A high value of this metric indicates that the ordering remains consistent as training progresses.</p>
+
+    <aside>We had no strict minimum value requirement for this property, instead using it to establish comparisons between tasks.</aside>
+
+    <div style="display: flex; grid-column: middle">
+        <div class="task-signal-plot" data-language="Arabic" data-task="xcsqa_ara_cf" data-show-controls="false" data-task-metrics="ordering" data-metric="acc_norm_token" data-group-seeds="true" data-title="✅ Good ordering: xcsqa_ara_cf [ar]"></div>
+        <div class="task-signal-plot" data-language="Thai" data-task="thai_exams_tha_cf:_average" data-show-controls="false" data-task-metrics="ordering" data-metric="acc_norm_token" data-group-seeds="true" data-title="❌ Bad ordering: thai_exams_tha_cf [th]"></div>
+    </div>
+
+
+    <h2>Important properties of evaluation impacting stability</h2>
+    <p>Now that we covered what we were looking for in our tasks, let's examine two important aspects that can affect the above properties: task formulations and metric choice.</p>
+
+    <aside>Both of these aspects are thoroughly described and studied in the brilliant OLMES paper<d-cite key="gu2024olmesstandardlanguagemodel"></d-cite>, which greatly inspired our work.</aside>
+
+    <h3>Task Formulations</h3>
+    <p>The way tasks are presented to the model is crucial, particularly for multiple-choice (MC) tasks. In these scenarios, we must carefully determine how the choices are displayed and what the model is expected to predict.</p>
+
+    <p>There are two common approaches: <b>Cloze Formulation</b> (CF) and <b>Multi-Choice Formulation</b> (MCF). In CF, choices are not provided in context, allowing the model to predict each option directly. In contrast, MCF presents the choices in the prompt, using A/B/C/D prefixes, with the targets being those letter prefixes.</p>
+
+    <!-- side-by-side comparison of MCF vs. CF on a specific task -->
+
+    <p>It's important to know that:</p>
+    <ul>
+        <li>The choice of formulation significantly impacts task scores <d-cite key="open-llm-leaderboard-v2"></d-cite>.</li>
+        <li>Both formulations <b>behave very differently during training</b>. As noted by both OLMES<d-cite key="gu2024olmesstandardlanguagemodel"></d-cite> and DataComp-LM<d-cite key="li2024datacomplmsearchgenerationtraining"></d-cite>, when employing MCF, task scores initially show random performance over extended training periods before experiencing a sudden increase. Conversely, with CF, task scores improve right from the beginning but tend to plateau relatively early.</li>
+    </ul>
+
+    <p>Therefore, we decided to utilize CF for task selection and MCF for later evaluation of major open source models, as they have generally undergone enough training for these evaluations to have a signal.</p>
+
+    <h3>Metrics</h3>
+    <p>As the targets in CF of multiple choice tasks are choices themselves, each target can have a different number of tokens, characters, and unconditional probability (probability of generating the choice without a context prefix).</p>
+    <aside>Measuring accuracy without normalization would have the models prefer answers with fewer tokens, for example.</aside>
+
+    <p>To account for this, we consider the following accuracy variations:</p>
+    <ul>
+        <li><b>Accuracy</b> : <br>
+            <code>acc</code> = <d-math>\underset{i}{\arg\max}(ln(P (a_i|q)))</d-math></li>
+        <li><b>Accuracy normalized over character length</b> : <br>
+            <code>acc_char</code> = <d-math> \underset{i}{\arg\max}\frac{ln(P (a_i|q))}{num\_characters(a_i)}</d-math></li>
+        <li><b>Accuracy normalized over token length</b> :<br>
+            <code>acc_token</code> = <d-math> \underset{i}{\arg\max}\frac{ln(P (a_i|q))}{num\_tokens(a_i)}</d-math></li>
+        <li><b>PMI Accuracy</b> : <br>
+            <code>acc_pmi</code> = <d-math> \underset{i}{\arg\max}ln\frac{P (a_i|q)}{P (a_i|u)}</d-math>, where <d-math>u =</d-math>''Answer:''</li>
+    </ul>
+
+    <p>Where <d-math>a_i</d-math> is the answer choice <d-math>i</d-math>, <d-math>q</d-math> is a question prompt and <d-math>P (a_i|q)</d-math> is the probability of having <d-math>a_i</d-math> follow <d-math>q</d-math>. For more details see <d-cite key="gu2024olmesstandardlanguagemodel"></d-cite> and <d-cite key="biderman2024lessonstrenchesreproducibleevaluation"></d-cite>.</p>
+
+    <aside><code>acc_pmi</code> metric measures how much more likely a model is to predict A_i if provided with question context compared to if there was no context at all. This can be useful if the correct choice contains generally unlikely tokens, making the model less likely to choose such an answer.</aside>
+
+    <p>For our generative tasks on the other hand, we used the following metrics:</p>
+    <ul>
+        <li><code>prefix_match</code>: Exact match where only the prefix of the answer must match</li>
+        <li><code>f1</code>: F1 score computed over predicted/gold words extracted using a word tokenizer</li>
+    </ul>
+
+    <p>For both generative metrics, minor preprocessing is applied to remove articles and punctuation, and lowercase the text.</p>
+
+    <h2>The Fine selection</h2>
+    <p>With our goals and evaluation setup properly defined, we proceeded with <b>task selection</b>!</p>
+
+    <p>We reviewed tasks one by one, choosing based on the quantified properties. For each language, we aimed to have at least one task for each of the four categories outlined above. Additionally we wanted to have at least 1 generative task for each language.</p>
+
+    <p>In cases where multiple versions of a task existed (e.g., MMLU with different translation methods or native versions), we <b>prioritized native versions</b> as long as their metrics were reasonable, followed by human translations of English tasks. If no such version was available, we made our selection entirely based on metrics.</p>
+
+    <p>Thus, <b>after removing about half of the tasks</b>, we arrived at <b>96 final ones</b>, forming "FineTasks."</p>
+
+    <h3>Explore tasks</h3>
+    <p>Use the dropdowns below to navigate the list of tasks and how different metrics affect them.</p>
+    <div id="fine-tasks-results"></div>
+
+    <p>All tasks from the selection <b>comply with the criteria</b> outlined in previous sections, with the only exception being indicqa_tel, which we chose to include to ensure we had at least one generative task for Telugu. Overall we managed to cover all task categories for each language (the only exception being Thai Reasoning, where all tasks were unfortunately too noisy with low monotonicity to consider them).</p>
+
+    <p>One of the <b>biggest surprises</b> was that some tasks, even when translated using the same method, were <b>reliable in one language but not in others</b>. This was evident with xWinograd, which worked quite well for Russian but did not meet our conditions for French. An even more extreme example was XNLI, which performed well for 6 out of 7 languages, failing to satisfy the reliability properties for Chinese. We had to test four different implementations before finding a reliable version, which, interestingly, was the only one that was created by native speakers and not machine translated.</p>
+
+    <p>Feel free to use the dropdowns below to explore the evolution of scores over training for all tested tasks and metrics.</p>
+    <div class="task-signal-plot" data-language="French" data-task="frenchbench_hellaswag_fra_cf" data-show-controls="true" data-metric="acc_norm_token" data-group-seeds="true" data-title=""></div>
+
+
+    <h3>Metrics recommendation</h3>
+    <p>Selecting the best evaluation metrics proved to be a <b>challenging task</b>. Not only is there no single metric that consistently outperforms the rest, but we often encountered situations where one metric had better monotonicity while another had a higher signal-to-noise ratio. In such cases, we typically made our decision based on the selected metric for tasks' implementation in a different language. We are aware that such hand-picking is often not possible and thus offer the following recommendations:</p>
+
+    <h4>Multichoice Tasks</h4>
+    <ul>
+        <li>We found <b>base accuracy</b> to perform well for tasks with answer options varying subtly (e.g. Yes/No/Also), particularly NLI tasks. In such cases, where the answer options are often each a single token, the base accuracy is advisable to use.</li>
+        <li>While OLMES<d-cite key="gu2024olmesstandardlanguagemodel"></d-cite> recommends using PMI for tasks with unusual words, we found <b>PMI</b> to be highly effective for "difficult" reasoning and knowledge tasks like AGIEVAL or MMLU. In these cases, PMI provided the best results and was often the only metric delivering performance above random. That said, PMI was, on average, the weakest metric across all other tasks, while also being two times more expensive to compute. We therefore only recommend its use for complex reasoning and knowledge tasks.</li>
+        <li>The metrics we found to be <b>most reliable overall</b> were length normalization metrics (token or character-based). However, the best choice was dependent on language, rather than being consistent for a given task. Due to that, we recommend using the maximum of acc_char and acc_token for the most reliable results.<d-footnote>Note that acc_token is heavily tokenizer dependent. On our ablations all models were trained using the same tokenizer.</d-footnote></li>
+    </ul>
+
+    <h4>Generative Tasks</h4>
+    <p>For <b>generative metrics</b>, the choice is clearer: we suggest using the F1 score unless exact matching is required, as in math-related tasks. F1 is generally less noisy and more resilient to small changes in the generations.</p>
+
+
+    <h2>Open/Closed Source models tackle FineTasks</h2>
+    <p>Since we spent a lot of time and compute on task selection, we were interested in how well major <b>open-source</b> models would do on FineTasks. Given that our evaluation suite primarily targets pretrained models, we focused on these, with a few exceptions for models that don't offer a base (pretrained) version. These exceptions were included mainly out of curiosity, and their results should be interpreted with <b>caution</b>. Such models may significantly outperform other models due to the inclusion of supervised fine-tuning (SFT) data.</p>
+
+    <p>To assess the multilingual performance disparity between open-source and closed-source models, we expanded our selection by adding a closed source model: <b>gpt-4o-mini</b>.</p>
+
+    <p>As outlined in the task formulations, we are using MCF for this evaluation and employing a 5-shot approach, as recommended by OLMES<d-cite key="gu2024olmesstandardlanguagemodel"></d-cite> (and made possible by the large context size of the models).</p>
+
+    <h3>Computing a global "multilingual" score</h3>
+    <p>In the previous sections, we treated each task independently. However, to determine an overall "multilingual" score of a model, we need to <b>aggregate</b> the results from these tasks. We begin by <b>rescaling</b> the individual task scores in line with the OpenLLM leaderboard <d-cite key="open-llm-leaderboard-v2"></d-cite>. Then, we <b>average the scores</b> across task types (GK, RES, etc) for each language separately. To compute the score for each language, we take the average of the task type scores.<d-footnote>We first average by task type to properly measure all model capabilities without letting a single category dominate.</d-footnote></p>
+
+    <p>For the final global "multilingual" score we followed a different approach. Instead of averaging the language scores directly, we <b>ranked the model's performance across languages</b> in comparison to other models and then averaged those rank scores. This method ensures that the result reflects the overall model's performance across all languages, preventing an exceptionally high score in one language from skewing the final outcome.</p>
+
+    <h3>FineTasks Results</h3>
+    <p>After spending <b>even more compute</b> 🔥 on evaluating the selected models, we gathered the results in the following table. Here are our insights:</p>
+
+    <div id="leaderboard-results" class="l-middle" data-caption="Chat models are indicated by 💬 while 🟢 indicates a base model.">
+    </div>
+
+    <h4>Qwen family of models takes both top spots!</h4>
+    <p>The Qwen models <b>perform exceptionally well</b>, taking both first and second place with their 72B and 32B versions. Their key strength appears to be in handling high- and mid-resource languages (particularly Chinese), where they consistently ranked first. However, they <b>struggled with lower-resource languages</b>, especially Swahili and Telugu, where their performance lagged.</p>
+
+    <h4>General Knowledge: The curse of monolingual models</h4>
+    <p>The most surprising finding from our evaluation is how models explicitly trained to specialize in a <b>narrow set of languages</b> — like Sarvam-2B-v0.5 for Telugu, or Typhoon-v1.5-8B for Thai — tend to <b>perform exceptionally well on generative tasks</b>, while <b>falling short when it comes to reasoning</b> and general knowledge (GK) tasks, oftentimes getting close to random performance. We hypothesize two explanations: The models haven't undergone extensive enough training to be able to understand the MCF format or the higher exposure to various languages and especially English allows the non-specialized models to perform better at such GK/RES tasks. We note that good generative task performance reveals a good understanding of the target language.</p>
+
+    <p>The only <b>exceptions to this rule</b> are typhoon-v1.5-72b and Yi-1.5-34B, both tackling the RES/GK tasks well and managing to rank in the top 4 for their respective languages. We note that typhoon-v1.5-72b is based on Qwen models, and that Yi also included English in its training data.</p>
+
+    <h4>A lower resource winner: Gemma-2</h4>
+    <p>Although it didn't take first place, Gemma2 performed really well in the multilingual domain, especially <b>considering its size</b>. It showed consistent results across all the languages we tested, <b>excelling in low-resource languages</b> like Telugu and Swahili. For anyone working with low-resource languages, we highly recommend Gemma-2 as a strong option.</p>
+
+    <h4>Is there even a gap between open and closed source models?</h4>
+    <p>As mentioned in the beginning, comparing closed-source models requires extra caution. These models often undergo extensive supervised fine-tuning (SFT), employ highly optimized prompting techniques, and may even generate multiple responses and select the best one. <b>Despite these advantages, the o4-mini ranks only just above the medium-sized 27B Gemma-2.</b> Based on this evidence, <b>we believe that the gap between open-source and closed-source models is very narrow, if not entirely negligible.</b></p>
+
+    <h3>Evaluating on FineTasks</h3>
+    <p>If you would like to evaluate your models on FineTasks and expand the above table we made it easy for you. Simply run the following command with your model of choice:</p>
+
+    <pre><code>lighteval accelerate\
+    --model_args vllm,pretrained=model_name,pairwise_tokenization=True \
+    --custom_task lighteval.tasks.multilingual.tasks \
+    --tasks 'examples/tasks/finetasks/{cf,mcf}/{ara,fra,rus,tur,swa,hin,tel,tha,zho}' \
+    --max_samples '1000'</code></pre>
+
+    <h2>Can we cover all the languages of the world together?</h2>
+    <p>FineTasks is <b>just the beginning</b> of our multilingual journey. As a first step in the creation of the <b>future FineWeb multilingual release</b>, we are using this evaluation setup to curate a high quality pretraining dataset covering a large number of languages. You can expect more news from us soon! We plan to also continue working to make evaluation in non-English domains as seamless as it is in English—and <b>we need your help to achieve that</b>!</p>
+
+    <p>LightEval now supports <b>over 550 tasks</b> across various non-English languages, making it the evaluation framework with the best multilingual coverage available. However, there's still much more to do. For many languages, no tasks exist yet, despite our ongoing work. This is where we believe <b>the strong Hugging Face community can make a difference</b>.</p>
+
+    <p>We've made it  <a href="https://github.com/huggingface/lighteval/wiki/Contributing-to-multilingual-evaluations"><b>incredibly easy</b> to contribute new tasks</a>, by developing a templating system which supports most of the popular task types, while maintaining authenticity of native language use, right down to correct punctuation. Even if you aren't able to contribute full evaluation tasks, you can still help. Many languages currently <b>lack translations</b> for anchor words used in evaluation, leaving hundreds of tasks unusable. You can help fill this gap by adding them following <a href="https://github.com/huggingface/lighteval/wiki/Contributing-to-multilingual-evaluations">our mini guide</a>.</p>
+
+    <p>We're looking forward to revisiting this analysis in the future, not with just 9 languages, but at least 50—thanks to community contributions! Let's level the playing field between English and other languages together! 🤗</p>
+</d-article>
+
+<d-appendix>
+    <d-bibliography src="bibliography.bib"></d-bibliography>
+    <style>
+      d-appendix .citation {
+        font-size: 11px;
+        line-height: 15px;
+        border-left: 1px solid rgba(0, 0, 0, 0.1);
+        padding-left: 18px;
+        border: 1px solid rgba(0,0,0,0.1);
+        background: rgba(0, 0, 0, 0.02);
+        padding: 10px 18px;
+        border-radius: 3px;
+        color: rgba(150, 150, 150, 1);
+        overflow: hidden;
+        margin-top: -12px;
+        white-space: pre-wrap;
+        word-wrap: break-word;
+      }
+    </style>
+
+    <h3 id="citation">Citation</h3>
+    <p>For attribution in academic contexts, please cite this work as</p>
+    <pre class="citation short">Kydlicek, et al., "FineTasks: Finding signal in a haystack of 200+ multilingual tasks", 2024.</pre>
+    <p>BibTeX citation</p>
+    <pre class="citation long">@misc{kydlicek2024finetasksmultilingualtasks,
+      title={FineTasks: Finding signal in a haystack of 200+ multilingual tasks},
+      author={Hynek Kydlíček and Guilherme Penedo and Clémentine Fourier and Nathan Habib and Thomas Wolf},
+      url={https://huggingface.co/spaces/HuggingFaceFW/blogpost-fine-tasks},
+}</pre>
+</d-appendix>
+
+<script>
+    const article = document.querySelector('d-article');
+    const toc = document.querySelector('d-contents');
+    if (toc) {
+        const headings = article.querySelectorAll('h2, h3, h4');
+        let ToC = `<nav role="navigation" class="l-text figcaption"><h3>Table of contents</h3>`;
+        let prevLevel = 0;
+
+        for (const el of headings) {
+            // should element be included in TOC?
+            const isInTitle = el.parentElement.tagName == 'D-TITLE';
+            const isException = el.getAttribute('no-toc');
+            if (isInTitle || isException) continue;
+            el.setAttribute('id', el.textContent.toLowerCase().replaceAll(" ", "_"))
+            const link = '<a target="_self" href="' + '#' + el.getAttribute('id') + '">' + el.textContent + '</a>';
+
+            const level = el.tagName === 'H2' ? 0 : (el.tagName === 'H3' ? 1 : 2);
+            while (prevLevel < level) {
+                ToC += '<ul>'
+                prevLevel++;
+            }
+            while (prevLevel > level) {
+                ToC += '</ul>'
+                prevLevel--;
+            }
+            if (level === 0)
+                ToC += '<div>' + link + '</div>';
+            else
+                ToC += '<li>' + link + '</li>';
+        }
+
+        while (prevLevel > 0) {
+            ToC += '</ul>'
+            prevLevel--;
+        }
+        ToC += '</nav>';
+        toc.innerHTML = ToC;
+        toc.setAttribute('prerendered', 'true');
+        const toc_links = document.querySelectorAll('d-contents > nav a');
+
+        window.addEventListener('scroll', (_event) => {
+            if (typeof (headings) != 'undefined' && headings != null && typeof (toc_links) != 'undefined' && toc_links != null) {
+                // Then iterate forwards, on the first match highlight it and break
+                find_active: {
+                    for (let i = headings.length - 1; i >= 0; i--) {
+                        if (headings[i].getBoundingClientRect().top - 50 <= 0) {
+                            if (!toc_links[i].classList.contains("active")) {
+                                toc_links.forEach((link, _index) => {
+                                    link.classList.remove("active");
+                                });
+                                toc_links[i].classList.add('active');
+                            }
+                            break find_active;
+                        }
+                    }
+                    toc_links.forEach((link, _index) => {
+                        link.classList.remove("active");
+                    });
+                }
+            }
+        });
+    }
+</script>
+</body>
+</html>

app/src/index.js

@@ -0,0 +1,10 @@
+import { initLeaderboardResults } from './leaderboard_results.js';
+import { initFineTasks } from './fine_tasks.js';
+import { initPlotApplets } from './plot_task.js';
+import 'simple-datatables/dist/style.css';
+
+document.addEventListener('DOMContentLoaded', () => {
+  initLeaderboardResults('leaderboard-results');
+  initFineTasks('fine-tasks-results');
+  initPlotApplets();
+}, { once: true });

app/src/leaderboard_results.js

@@ -0,0 +1,397 @@
+import Papa from 'papaparse';
+import { DataTable } from 'simple-datatables';
+
+const languageMap = {
+  'All Languages': 'final_rankings.csv',
+  'Arabic': 'results_ar.csv',
+  'Turkish': 'results_tr.csv',
+  'Swahili': 'results_sw.csv',
+  'Russian': 'results_ru.csv',
+  'Telugu': 'results_te.csv',
+  'Thai': 'results_th.csv',
+  'Chinese': 'results_zh.csv',
+  'French': 'results_fr.csv',
+  'Hindi': 'results_hi.csv',
+};
+
+const versionMap = {
+  'v1': 'v1',
+  'v2': 'v2'
+};
+
+const versionChangelog = {
+  'v1': 'Initial release of FineTasks Leaderboard',
+  'v2': 'Changes in v2:\n' +
+        '• Fixed a bug in the rescaling of scores\n' +
+        '• Switched to using Native choice prefixes for Thai/Telugu/Hindi/Arabics\n' +
+        '• Added Options: anchors before showing options for continuation tasks (e.g Hellawag) - consistent improvement in scores\n' +
+        '• Removed openai/gpt-4o-mini'
+};
+
+const columnNameMap = {
+  'runname': 'Model',
+  'agg_score_macro': 'Score',
+  'agg_score_RES': 'RES Score',
+  'agg_score_RC': 'RC Score',
+  'agg_score_GK': 'GK Score',
+  'agg_score_NLU': 'NLU Score',
+  'avg_rank_macro': 'Multilingual Score',
+  'rank': 'Rank'
+};
+
+function createDropdown(options, onChange, initialValue = null) {
+  const select = document.createElement('select');
+  options.forEach(option => {
+    const optionElement = document.createElement('option');
+    optionElement.value = option;
+    optionElement.textContent = option;
+    if (initialValue && option === initialValue) {
+      optionElement.selected = true;
+    }
+    select.appendChild(optionElement);
+  });
+  select.addEventListener('change', onChange);
+  return select;
+}
+
+function processTaskName(taskName) {
+  const parts = taskName.split('|');
+  let processedName = parts.length > 1 ? parts[1] : taskName;
+  processedName = processedName.split('_mcf')[0].split('_cf')[0];
+  return processedName;
+}
+
+function sanitizeColumnName(name) {
+  return name.replace(/[^a-zA-Z0-9-_]/g, '_');
+}
+
+function createResultsTable(data, extraColumn) {
+  const tableWrapper = document.createElement('div');
+  tableWrapper.className = 'table-wrapper leaderboard-table-wrapper';
+
+  const table = document.createElement('table');
+  table.className = 'results-table leaderboard-results-table';
+
+  const columns = extraColumn === 'All Languages' 
+    ? ['rank', 'runname', 'avg_rank_macro']
+    : ['rank', 'runname', 'agg_score_macro', extraColumn].filter(Boolean);
+
+  const header = table.createTHead();
+  const headerRow = header.insertRow();
+  columns.forEach(column => {
+    const th = document.createElement('th');
+    th.textContent = columnNameMap[column] || processTaskName(column);
+    th.className = `column-${sanitizeColumnName(column)}`; // Sanitize the column name
+    headerRow.appendChild(th);
+  });
+
+  const body = table.createTBody();
+  data.forEach((row, index) => {
+    if (!row.runname) return; // Skip rows without a model name
+    const tr = body.insertRow();
+    
+    // Add gradient background for top 3 positions
+    if (index < 3) {
+      const opacity = 1 - (index * 0.25); // Creates a fading effect: 1, 0.75, 0.5
+      tr.style.backgroundColor = `rgba(255, 165, 0, ${opacity * 0.2})`; // Light orange with fading opacity
+      tr.style.fontWeight = 600; // Make text slightly bolder for top 3
+    }
+
+    columns.forEach(column => {
+      const td = tr.insertCell();
+      td.className = `column-${sanitizeColumnName(column)}`;
+      
+      if (column === 'rank') {
+        td.textContent = index + 1;
+        // Add special styling for top 3 ranks
+        if (index < 3) {
+          td.style.fontWeight = 'bold';
+          switch(index) {
+            case 0:
+              td.style.color = '#FFB800'; // Gold
+              break;
+            case 1:
+              td.style.color = '#C0C0C0'; // Silver
+              break;
+            case 2:
+              td.style.color = '#CD7F32'; // Bronze
+              break;
+          }
+        }
+      } else if (column === 'runname') {
+        const modelName = row[column];
+        let displayName;
+        
+        // Check if it's a chat model
+        const chatModels = [
+          'CohereForAI/c4ai-command-r-plus-08-2024',
+          'openai/gpt-4o-mini',
+          'silma-ai/SILMA-9B-Instruct-v1.0',
+          'microsoft/Phi-3.5-mini-instruct',
+          'TURKCELL/Turkcell-LLM-7b-v1'
+        ];
+        
+        if (chatModels.some(chatModel => modelName.includes(chatModel))) {
+          displayName = `💬 ${modelName}`;
+        } else {
+          displayName = `🟢 ${modelName}`;
+        }
+
+        if (modelName.split("/")[0] !== "openai")
+          displayName = `<a href="https://huggingface.co/${modelName}">${displayName}</a>`;
+        td.innerHTML = displayName;
+        td.title = modelName; // Add full model name as tooltip
+        td.style.cursor = 'help'; // Change cursor to indicate hover functionality
+      } else {
+        const value = row[column];
+        td.textContent = typeof value === 'number' ? value.toFixed(2) : value;
+      }
+    });
+  });
+
+  tableWrapper.appendChild(table);
+  return tableWrapper;
+}
+
+function createChangelog() {
+  const changelogContainer = document.createElement('div');
+  changelogContainer.className = 'changelog-container';
+
+  const changelogHeader = document.createElement('div');
+  changelogHeader.className = 'changelog-header';
+  
+  const arrow = document.createElement('span');
+  arrow.className = 'changelog-arrow';
+  arrow.textContent = '▶';
+  
+  const label = document.createElement('span');
+  label.textContent = 'Changelog';
+  label.className = 'changelog-label';
+
+  const content = document.createElement('div');
+  content.className = 'changelog-content';
+  content.style.display = 'none';
+
+  changelogHeader.appendChild(arrow);
+  changelogHeader.appendChild(label);
+  changelogContainer.appendChild(changelogHeader);
+  changelogContainer.appendChild(content);
+
+  // Toggle changelog visibility
+  changelogHeader.addEventListener('click', () => {
+    const isVisible = content.style.display !== 'none';
+    content.style.display = isVisible ? 'none' : 'block';
+    arrow.textContent = isVisible ? '▶' : '▼';
+  });
+
+  return { container: changelogContainer, content };
+}
+
+export function initLeaderboardResults(containerId) {
+  const container = document.getElementById(containerId);
+  if (!container) return;
+
+  const titleElement = document.createElement('h3');
+  titleElement.textContent = 'FineTasks Leaderboard';
+  titleElement.className = 'leaderboard-title';
+
+  // Create changelog
+  const { container: changelogContainer, content: changelogContent } = createChangelog();
+
+  const tableContainer = document.createElement('div');
+  tableContainer.className = 'table-container';
+  
+  let leaderboardDataTable;
+  let currentData = [];
+
+  // Create caption element
+  const captionElement = document.createElement('figcaption');
+  captionElement.className = 'table-caption';
+  captionElement.textContent = container.dataset.caption || '';
+
+  // Define update functions first
+  async function updateLanguageTable() {
+    const selectedVersion = versionDropdown.value;
+    const selectedLanguage = languageDropdown.value;
+    const csvFile = languageMap[selectedLanguage];
+
+    try {
+      const response = await fetch(`data/os_models/${selectedVersion}/${csvFile}`);
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+      const csvText = await response.text();
+      const results = Papa.parse(csvText, { header: true, dynamicTyping: true }).data;
+      currentData = selectedLanguage === 'All Languages'
+        ? results.sort((a, b) => a.avg_rank_macro - b.avg_rank_macro)
+        : results.sort((a, b) => b.agg_score_macro - a.agg_score_macro);
+
+      if (selectedLanguage !== 'All Languages') {
+        const columnOptions = ['None'].concat(Object.keys(currentData[0]).filter(key => 
+          !['runname', 'seed', 'steps', 'agg_score_micro', 'rank', 'avg_rank_macro', ''].includes(key)
+        ));
+        extraColumnDropdown.innerHTML = '';
+        columnOptions.forEach(option => {
+          const optionElement = document.createElement('option');
+          optionElement.value = option;
+          optionElement.textContent = option === 'None' ? 'None' : processTaskName(option);
+          extraColumnDropdown.appendChild(optionElement);
+        });
+        
+        extraColumnDropdown.value = 'None';
+        extraColumnLabel.style.display = 'inline';
+        extraColumnDropdown.style.display = 'inline';
+      } else {
+        extraColumnLabel.style.display = 'none';
+        extraColumnDropdown.style.display = 'none';
+      }
+
+      updateTable();
+      updateChangelog();
+    } catch (error) {
+      console.error('Error fetching CSV:', error);
+      tableContainer.innerHTML = `<p>Error loading data: ${error.message}</p>`;
+    }
+  }
+
+  function updateTable() {
+    const extraColumn = languageDropdown.value === 'All Languages' ? 'All Languages' : 
+                       (extraColumnDropdown.value === 'None' ? null : extraColumnDropdown.value);
+    
+    tableContainer.innerHTML = '';
+    const tableWrapper = createResultsTable(currentData, extraColumn);
+    tableContainer.appendChild(tableWrapper);
+
+    if (leaderboardDataTable) {
+      leaderboardDataTable.destroy();
+    }
+
+    leaderboardDataTable = new DataTable('.leaderboard-results-table', {
+      perPage: 10,
+      perPageSelect: false,
+      searchable: false,
+      sortable: true,
+      fixedHeight: true,
+      labels: {
+        info: ''
+      }
+    });
+
+    setTimeout(adjustColumnWidths, 0);
+  }
+
+  function updateChangelog() {
+    const selectedVersion = versionDropdown.value;
+    changelogContent.textContent = versionChangelog[selectedVersion];
+  }
+
+  // Add this function to get URL parameters
+  function getUrlParameter(name) {
+    const urlParams = new URLSearchParams(window.location.search);
+    return urlParams.get(name);
+  }
+
+  // Add this function to set URL parameters
+  function updateUrlParameter(key, value) {
+    const urlParams = new URLSearchParams(window.location.search);
+    if (value) {
+      urlParams.set(key, value);
+    } else {
+      urlParams.delete(key);
+    }
+    const newUrl = `${window.location.pathname}${urlParams.toString() ? '?' + urlParams.toString() : ''}`;
+    window.history.pushState({ path: newUrl }, '', newUrl);
+  }
+
+  // Get initial language from URL
+  const urlLanguage = getUrlParameter('language');
+  const initialLanguage = urlLanguage && Object.keys(languageMap).includes(urlLanguage) 
+    ? urlLanguage 
+    : 'All Languages';
+
+  // Create dropdowns with initial values
+  const languageLabel = document.createElement('label');
+  languageLabel.textContent = 'Language: ';
+  const languageDropdown = createDropdown(
+    Object.keys(languageMap), 
+    (e) => {
+      updateLanguageTable();
+      updateUrlParameter('language', e.target.value === 'All Languages' ? null : e.target.value);
+    }, 
+    initialLanguage
+  );
+
+  const extraColumnLabel = document.createElement('label');
+  extraColumnLabel.textContent = 'Task: ';
+  const extraColumnDropdown = createDropdown(['None'], updateTable);
+
+  const versionLabel = document.createElement('label');
+  versionLabel.textContent = 'Version: ';
+  const versionDropdown = createDropdown(Object.keys(versionMap), updateLanguageTable);
+
+  // Create controls
+  const controls = document.createElement('div');
+  controls.className = 'controls leaderboard-controls fine-tasks-controls';
+
+  const versionControlGroup = document.createElement('div');
+  versionControlGroup.className = 'control-group';
+  versionControlGroup.appendChild(versionLabel);
+  versionControlGroup.appendChild(versionDropdown);
+
+  const languageControlGroup = document.createElement('div');
+  languageControlGroup.className = 'control-group';
+  languageControlGroup.appendChild(languageLabel);
+  languageControlGroup.appendChild(languageDropdown);
+
+  const extraColumnControlGroup = document.createElement('div');
+  extraColumnControlGroup.className = 'control-group';
+  extraColumnControlGroup.appendChild(extraColumnLabel);
+  extraColumnControlGroup.appendChild(extraColumnDropdown);
+
+  controls.appendChild(versionControlGroup);
+  controls.appendChild(languageControlGroup);
+  controls.appendChild(extraColumnControlGroup);
+
+  // Add elements to container in new order
+  container.appendChild(titleElement);
+  container.appendChild(tableContainer);
+  container.appendChild(captionElement);
+  container.appendChild(controls);
+  container.appendChild(changelogContainer);
+
+  // Initialize with URL language if present
+  versionDropdown.value = 'v2';
+  languageDropdown.value = initialLanguage;
+  updateLanguageTable();
+}
+
+function adjustColumnWidths() {
+  const table = document.querySelector('.leaderboard-results-table');
+  if (!table) return;
+
+  const columns = table.querySelectorAll('th');
+  columns.forEach((column, index) => {
+    const columnClass = column.className;
+    const cells = table.querySelectorAll(`td.${columnClass}`);
+    let maxWidth = column.offsetWidth;
+    cells.forEach(cell => {
+      maxWidth = Math.max(maxWidth, cell.offsetWidth);
+    });
+
+    let adjustedWidth;
+    if (index === 0) { // Rank column
+      adjustedWidth = 50;
+    } else if (index === 1) { // Model name column
+      adjustedWidth = 200;
+    } else if (index === 2) { // Macro score column
+      adjustedWidth = 100;
+    } else { // Extra column or any other column
+      adjustedWidth = Math.min(maxWidth, 150); // Set a maximum width of 150px for other columns
+    }
+
+    column.style.width = `${adjustedWidth}px`;
+    cells.forEach(cell => {
+      cell.style.width = `${adjustedWidth}px`;
+    });
+  });
+}

app/src/plot_task.js

@@ -0,0 +1,619 @@
+import Plotly from 'plotly.js-basic-dist-min';
+import Papa from 'papaparse';
+import _ from 'lodash';
+import { getColor } from './colors.mjs';
+
+const languageMap = {
+  'Arabic': 'ar',
+  'Turkish': 'tr',
+  'Swahili': 'sw',
+  'Russian': 'ru',
+  'Telugu': 'te',
+  'Thai': 'th',
+  'Chinese': 'zh',
+  'French': 'fr',
+  'Hindi': 'hi'
+};
+
+const runNameMap = {
+  "orion": "Dataset-A",
+  "helios": "Dataset-B",
+  "lynx": "Dataset-C",
+  "aquila": "Dataset-D",
+  "commoncrawl": "CommonCrawl",
+  "baseline": "Baseline"
+};
+
+const taskLists = {
+  ar: ['acva_ara:_average', 'alfgahafa_mlqa_ara_cf', 'alghafa_arc_ara_cf:easy', 'alghafa_facts_ara_cf', 'alghafa_meta_dialects_ara_cf', 'alghafa_mmlu_ara_cf:_average', 'alghafa_openbookqa_ara_cf', 'alghafa_piqa_ara_cf', 'alghafa_race_ara_cf', 'alghafa_rating_sentiment_ara_cf', 'alghafa_rating_sentiment_no_neutral_ara_cf', 'alghafa_sciqa_ara_cf', 'alghafa_sentiment_ara_cf', 'arcd_ara', 'belebele_arb_Arab_cf', 'boolq_ara', 'exams_ara_cf:_average', 'mkqa_ara:_average', 'mlmm_arc_ara_cf:challenge', 'mlmm_hellaswag_ara_cf', 'mlmm_mmlu_ara_cf:_average', 'mlmm_truthfulqa_ara_cf:mc1', 'mlmm_truthfulqa_ara_cf:mc2', 'mlqa_ara', 'mmlu_ara_cf:_average', 'soqal_ara_cf', 'toxigen_ara_cf', 'tydiqa_ara', 'xcodah_ara_cf', 'xcopa_ara_cf', 'xcsqa_ara_cf', 'xnli2.0_ara_cf', 'xnli_ara_cf', 'xquad_ara', 'xstory_cloze_ara_cf'],
+  fr: ['belebele_fra_Latn_cf', 'community_boolq_fra_cf', 'exams_fra_cf:_average', 'fquadv2_fra', 'frenchbench_arc_fra_cf:challenge', 'frenchbench_hellaswag_fra_cf', 'meta_mmlu_fra_cf:_average', 'mintaka_fra', 'mkqa_fra:_average', 'mlmm_arc_fra_cf:challenge', 'mlmm_hellaswag_fra_cf', 'mlmm_mmlu_fra_cf:_average', 'mlmm_truthfulqa_fra_cf:mc1', 'mlmm_truthfulqa_fra_cf:mc2', 'pawsx_fra_cf', 'xcodah_fra_cf', 'xcsqa_fra_cf', 'xnli2.0_fra_cf', 'xwinograd_fra_cf'],
+  hi: ['belebele_hin_Deva_cf', 'community_arc_hin_cf:challenge', 'community_arc_hin_cf:easy', 'community_boolq_hin', 'community_hellaswag_hin_cf', 'indicnxnli_hin_cf', 'indicqa_hin', 'indicxcopa_hin_cf', 'meta_mmlu_hin_cf:_average', 'mintaka_hin', 'mlmm_arc_hin_cf:challenge', 'mlmm_hellaswag_hin_cf', 'mlmm_mmlu_hin_cf:_average', 'mlmm_truthfulqa_hin_cf:mc1', 'mlmm_truthfulqa_hin_cf:mc2', 'mlqa_hin', 'xcodah_hin_cf', 'xcsqa_hin_cf', 'xnli2.0_hin_cf', 'xnli_hin_cf', 'xquad_hin', 'xstory_cloze_hin_cf'],
+  ru: ['belebele_rus_Cyrl_cf', 'chegeka_rus', 'mathlogic_qa_rus_cf', 'mera_openbookqa_rus_cf', 'mera_worldtree_rus_cf', 'mkqa_rus:_average', 'mlmm_arc_rus_cf:challenge', 'mlmm_hellaswag_rus_cf', 'mlmm_mmlu_rus_cf:_average', 'mlmm_truthfulqa_rus_cf:mc1', 'mlmm_truthfulqa_rus_cf:mc2', 'parus_rus_cf', 'rcb_rus_cf', 'rummlu_rus_cf:_average', 'sber_squad_rus', 'tydiqa_rus', 'xcodah_rus_cf', 'xcsqa_rus_cf', 'xnli2.0_rus_cf', 'xquad_rus', 'xstory_cloze_rus_cf', 'xwinograd_rus_cf'],
+  sw: ['afric_mmlu_swa_cf:_average', 'afric_xnli_swa_cf', 'belebele_swh_Latn_cf', 'community_arc_swa_cf:challenge', 'community_arc_swa_cf:easy', 'community_mmlu_swa_cf', 'kenswquad_swa', 'm3exams_swa_cf', 'openai_mmlu_swa_cf:_average', 'tydiqa_swa', 'xcodah_swa_cf', 'xcopa_swa_cf', 'xcsqa_swa_cf', 'xnli2.0_swa_cf', 'xnli_swa_cf', 'xstory_cloze_swa_cf'],
+  te: ['belebele_tel_Telu_cf', 'community_hellaswag_tel_cf', 'indicnxnli_tel_cf', 'indicqa_tel', 'indicxcopa_tel_cf', 'mlmm_arc_tel_cf:challenge', 'mlmm_hellaswag_tel_cf', 'mlmm_mmlu_tel_cf:_average', 'mlmm_truthfulqa_tel_cf:mc1', 'mlmm_truthfulqa_tel_cf:mc2', 'tydiqa_tel', 'xstory_cloze_tel_cf'],
+  th: ['belebele_tha_Thai_cf', 'community_hellaswag_tha_cf', 'm3exams_tha_cf', 'meta_mmlu_tha_cf:_average', 'mkqa_tha:_average', 'thai_exams_tha_cf:_average', 'thai_exams_tha_cf:tgat', 'thaiqa_tha', 'wsci_tha_cf', 'xcopa_tha_cf', 'xnli2.0_tha_cf', 'xnli_tha_cf', 'xquad_tha'],
+  tr: ['belebele_tur_Latn_cf', 'community_arc_tur_cf:easy', 'community_hellaswag_tur_cf', 'community_mmlu_tur_cf:_average', 'community_truthfulqa_tur_cf:mc1', 'community_truthfulqa_tur_cf:mc2', 'community_xwinograd_tur_cf', 'exams_tur_cf:_average', 'mkqa_tur:_average', 'tquadv2_tur', 'xcopa_tur_cf', 'xnli2.0_tur_cf', 'xnli_tur_cf', 'xquad_tur'],
+  zh: ['agieval_zho_cf:_average', 'belebele_zho_Hans_cf', 'c3_zho_cf', 'ceval_zho_cf:_average', 'chinese_squad_zho', 'cmath_zho_cf', 'cmmlu_zho_cf:_average', 'cmnli_zho_cf', 'cmrc2018_zho', 'm3exams_zho_cf', 'mkqa_zho:_average', 'mlmm_arc_zho_cf:challenge', 'mlmm_hellaswag_zho_cf', 'mlmm_mmlu_zho_cf:_average', 'mlmm_truthfulqa_zho_cf:mc1', 'mlmm_truthfulqa_zho_cf:mc2', 'ocnli_zho_cf', 'pawsx_zho_cf', 'xcodah_zho_cf', 'xcopa_zho_cf', 'xcsqa_zho_cf', 'xnli2.0_zho_cf', 'xnli_zho_cf', 'xquad_zho', 'xstory_cloze_zho_cf', 'xwinograd_zho_cf']
+};
+
+const LINE_SETTINGS = {
+  width: 2.5,
+  type: "scatter",
+  mode: "lines+markers",
+};
+
+const DEFAULT_LAYOUT = {
+  font: {
+    family: "apple-system, Arial, sans-serif",
+  },
+  title: {
+    font: {
+      size: 15,
+    },
+  },
+  xaxis: {
+    title: {
+      text: "Training Tokens (billions)",
+      font: {
+        size: 14,
+      },
+    },
+    tickfont: {
+      size: 12,
+    },
+    showgrid: false,
+    mirror: true,
+    ticks: "outside",
+    showline: true,
+  },
+  yaxis: {
+    title: {
+      font: {
+        size: 14,
+      },
+      standoff: 10,
+    },
+    showgrid: false,
+    mirror: true,
+    ticks: "outside",
+    showline: true,
+    tickfont: {
+      size: 12,
+    },
+  },
+  height: 300, // You can adjust this value
+  autosize: true,
+  legend: {
+    orientation: 'h',        // Set to 'h' for horizontal legend (required for columns)
+    yanchor: 'bottom',
+    y: 0,                    // Position at the bottom
+    xanchor: 'right',
+    x: 1,                    // Position at the right
+    traceorder: 'normal',
+    font: { size: 12 },
+    tracegroupgap: 0,        // Space between legend items
+    bgcolor: 'rgba(255, 255, 255, 0.8)' // White background with 70% transparency (1 - 0.3 = 70%)
+  },
+  margin: {
+    t: 25,
+    b: 60,
+    l: 60,
+    r: 40,
+  },
+};
+
+export function initPlotApplets() {
+  const plotContainers = document.querySelectorAll('.task-signal-plot');
+  plotContainers.forEach(container => {
+    initPlotApplet(container);
+  });
+}
+
+function initPlotApplet(container) {
+  const defaultLanguage = container.dataset.language || 'Arabic';
+  const defaultTask = container.dataset.task || '';
+  const defaultMetric = container.dataset.metric || '';
+  const groupSeeds = container.dataset.groupSeeds === 'true';
+  const showControls = container.dataset.showControls === 'true';
+  const taskMetrics = (container.dataset.taskMetrics || 'monotonicity,snr,ordering,randomness').split(",");
+
+  const controls = createControls(container, defaultLanguage, defaultTask, defaultMetric, taskMetrics);
+  if (!showControls)
+    controls.style.display = 'none';
+  container.appendChild(controls);
+
+  const plotContainer = document.createElement('div');
+  plotContainer.className = 'plot-container';
+  container.appendChild(plotContainer);
+
+  const statsContainer = document.createElement('div');
+  statsContainer.className = 'stats-container';
+  container.appendChild(statsContainer);
+
+
+  // Create an initial empty plot
+  Plotly.newPlot(plotContainer, []);
+
+  // Set up the resize function
+  const resizePlot = () => {
+    const width = container.offsetWidth;
+    Plotly.relayout(plotContainer, { width: width });
+  };
+
+  // Add resize listener
+  window.addEventListener('resize', resizePlot);
+
+  // Initial resize
+  resizePlot();
+
+  // Load the initial data
+  updateLanguageTasks(container, defaultTask, defaultMetric, groupSeeds, taskMetrics);
+}
+
+function createControls(container, defaultLanguage, defaultTask, defaultMetric, taskMetrics) {
+  const controls = document.createElement('div');
+  controls.className = 'controls';
+
+  const languageSelect = createSelect('language', Object.keys(languageMap), () => updateLanguageTasks(container, '', '', true, taskMetrics));
+  languageSelect.value = defaultLanguage;
+
+  const taskSelect = createSelect('task', [], () => updateMetrics(container, '', true, taskMetrics));
+  const metricSelect = createSelect('metric', [], () => updatePlot(container, taskMetrics));
+
+  controls.appendChild(createControlGroup('Language:', languageSelect));
+  controls.appendChild(createControlGroup('Task:', taskSelect));
+  controls.appendChild(createControlGroup('Metric:', metricSelect));
+
+  return controls;
+}
+
+function createSelect(id, options, onChangeHandler) {
+  const select = document.createElement('select');
+  select.id = id;
+  options.forEach(option => {
+    const optionElement = document.createElement('option');
+    optionElement.value = option;
+    optionElement.textContent = option;
+    select.appendChild(optionElement);
+  });
+  select.addEventListener('change', onChangeHandler);
+  return select;
+}
+
+function createControlGroup(labelText, inputElement) {
+  const group = document.createElement('div');
+  group.className = 'control-group';
+  
+  const label = document.createElement('label');
+  label.textContent = labelText;
+  label.className = 'control-label';
+  
+  group.appendChild(label);
+  group.appendChild(inputElement);
+  
+  return group;
+}
+
+async function updateLanguageTasks(container, defaultTask = '', defaultMetric = '', groupSeeds, taskMetrics) {
+  const languageSelect = container.querySelector('#language');
+  const taskSelect = container.querySelector('#task');
+  const language = languageSelect.value;
+  const langCode = languageMap[language];
+
+  taskSelect.innerHTML = '<option value="">Loading tasks...</option>';
+
+  try {
+    const tasks = await getTasksForLanguage(langCode);
+    
+    taskSelect.innerHTML = '';
+    if (tasks.length > 0) {
+      tasks.forEach(task => {
+        const option = document.createElement('option');
+        option.value = task;
+        option.textContent = truncateText(task, 25); // Reduced from 30 to 25
+        option.title = task; // Set full task name as title for tooltip
+        taskSelect.appendChild(option);
+      });
+      
+      if (defaultTask && tasks.includes(defaultTask)) {
+        taskSelect.value = defaultTask;
+      } else {
+        taskSelect.selectedIndex = 0;
+      }
+      
+      await updateMetrics(container, defaultMetric, groupSeeds, taskMetrics);
+    } else {
+      taskSelect.innerHTML = '<option value="">No tasks available</option>';
+      clearPlot(container);
+    }
+  } catch (error) {
+    console.error('Error fetching tasks:', error);
+    taskSelect.innerHTML = '<option value="">Error loading tasks</option>';
+    clearPlot(container);
+  }
+}
+
+async function getTasksForLanguage(langCode) {
+  return taskLists[langCode] || [];
+}
+
+async function updateMetrics(container, defaultMetric = '', groupSeeds, taskMetrics) {
+  const language = container.querySelector('#language').value;
+  const task = container.querySelector('#task').value;
+  const langCode = languageMap[language];
+  const metricSelect = container.querySelector('#metric');
+
+  metricSelect.innerHTML = '<option value="">Loading metrics...</option>';
+
+  try {
+    const metrics = await getMetricsForTask(langCode, task);
+    
+    metricSelect.innerHTML = '';
+    metrics.forEach(metric => {
+      const option = document.createElement('option');
+      option.value = metric;
+      option.textContent = metric;
+      metricSelect.appendChild(option);
+    });
+
+    if (defaultMetric && metrics.includes(defaultMetric)) {
+      metricSelect.value = defaultMetric;
+    } else if (metricSelect.options.length > 0) {
+      metricSelect.selectedIndex = 0;
+    }
+
+    await updatePlot(container, taskMetrics);
+  } catch (error) {
+    console.error('Error fetching metrics:', error);
+    metricSelect.innerHTML = '<option value="">Error loading metrics</option>';
+    clearPlot(container);
+  }
+}
+
+async function getMetricsForTask(langCode, task) {
+  return new Promise((resolve, reject) => {
+    Papa.parse(`data/nanotron_tasks/${langCode}/${task}_stats.csv`, {
+      download: true,
+      header: true,
+      complete: function(results) {
+        const metrics = [...new Set(results.data.map(row => row.metric).filter(metric => metric))];
+        resolve(metrics);
+      },
+      error: function(error) {
+        console.error('Error fetching metrics:', error);
+        reject(error);
+      }
+    });
+  });
+}
+
+function updatePlot(container, taskMetrics) {
+  const language = container.querySelector('#language').value;
+  const task = container.querySelector('#task').value;
+  const metric = container.querySelector('#metric').value;
+  const title = container.dataset.title;
+  const langCode = languageMap[language];
+
+  if (!langCode || !task || !metric) {
+    clearPlot(container);
+    return;
+  }
+
+  const dataUrl = `data/nanotron_tasks/${langCode}/${task}_data.csv`;
+  const statsUrl = `data/nanotron_tasks/${langCode}/${task}_stats.csv`;
+
+  Promise.all([
+    new Promise((resolve, reject) => {
+      Papa.parse(dataUrl, {
+        download: true,
+        header: true,
+        dynamicTyping: true,
+        complete: resolve,
+        error: reject
+      });
+    }),
+    new Promise((resolve, reject) => {
+      Papa.parse(statsUrl, {
+        download: true,
+        header: true,
+        dynamicTyping: true,
+        complete: resolve,
+        error: reject
+      });
+    })
+  ]).then(([dataResult, statsResult]) => {
+    const taskData = dataResult.data;
+    const statsData = statsResult.data;
+    plotData(container, taskData, statsData, metric, title, taskMetrics);
+  }).catch(error => {
+    console.error('Error parsing CSV:', error);
+    clearPlot(container);
+  });
+}
+
+function plotData(container, data, stats, metric, title, taskMetrics) {
+  const groupSeeds = container.dataset.groupSeeds === 'true';
+  const sortedData = sortDataByTokens(data);
+  const groupedData = groupDataByRunname(sortedData, groupSeeds, metric);
+  const interpolatedData = interpolateData(groupedData, metric);
+  const smoothedData = smoothData(interpolatedData, metric);
+  const traces = createTraces(smoothedData, metric);
+
+  const plotContainer = container.querySelector('.plot-container');
+
+  const layout = _.merge({}, DEFAULT_LAYOUT, {
+    title: { text: `${title}` },
+    xaxis: { 
+      title: { text: 'Training Tokens (billions)' },
+      tickvals: [0, 5, 10, 15, 20, 25],
+      ticktext: ['0', '5B', '10B', '15B', '20B', '25B'],
+      tickangle: 45,
+      range: [0, 30], // Set the range to start from 0 and end at 30B
+    },
+    yaxis: { 
+      title: { text: 'Score' },
+      range: [Math.min(...traces.flatMap(trace => trace.y)) * 0.95, Math.max(...traces.flatMap(trace => trace.y)) * 1.05], // Add 5% padding to the top and bottom
+    },
+    width: container.offsetWidth,
+  });
+
+  Plotly.newPlot(plotContainer, traces, layout, {responsive: true});
+
+  // Display statistics
+  displayStatistics(container, stats, metric, taskMetrics);
+}
+
+function displayStatistics(container, stats, metric, taskMetrics) {
+  const statsContainer = container.querySelector('.stats-container');
+  const metricStats = stats.find(stat => stat.metric === metric);
+  if (metricStats) {
+    statsContainer.innerHTML = `
+      <div class="compact-stats${taskMetrics.length === 1 ? '-single' : ''}">
+        ${taskMetrics.includes('monotonicity') ? '<span title="Average Spearman Correlation">Monotonicity: ' + metricStats.avg_spearman.toFixed(2) + '</span>' : ''}
+        ${taskMetrics.includes('snr') ? '<span title="Average Signal-to-Noise Ratio">Signal-to-Noise: ' + metricStats.avg_snr.toFixed(2) + '</span>' : ''}
+        ${taskMetrics.includes('ordering') ? '<span title="Average Kendall Tau-a">Ordering Consistency: ' + metricStats.avg_kendall_tau_a.toFixed(2) + '</span>' : ''}
+        ${taskMetrics.includes('randomness') ? '<span title="Max N Standard Deviations">Non-Randomness: ' + metricStats.max_n_std.toFixed(2) + '</span>' : ''}
+      </div>
+    `;
+  } else {
+    statsContainer.innerHTML = '<p>No statistics available for this metric.</p>';
+  }
+}
+
+function getReducedTickValues(tokens) {
+  const uniqueTokens = [...new Set(tokens)].sort((a, b) => a - b);
+  const tokenCount = uniqueTokens.length;
+  const targetTickCount = 10; // Adjust this value to increase/decrease the number of ticks
+
+  if (tokenCount <= targetTickCount) {
+    return uniqueTokens;
+  }
+
+  const stride = Math.ceil(tokenCount / targetTickCount);
+  return uniqueTokens.filter((_, index) => index % stride === 0);
+}
+
+function formatTickLabel(value) {
+  if (value >= 1e9) {
+    return (value / 1e9).toFixed(1) + 'B';
+  } else if (value >= 1e6) {
+    return (value / 1e6).toFixed(1) + 'M';
+  } else if (value >= 1e3) {
+    return (value / 1e3).toFixed(1) + 'K';
+  }
+  return value.toString();
+}
+
+function computeStatistics(data, metric) {
+  const stats = {
+    avg_spearman: 0,
+    avg_kendall_tau_a: 0,
+    avg_snr: 0,
+    max_n_std: 0
+  };
+
+  const baselineRun = Object.keys(data).find(key => key.toLowerCase().includes('baseline'));
+  const nonBaselineRuns = Object.keys(data).filter(key => key !== baselineRun);
+
+  // Compute statistics for each non-baseline run
+  nonBaselineRuns.forEach(run => {
+    const runData = data[run];
+    const tokens = runData.map(row => row.tokens);
+    const scores = runData.map(row => row[metric]);
+
+    // Spearman correlation
+    stats.avg_spearman += spearmanCorrelation(tokens, scores);
+
+    // Kendall Tau-a
+    const lastHalf = Math.floor(runData.length / 2);
+    const kendallTauValues = [];
+    for (let i = lastHalf; i < runData.length - 1; i++) {
+      kendallTauValues.push(kendallTauA(scores.slice(0, i + 1), scores.slice(0, i + 2)));
+    }
+    stats.avg_kendall_tau_a += _.mean(kendallTauValues);
+
+    // SNR and max_n_std
+    if (baselineRun) {
+      const baselineScores = data[baselineRun].map(row => row[metric]);
+      const stdDev = standardDeviation(scores);
+      stats.avg_snr += _.mean(scores) / stdDev;
+      stats.max_n_std = Math.max(stats.max_n_std, (_.max(scores) - _.mean(baselineScores)) / stdDev);
+    }
+  });
+
+  // Average the statistics
+  const numRuns = nonBaselineRuns.length;
+  stats.avg_spearman /= numRuns;
+  stats.avg_kendall_tau_a /= numRuns;
+  stats.avg_snr /= numRuns;
+
+  return stats;
+}
+
+function spearmanCorrelation(x, y) {
+  const n = x.length;
+  const rankX = rankData(x);
+  const rankY = rankData(y);
+  
+  let sum_d_squared = 0;
+  for (let i = 0; i < n; i++) {
+    const d = rankX[i] - rankY[i];
+    sum_d_squared += d * d;
+  }
+  
+  return 1 - (6 * sum_d_squared) / (n * (n * n - 1));
+}
+
+function rankData(data) {
+  const sorted = [...data].sort((a, b) => a - b);
+  return data.map(x => sorted.indexOf(x) + 1);
+}
+
+function kendallTauA(x, y) {
+  const n = x.length;
+  let concordant = 0;
+  let discordant = 0;
+
+  for (let i = 0; i < n; i++) {
+    for (let j = i + 1; j < n; j++) {
+      const sign_x = Math.sign(x[j] - x[i]);
+      const sign_y = Math.sign(y[j] - y[i]);
+      if (sign_x * sign_y > 0) concordant++;
+      else if (sign_x * sign_y < 0) discordant++;
+    }
+  }
+
+  return (concordant - discordant) / (n * (n - 1) / 2);
+}
+
+function standardDeviation(values) {
+  const mean = _.mean(values);
+  const squareDiffs = values.map(value => {
+    const diff = value - mean;
+    return diff * diff;
+  });
+  const avgSquareDiff = _.mean(squareDiffs);
+  return Math.sqrt(avgSquareDiff);
+}
+
+function interpolateData(data, metric) {
+  return _.mapValues(data, (rows) => {
+    const sortedRows = _.sortBy(rows, 'tokens');
+    const allTokens = _.uniq(_.flatMap(Object.values(data), rows => rows.map(r => r.tokens))).sort((a, b) => a - b);
+    
+    return allTokens.map(token => {
+      const exactMatch = _.find(sortedRows, { tokens: token });
+      if (exactMatch) return exactMatch;
+
+      const lowerRow = _.findLast(sortedRows, r => r.tokens < token);
+      const upperRow = _.find(sortedRows, r => r.tokens > token);
+
+      if (!lowerRow) return { ...upperRow, tokens: token };
+      if (!upperRow) return { ...lowerRow, tokens: token };
+
+      const ratio = (token - lowerRow.tokens) / (upperRow.tokens - lowerRow.tokens);
+      const interpolatedMetric = lowerRow[metric] + (upperRow[metric] - lowerRow[metric]) * ratio;
+
+      return {
+        ...lowerRow,
+        tokens: token,
+        [metric]: interpolatedMetric
+      };
+    });
+  });
+}
+
+function smoothData(data, metric, windowSize = 3) {
+  return _.mapValues(data, (rows) => {
+    return rows.map((row, index, array) => {
+      const window = array.slice(Math.max(0, index - windowSize + 1), index + 1);
+      const smoothedMetric = _.meanBy(window, r => r[metric]);
+      return { ...row, [metric]: smoothedMetric };
+    });
+  });
+}
+
+function sortDataByTokens(data) {
+  return _.sortBy(data, 'tokens');
+}
+
+function groupDataByRunname(data, groupSeeds, metric) {
+  // Remove null or undefined runs
+  data = data.filter(row => row.runname != null && row.runname !== 'null_undefined');
+
+  if (!groupSeeds) {
+    return _.groupBy(data, row => `${processRunName(row.runname)}_${row.seed}`);
+  }
+
+  const grouped = _.groupBy(data, row => processRunName(row.runname));
+  
+  return _.mapValues(grouped, (rows) => {
+    const stepGroups = _.groupBy(rows, 'tokens');
+    return _.map(stepGroups, (stepRows) => {
+      const meanMetric = _.meanBy(stepRows, row => parseFloat(row[metric]) || 0);
+      return {
+        ...stepRows[0],
+        [metric]: meanMetric
+      };
+    });
+  });
+}
+
+function processRunName(runname) {
+  for (const [key, value] of Object.entries(runNameMap)) {
+    if (runname.includes(key)) {
+      return value;
+    }
+  }
+  return runname;
+}
+
+function createTraces(groupedData, metric) {
+  const colorsMapping = new Map();
+  const sortedRunnames = Object.keys(groupedData).sort((a, b) => {
+    if (a.includes('baseline')) return 1;
+    if (b.includes('baseline')) return -1;
+    return a.localeCompare(b);
+  });
+
+  return sortedRunnames.map((runname, index) => {
+    const color = getColorForTrace(runname, colorsMapping, index);
+    return {
+      x: groupedData[runname].map(row => row.tokens),
+      y: groupedData[runname].map(row => row[metric]),
+      name: runname,
+      line: { 
+        color: color,
+        shape: 'spline',
+        ...LINE_SETTINGS
+      },
+      marker: {
+        color: color,
+        size: 6,
+      },
+      mode: 'lines+markers',
+    };
+  });
+}
+
+function getColorForTrace(traceName, colorsMapping, index) {
+  const reusedColor = colorsMapping.get(traceName);
+  if (reusedColor) {
+    return reusedColor;
+  }
+
+  const color = getColor(index);
+  colorsMapping.set(traceName, color);
+  return color;
+}
+
+function clearPlot(container) {
+  const plotContainer = container.querySelector('.plot-container');
+  Plotly.purge(plotContainer);
+}
+
+function truncateText(text, maxLength) {
+  if (text.length <= maxLength) return text;
+  return text.substr(0, maxLength - 2) + '..';
+}
+

app/src/style.css

@@ -0,0 +1,428 @@
+/* style.css */
+/* Define colors */
+:root {
+    --distill-gray: rgb(107, 114, 128);
+    --distill-gray-light: rgb(185, 185, 185);
+    --distill-gray-lighter: rgb(228, 228, 228);
+    --distill-gray-lightest: rgb(245, 245, 245);
+    --distill-blue: #007BFF;
+}
+
+d-byline .byline {
+  grid-template-columns: 1fr;
+  grid-column: text;
+  font-size: 0.9rem;
+  line-height: 1.8em;
+}
+
+@media (min-width: 768px) {
+  d-byline .byline {
+    grid-template-columns: 5fr 1fr 1fr;
+  }
+}
+
+d-contents > nav a.active {
+    text-decoration: underline;
+}
+
+@media (max-width: 1199px) {
+    d-contents {
+        display: none;
+        justify-self: start;
+        align-self: start;
+        padding-bottom: 0.5em;
+        margin-bottom: 1em;
+        padding-left: 0.25em;
+        border-bottom: 1px solid rgba(0, 0, 0, 0.1);
+        border-bottom-width: 1px;
+        border-bottom-style: solid;
+        border-bottom-color: rgba(0, 0, 0, 0.1);
+    }
+}
+
+d-contents a:hover {
+    border-bottom: none;
+}
+
+
+@media (min-width: 1200px) {
+    d-article {
+        /* Ensure d-article does not prevent sticky positioning */
+        overflow: visible;
+    }
+
+    d-contents {
+        align-self: start;
+        grid-column-start: 1 !important;
+        grid-column-end: 4 !important;
+        grid-row: auto / span 6;
+        justify-self: end;
+        margin-top: 0em;
+        padding-right: 3em;
+        padding-left: 2em;
+        /*border-right: 1px solid rgba(0, 0, 0, 0.1);*/
+        /*border-right-width: 1px;*/
+        /*border-right-style: solid;*/
+        /*border-right-color: rgba(0, 0, 0, 0.1);*/
+        position: -webkit-sticky; /* For Safari */
+        position: sticky;
+        top: 10px; /* Adjust this value if needed */
+        z-index: -1;
+    }
+}
+
+d-contents nav h3 {
+    margin-top: 0;
+    margin-bottom: 1em;
+}
+
+d-contents nav div {
+    color: rgba(0, 0, 0, 0.8);
+    font-weight: bold;
+}
+
+d-contents nav a {
+    color: rgba(0, 0, 0, 0.8);
+    border-bottom: none;
+    text-decoration: none;
+}
+
+d-contents li {
+    list-style-type: none;
+}
+
+d-contents ul, d-article d-contents ul {
+    padding-left: 1em;
+}
+
+d-contents nav ul li {
+    margin-bottom: .25em;
+}
+
+d-contents nav a:hover {
+    text-decoration: underline solid rgba(0, 0, 0, 0.6);
+}
+
+d-contents nav ul {
+    margin-top: 0;
+    margin-bottom: 6px;
+}
+
+
+d-contents nav > div {
+    display: block;
+    outline: none;
+    margin-bottom: 0.5em;
+}
+
+d-contents nav > div > a {
+    font-size: 13px;
+    font-weight: 600;
+}
+
+d-article aside {
+    display: none;
+}
+
+@media (min-width: 768px) {
+    d-article aside {
+        display: block;
+        height: 0px;
+        overflow: visible;
+        margin-bottom: 1em;
+        z-index: 1000;
+    }
+}
+
+@media (min-width: 768px) {
+    d-article aside {
+        margin-bottom: 0;
+    }
+}
+
+d-contents nav > div > a:hover,
+d-contents nav > ul > li > a:hover {
+    text-decoration: none;
+}
+
+
+/* Controlls for Plotting Applet */
+.controls {
+  display: flex;
+  flex-wrap: nowrap;
+  gap: 10px;
+  justify-content: center;
+}
+
+.control-group {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+}
+
+.controls select {
+  padding: 2px 4px;
+  line-height: 1.5em;
+  text-align: center;
+  border-radius: 4px;
+  font-size: 0.7em;
+  background-color: var(--distill-gray-lightest);
+  outline: none;
+}
+
+.controls label {
+  font-size: 0.8em;
+  font-weight: bold;
+}
+
+
+/* Specific style for the task dropdown */
+#task {
+  max-width: 180px;
+}
+
+.controls select option {
+  max-width: 300px;
+  overflow: visible;
+}
+
+.task-signal-plot {
+  width: 100%;
+  max-width: 500px; /* Adjust this value as needed */
+  margin: 0 auto;
+}
+
+
+.stats-container {
+  margin-bottom: 5px;
+}
+.compact-stats {
+  display: grid;
+  place-items: center;
+  grid-template-columns: 1fr 1fr;
+  gap: 5px;
+  font-weight: bold;
+  font-size: 12px;
+}
+.compact-stats-single {
+  display: grid;
+  place-items: center;
+  font-weight: bold;
+  font-size: 12px;
+}
+
+.fine-tasks-controls,
+.leaderboard-controls {
+  margin-bottom: 20px;
+}
+
+.fine-tasks-table-wrapper {
+  margin-top: 20px;
+}
+
+.fine-tasks-results-table {
+  width: 100%;
+  border-collapse: separate;
+  border-spacing: 0;
+  table-layout: fixed; /* This ensures that the table respects column widths */
+}
+
+.fine-tasks-results-table th,
+.fine-tasks-results-table td {
+  border: 1px solid #ddd;
+  padding: 8px;
+  text-align: left;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.fine-tasks-results-table th {
+  background-color: #f9f9f9;
+  font-weight: bold;
+}
+
+.fine-tasks-results-table tr:nth-child(even) {
+  background-color: #f2f2f2;
+}
+
+.fine-tasks-results-table tr:nth-child(odd) {
+  background-color: #fff;
+}
+
+.fine-tasks-results-table tr:hover {
+  background-color: #ddd;
+}
+
+/* Remove the horizontal line above the pagination */
+.fine-tasks-table-wrapper .datatable-bottom {
+  border-top: none;
+}
+
+/* Hide the "Showing X to Y of Z entries" text */
+.fine-tasks-table-wrapper .datatable-info {
+  display: none;
+}
+
+.fine-tasks-title {
+  text-align: center;
+  margin-top: 20px;
+  margin-bottom: 20px;
+}
+
+.fine-tasks-results-table td[title] {
+  cursor: help;
+}
+
+.leaderboard-title {
+  text-align: center;
+  margin-top: 20px;
+  margin-bottom: 20px;
+}
+
+
+.leaderboard-results-table {
+  width: 100%;
+  border-collapse: separate;
+  border-spacing: 0;
+  table-layout: fixed;
+}
+
+.leaderboard-results-table th,
+.leaderboard-results-table td {
+  border: 1px solid #ddd;
+  padding: 8px;
+  text-align: left;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.leaderboard-results-table th:first-child,
+.leaderboard-results-table td:first-child {
+  width: 50px; /* For the rank column */
+}
+
+.leaderboard-results-table th:nth-child(2),
+.leaderboard-results-table td:nth-child(2) {
+  width: 200px; /* For the model name column */
+}
+
+.leaderboard-results-table th:nth-child(3),
+.leaderboard-results-table td:nth-child(3) {
+  width: 100px; /* For the macro score column */
+}
+
+.leaderboard-results-table th:nth-child(4),
+.leaderboard-results-table td:nth-child(4) {
+  width: 150px; /* For the extra column */
+}
+
+.leaderboard-results-table th {
+  background-color: #f9f9f9;
+  font-weight: bold;
+}
+
+.leaderboard-results-table tr:nth-child(even) {
+  background-color: #f2f2f2;
+}
+
+.leaderboard-results-table tr:hover {
+  background-color: #ddd;
+}
+
+/* Remove the horizontal line above the pagination */
+.leaderboard-table-wrapper .datatable-bottom {
+  border-top: none;
+}
+
+/* Hide the "Showing X to Y of Z entries" text */
+.leaderboard-table-wrapper .datatable-info {
+  display: none;
+}
+
+.leaderboard-results-table td[title] {
+  cursor: help;
+}
+
+/* Tooltip styles */
+.leaderboard-results-table td[title]:hover::after {
+  content: attr(title);
+  position: absolute;
+  left: 0;
+  top: 100%;
+  background-color: #f9f9f9;
+  color: #000;
+  padding: 5px;
+  border: 1px solid #ddd;
+  border-radius: 4px;
+  z-index: 1000;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  max-width: 300px;
+}
+
+.table-caption {
+  text-align: center;
+  margin-bottom: 10px;
+  font-style: italic;
+  color: #666;
+  width: 100%; /* Ensure the figcaption takes full width */
+  display: block; /* Make it a block element */
+}
+
+html, body {
+    height: 100%;
+    overflow-y: auto;
+    -webkit-overflow-scrolling: touch; /* for smoother scrolling on iOS */
+}
+
+.main-container {
+    -webkit-overflow-scrolling: touch;
+}
+
+body {
+    -webkit-text-size-adjust: 100%;
+}
+
+/* Changelog styles */
+
+.changelog-container {
+  margin-top: 20px;
+  font-size: 14px;
+}
+
+.changelog-header {
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  gap: 12px; /* Increased space between arrow and label */
+  padding: 8px;
+  background-color: #f5f5f5;
+  border-radius: 4px;
+}
+
+.changelog-header:hover {
+  background-color: #ebebeb;
+}
+
+.changelog-arrow {
+  font-size: 12px;
+  color: #666;
+}
+
+.changelog-label {
+  font-weight: 600;
+  color: #333;
+}
+
+.changelog-content {
+  margin-top: 8px;
+  padding: 12px;
+  background-color: #fff;
+  border: 1px solid #eee;
+  border-radius: 4px;
+  line-height: 1.4;
+  white-space: pre-line;
+}

app/webpack.config.js

@@ -0,0 +1,111 @@
+const path = require("path");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const CopyPlugin = require("copy-webpack-plugin");
+const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
+
+const COLOR_KEYS = ["color", "bgColor", "fillcolor"];
+
+const transformDataColors = async (data, path) => {
+    const {getNamedColor} = await import('./src/colors.mjs');
+    // if not json file, return
+    if (!path.endsWith(".json")) {
+        return data;
+    }
+    const parsedData = JSON.parse(data);
+    // Change the color of the data
+    const deepIterateAndSetColor = (key, val) => {
+        if (val === null) {
+            return null;
+        }
+        if (val == undefined) {
+            return undefined;
+        }
+        if (Array.isArray(val)) {
+            return val.map(item => deepIterateAndSetColor(key, item));
+        }
+        if (typeof val === "object") {
+            return Object.entries(val).reduce((newObj, [key, value]) => {
+                newObj[key] = deepIterateAndSetColor(key, value);
+                return newObj;
+            }, {});
+        }
+        if (COLOR_KEYS.includes(key)) {
+            const [colorName, opacity, ...rest] = val.trim().split(/\s+/);
+            const floatOpacity = parseFloat(opacity);
+            const newColor = getNamedColor(colorName, floatOpacity);
+            if (newColor !== undefined && rest.length === 0 && !isNaN(floatOpacity)) {
+                console.log(`key: ${key} in file ${path} changed from ${val} to ${newColor}`);
+                return newColor;
+            } else {
+                return val;
+            }
+        }
+        return val;
+    };
+    return JSON.stringify(deepIterateAndSetColor(undefined, parsedData))
+};
+
+module.exports = {
+    entry: {
+        distill: "./src/distill.js",
+        main: "./src/index.js",
+    },
+    output: {
+        filename: "[name].bundle.js",
+        path: path.resolve(__dirname, "dist"),
+    },
+    module: {
+        rules: [
+            {
+                test: /\.(js|mjs)$/,
+                exclude: /node_modules/,
+                use: {
+                    loader: "babel-loader",
+                    options: {
+                        presets: ["@babel/preset-env"],
+                    },
+                },
+            },
+            {
+                test: /\.css$/,
+                use: ['style-loader', 'css-loader'],
+            },
+        ],
+    },
+    plugins: [
+        new CleanWebpackPlugin(),
+        new CopyPlugin({
+            patterns: [
+                {
+                    from: "assets",
+                    to: "assets",
+                },
+                { from: "src/style.css", to: "style.css" },
+                { from: "src/bibliography.bib", to: "bibliography.bib" },
+                { from: "src/index.html", to: "index.html" },
+                { 
+                    from: "../analysis/data",
+                    to: "data",
+                    globOptions: {
+                        ignore: ["**/*.json"],
+                    },
+                },
+            ],
+        }),
+    ],
+    devtool: process.env.NODE_ENV === 'production' ? 'source-map' : 'eval-source-map',
+    devServer: {
+        static: "./dist",
+        open: process.env.NODE_ENV !== 'production',
+        hot: process.env.NODE_ENV !== 'production',
+        liveReload: process.env.NODE_ENV !== 'production',
+        headers: {
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, PATCH, OPTIONS",
+            "Access-Control-Allow-Headers": "X-Requested-With, content-type, Authorization"
+        }
+    },
+    mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
+};
+
+console.log(process.env.NODE_ENV)

nginx.conf

@@ -0,0 +1,31 @@
+worker_processes auto;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections 1024;
+    use epoll;
+    multi_accept on;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    access_log /tmp/access.log;
+    error_log /tmp/error.log;
+
+    sendfile on;
+    keepalive_timeout 65;
+
+    server {
+        listen 8080;
+        server_name localhost;
+
+        root /usr/share/nginx/html;
+        index index.html;
+
+        location / {
+            try_files $uri $uri/ /index.html;
+        }
+    }
+}