Create app.py

app.py

@@ -0,0 +1,92 @@
+from functools import lru_cache
+from typing import Dict, List
+
+import plotly.express as px
+import streamlit as st
+
+from datasets import Dataset, get_dataset_infos, load_dataset
+
+BASE_DATASET: str = "lion-ai/pl_med_data"
+
+dataset_names_map: Dict[str, str] = {
+    "znany_lekarz": "Porady - pytania i odpowiedzi",
+    "kor_epikryzy_qa": "Dokumentacja medyczna - pytania i odpowiedzi",
+    "wikipedia": "Ogólna wiedza medyczna - pytania i opowiedzi",
+}
+
+reverse_dataset_names_map: Dict[str, str] = {v: k for k, v in dataset_names_map.items()}
+
+
+@st.cache_resource
+def list_datasets() -> Dict[str, Dataset]:
+    """
+    Retrieves a list of dataset information.
+
+    Returns:
+        List[Dict[str, str]]: A list of dataset information.
+    """
+    return get_dataset_infos(BASE_DATASET)
+
+
+def show_examples(dataset_name: str, split: str) -> None:
+    dataset_name = reverse_dataset_names_map.get(dataset_name, dataset_name)
+
+    dataset: Dataset = load_dataset(BASE_DATASET, dataset_name, split=f"{split}[:10]")
+    st.data_editor(dataset.to_pandas(), use_container_width=True)
+
+
+def count_all_examples(datasets: Dict[str, Dataset]) -> None:
+    count: int = 0
+    for dataset_name, dataset_info in datasets.items():
+        count += dataset_info.num_examples
+    st.metric(label="Total no. of instructions", value=f"{count:,}")
+
+
+def filter_splits(dataset: Dict[str, Dataset], split: str) -> Dict[str, Dataset]:
+    """
+    Filter the dataset based on the specified split.
+
+    Args:
+        dataset (Dict[str, Dataset]): A dictionary containing dataset information.
+        split (str): The split to filter the dataset by.
+
+    Returns:
+        Dict[str, Dataset]: A dictionary containing the filtered dataset splits.
+    """
+
+    dataset_splits: Dict[str, Dataset] = {}
+    for dataset_name, dataset_info in dataset.items():
+        if split in dataset_info.splits:
+            dataset_name = dataset_names_map.get(dataset_name, dataset_name)
+            dataset_splits[dataset_name] = dataset_info.splits[split]
+    return dataset_splits
+
+
+split: str = st.selectbox("splits", ["raw", "processed"])
+
+datasets: Dict[str, Dataset] = list_datasets()
+# st.write(datasets)
+
+filtered_datasets: Dict[str, Dataset] = filter_splits(datasets, split)
+# st.write(filtered_datasets)
+
+count_all_examples(filtered_datasets)
+
+# Create a pie chart showing the number of examples per dataset
+fig = px.pie(
+    values=[split.num_examples for split in filtered_datasets.values()],
+    names=list(filtered_datasets.keys()),
+    # title=f"Number of Examples per Dataset ({split} split)",
+    labels={"label": "Dataset", "value": "Number of Examples"},
+)
+
+# Update layout for better readability
+fig.update_traces(textposition="inside", textinfo="value+label")
+fig.update_layout(legend_title_text="Datasets", uniformtext_minsize=12, uniformtext_mode="hide")
+
+chart = st.plotly_chart(fig, use_container_width=True)
+
+
+dataset_name = st.selectbox("Select a dataset", list(filtered_datasets.keys()))
+
+show_examples(dataset_name, split)