Swap test dataset

app/cli.py

@@ -141,13 +141,13 @@ def evaluate(
     import joblib
     import pandas as pd
 
-    from app.constants import TOKENIZER_CACHE_PATH
+    from app.constants import TOKENIZER_CACHE_DIR
     from app.data import load_data, tokenize
     from app.model import evaluate_model
     from app.utils import deserialize, serialize
 
-    token_cache_path = TOKENIZER_CACHE_PATH / f"{dataset}_tokenized.pkl"
-    label_cache_path = TOKENIZER_CACHE_PATH / f"{dataset}_labels.pkl"
+    token_cache_path = TOKENIZER_CACHE_DIR / f"{dataset}_tokenized.pkl"
+    label_cache_path = TOKENIZER_CACHE_DIR / f"{dataset}_labels.pkl"
     use_cached_data = False
 
     if token_cache_path.exists():
@@ -168,8 +168,6 @@ def evaluate(
 
         click.echo("Tokenizing data... ")
         token_data = tokenize(text_data, batch_size=token_batch_size, n_jobs=token_jobs, show_progress=True)
-
-        click.echo("Caching tokenized data... ")
         serialize(token_data, token_cache_path, show_progress=True)
         joblib.dump(label_data, label_cache_path, compress=3)
 
@@ -184,7 +182,13 @@ def evaluate(
     model = joblib.load(model_path)
     click.echo(DONE_STR)
 
-    click.echo("Evaluating model... ", nl=False)
+    if cv == 1:
+        click.echo("Evaluating model... ", nl=False)
+        acc = model.score(token_data, label_data)
+        click.secho(f"{acc:.2%}", fg="blue")
+        return
+
+    click.echo("Evaluating model... ")
     acc_mean, acc_std = evaluate_model(
         model,
         token_data,
@@ -282,7 +286,7 @@ def train(
     import joblib
     import pandas as pd
 
-    from app.constants import MODEL_DIR, TOKENIZER_CACHE_PATH
+    from app.constants import MODEL_DIR, TOKENIZER_CACHE_DIR
     from app.data import load_data, tokenize
     from app.model import train_model
     from app.utils import deserialize, serialize
@@ -291,8 +295,8 @@ def train(
     if model_path.exists() and not overwrite:
         click.confirm(f"Model file '{model_path}' already exists. Overwrite?", abort=True)
 
-    token_cache_path = TOKENIZER_CACHE_PATH / f"{dataset}_tokenized.pkl"
-    label_cache_path = TOKENIZER_CACHE_PATH / f"{dataset}_labels.pkl"
+    token_cache_path = TOKENIZER_CACHE_DIR / f"{dataset}_tokenized.pkl"
+    label_cache_path = TOKENIZER_CACHE_DIR / f"{dataset}_labels.pkl"
     use_cached_data = False
 
     if token_cache_path.exists():
@@ -313,8 +317,6 @@ def train(
 
         click.echo("Tokenizing data... ")
         token_data = tokenize(text_data, batch_size=token_batch_size, n_jobs=token_jobs, show_progress=True)
-
-        click.echo("Caching tokenized data... ")
         serialize(token_data, token_cache_path, show_progress=True)
         joblib.dump(label_data, label_cache_path, compress=3)
 

app/constants.py

@@ -4,10 +4,16 @@ import os
 from pathlib import Path
 
 CACHE_DIR = Path(os.getenv("CACHE_DIR", ".cache"))
+CACHE_DIR.mkdir(exist_ok=True, parents=True)
+
 DATA_DIR = Path(os.getenv("DATA_DIR", "data"))
+DATA_DIR.mkdir(exist_ok=True, parents=True)
+
 MODEL_DIR = Path(os.getenv("MODEL_DIR", "models"))
+MODEL_DIR.mkdir(exist_ok=True, parents=True)
 
-TOKENIZER_CACHE_PATH = CACHE_DIR / "tokenizer"
+TOKENIZER_CACHE_DIR = CACHE_DIR / "tokenizer"
+TOKENIZER_CACHE_DIR.mkdir(exist_ok=True, parents=True)
 
 SENTIMENT140_PATH = DATA_DIR / "sentiment140.csv"
 SENTIMENT140_URL = "https://www.kaggle.com/datasets/kazanova/sentiment140"
@@ -19,13 +25,7 @@ IMDB50K_PATH = DATA_DIR / "imdb50k.csv"
 IMDB50K_URL = "https://www.kaggle.com/datasets/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews"
 
 TEST_DATASET_PATH = DATA_DIR / "test.csv"
-TEST_DATASET_URL = "https://huggingface.co/datasets/Sp1786/multiclass-sentiment-analysis-dataset"
+TEST_DATASET_URL = "https://github.com/Tymec/sentiment-analysis/blob/main/data/test.csv?raw=true"
 
 SLANGMAP_PATH = DATA_DIR / "slang.json"
-SLANGMAP_URL = "Https://www.kaggle.com/code/nmaguette/up-to-date-list-of-slangs-for-text-preprocessing"
-
-CACHE_DIR.mkdir(exist_ok=True, parents=True)
-DATA_DIR.mkdir(exist_ok=True, parents=True)
-MODEL_DIR.mkdir(exist_ok=True, parents=True)
-
-TOKENIZER_CACHE_PATH.mkdir(exist_ok=True, parents=True)
+SLANGMAP_URL = "https://github.com/Tymec/sentiment-analysis/blob/main/data/slang.json?raw=true"

app/data.py

@@ -55,7 +55,6 @@ def slang() -> tuple[Pattern, dict[str, str]]:
         FileNotFoundError: If the file is not found
     """
     if not SLANGMAP_PATH.exists():
-        # msg = f"Missing slang mapping file: {SLANG_PATH}"
         msg = (
             f"Slang mapping file not found at: '{SLANGMAP_PATH}'\n"
             "Please download the file from:\n"
@@ -89,7 +88,6 @@ def _clean(text: str) -> str:
     text = slang_pattern.sub(lambda x: slang_mapping[x.group()], text)
 
     # Remove acronyms and abbreviations
-    # text = re.sub(r"(?:[a-z]\.){2,}", "", text)
     text = re.sub(r"\b(?:[a-z]\.?)(?:[a-z]\.)\b", "", text)
 
     # Remove honorifics
@@ -161,15 +159,6 @@ def tokenize(
     Returns:
         Tokenized text data
     """
-    # text_data = [
-    #     _clean(text)
-    #     for text in tqdm(
-    #         text_data,
-    #         desc="Cleaning",
-    #         unit="doc",
-    #         disable=not show_progress,
-    #     )
-    # ]
     text_data = Parallel(n_jobs=n_jobs)(
         delayed(_clean)(text)
         for text in tqdm(
@@ -310,12 +299,9 @@ def load_imdb50k() -> tuple[list[str], list[int]]:
     return data["review"].tolist(), data["sentiment"].tolist()
 
 
-def load_test(include_neutral: bool = False) -> tuple[list[str], list[int]]:
+def load_test() -> tuple[list[str], list[int]]:
     """Load the test dataset and make it suitable for use.
 
-    Args:
-        include_neutral: Whether to include neutral sentiment
-
     Returns:
         Text and label data
 
@@ -334,21 +320,8 @@ def load_test(include_neutral: bool = False) -> tuple[list[str], list[int]]:
     # Load the dataset
     data = pd.read_csv(TEST_DATASET_PATH)
 
-    # Ignore rows with neutral sentiment
-    if not include_neutral:
-        data = data[data["label"] != 1]
-
-    # Map sentiment values
-    data["label"] = data["label"].map(
-        {
-            0: 0,  # Negative
-            1: 2,  # Neutral
-            2: 1,  # Positive
-        },
-    )
-
     # Return as lists
-    return data["text"].tolist(), data["label"].tolist()
+    return data["text"].tolist(), data["sentiment"].tolist()
 
 
 def load_data(dataset: Literal["sentiment140", "amazonreviews", "imdb50k", "test"]) -> tuple[list[str], list[int]]:
@@ -371,7 +344,7 @@ def load_data(dataset: Literal["sentiment140", "amazonreviews", "imdb50k", "test
         case "imdb50k":
             return load_imdb50k()
         case "test":
-            return load_test(include_neutral=False)
+            return load_test()
         case _:
             msg = f"Unknown dataset: {dataset}"
             raise ValueError(msg)

app/utils.py

@@ -23,6 +23,7 @@ def serialize(data: Sequence[str | int], path: Path, max_size: int = 100_000, sh
     for i, chunk in enumerate(
         tqdm(
             [data[i : i + max_size] for i in range(0, len(data), max_size)],
+            desc="Serializing",
             unit="chunk",
             disable=not show_progress,
         ),