<repo_name>ngpraveen/ml_product_template<file_sep>/data/download_data.py
#!/usr/bin/env python


def get_data(destin_dir=None, file_name=None):
    # import libraries
    import pandas as pd
    from sklearn.datasets import load_iris
    from pathlib import Path

    # load data
    data = load_iris()

    # load featuers and target variable
    X = data['data']
    y = data['target']
    # target_names = data['target_names']
    feature_names = data['feature_names']

    # convert to pandas dataframes and
    X = pd.DataFrame(X, columns=feature_names)
    y = pd.DataFrame(y)

    # merget them to a single dataframe
    df = pd.concat([X, y], axis=1).rename(columns={0: 'target'})
    df.head()

    # clean up the column names
    df.columns = df.columns.str.strip(" (cm)").str.replace(" ", "_")
    df.head()

    # if a path is provided as an argument, file is written to that path
    # else the file is written to default location
    if not destin_dir:
        destin_dir = "data"
    destin_dir = Path(destin_dir)

    if not file_name:
        file_name = "train.csv"
    file_name = Path(file_name)

    file_full_path = Path.joinpath(destin_dir, file_name)
    print(file_full_path)

    df.to_csv(file_full_path, index=False)


if __name__ == "__main__":
    get_data()
<file_sep>/README.md
## A Machine Learning App Template

This app is created using a rather simple Random Forest ML model. The goal is to predict the class of Iris plant based on 4 features. The ML model was trained using the famous Iris data set. The app (web UI) is created using Gradio. The code is in the folder "gradioapp". I'm planning to switch from Gradio to FastAPI. The basic code using FastAPI is in "app" folder. However, it is missing HTML/CSS files, and adding them is in my to do list.  


This app is hosted here: [https://warm-spire-39227.herokuapp.com/](https://warm-spire-39227.herokuapp.com/ "here")


#### To do:

* Moving hard coded config params to config/config.yml file
* Currently the Web UI is run using Gradio. Replace it with FastAPI (partly done)
* Add more documentation (this document)


If you have any questions/comments, please do not hesitate to let me know. <file_sep>/src/pipeline.py
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import RandomForestClassifier
from config.core import config
from src.features import multiplyByValue

pipe = Pipeline([
    ("multiply_variable",
     multiplyByValue(config.multiply_variables, config.multiply_value)),
    ('scaler', MinMaxScaler()),
    ('model', RandomForestClassifier(random_state=config.random_state))
    ])
<file_sep>/tox.ini


[tox]
envlist = [train]
skipsdist = True


[testenv]
deps = -rrequirements.txt

setenv = 
        PYTHONDONTWRITEBYTECODE=1


[testenv:train]
envdir = {toxworkdir}/testenv
commands = python train.py


[testenv:predict]
envdir = {toxworkdir}/testenv
commands = python predict.py
<file_sep>/gradioapp/mainapp.py
import gradio as gr
import sys


sys.path.insert(0, "../")

from app.app import make_prediction


def xyz(Sepal_Length, Sepal_Width, Petal_Length, Petal_Width):
    sl = float(Sepal_Length)
    sw = float(Sepal_Width)
    pl = float(Petal_Length)
    pw = float(Petal_Width)
    output = make_prediction(sl, sw, pl, pw)
    output = output.capitalize()
    return output


Sepal_Length = gr.inputs.Slider(4.3, 7.9)
Sepal_Width = gr.inputs.Slider(2.0, 4.4)
Petal_Length = gr.inputs.Slider(1, 6.9)
Petal_Width = gr.inputs.Slider(0.1, 2.5)

"""
sl1 = 1.2
sw1 = 3.4
pl1 = 5.1
pw1 = 0.2
"""

# predict = make_prediction(sl1, sw1, pl1, pw1)

face = gr.Interface(
    fn=xyz,
    inputs=[Sepal_Length, Sepal_Width, Petal_Length, Petal_Width],
    outputs="text",
)
face.launch()
<file_sep>/src/features.py
from sklearn.base import BaseEstimator, TransformerMixin


class multiplyByValue(BaseEstimator, TransformerMixin):
    """ A custom transformer usable within pipeline. \
        This particular one is for demo purpose only - \
        it doesn't have any consequence. In this case \
        this transformer only multiplies features \
        (listed in yml file) by a constant (also specified \
        in yml file). More similar transformer classes can \
        be added."""

    def __init__(self, variables, multiply_value):
        self.variables = variables
        self.multiply_value = multiply_value

    def fit(self, X, y):
        return self

    def transform(self, X):
        for var in self.variables:
            X[var] = X[var] * self.multiply_value
        return X
<file_sep>/src/data_manager.py
import pandas as pd
from pathlib import Path
from config.core import DATADIR, ROOT, config
from data.download_data import get_data
import joblib

download_new_data = config.download_new_data
train_data_file_name = config.train_file_name

file_full_dir = ROOT / config.pipeline_save_dir
file_full_path = file_full_dir / config.pipeline_save_file


def load_data():
    file_full_path = Path.joinpath(DATADIR, train_data_file_name)

    if download_new_data:
        get_data(DATADIR, train_data_file_name)

    df = pd.read_csv(file_full_path)
    return df


def save_pipeline(pipe_name):
    if not file_full_dir.is_dir():
        print(f"The output directory {file_full_dir} \
                does not exist. Creating it.")
        file_full_dir.mkdir(exist_ok=True)
    joblib.dump(pipe_name, file_full_path)


def load_pipeline():
    pipe = joblib.load(file_full_path)
    return pipe
<file_sep>/app/app.py
import sys

import joblib
import pandas as pd
from fastapi import FastAPI, Form, Request
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates

sys.path.insert(0, "../")


app = FastAPI()
templates = Jinja2Templates(directory="templates")


def make_prediction(sl, sw, pl, pw):
    target = ["setosa", "versicolor", "virginica"]
    target_dict = {i: t for i, t in enumerate(target)}

    [sl, sw, pl, pw] = [float(var) for var in [sl, sw, pl, pw]]

    data = pd.DataFrame(
        {
            "sepal_length": [sl],
            "sepal_width": [sw],
            "petal_length": [pl],
            "petal_width": [pw],
        }
    )

    model = joblib.load("../trained_models/model_output.joblib")
    prediction_ = model.predict(data)[0]
    output = target_dict.get(prediction_, "None")
    print(f"Prediction is {output}")
    return output


@app.get("/", response_class=HTMLResponse)
def p1(request: Request):
    return templates.TemplateResponse("home.html", {"request": request})


@app.post("/predict/")
async def predict_func(
    sepal_length: str = Form(...),
    sepal_width: str = Form(...),
    petal_length: str = Form(...),
    petal_width: str = Form(...),
):

    prediction = make_prediction(
        sepal_length, sepal_width, petal_length, petal_width
    )
    return {"Prediction": prediction}
<file_sep>/train.py
from sklearn.model_selection import train_test_split

from config.core import config
from src.data_manager import load_data, save_pipeline
from src.pipeline import pipe

# load data from local file
data = load_data()


X_train, X_test, y_train, y_test = \
    train_test_split(data[config.features],
                     data[config.target],
                     test_size=config.test_size,
                     random_state=config.random_state)


print(y_test.values)
pipe.fit(X_train, y_train)
print(pipe.predict(X_test))
save_pipeline(pipe)
<file_sep>/predict.py
from config.core import config
from src.data_manager import load_data, load_pipeline

TARGET = config.target

pipe = load_pipeline()
data = load_data()
print(pipe.predict(data.drop(TARGET, axis=1).sample(2)))
<file_sep>/config/core.py
from pathlib import Path
from typing import List
from pydantic import BaseModel
from strictyaml import load
import src


ROOT = Path(src.__file__).parent.parent
DATADIR = ROOT / "data"
CONFIG_FILE = ROOT / "config/config.yml"


class Config(BaseModel):
    download_new_data: bool
    train_file_name: str
    pipeline_save_file: str
    pipeline_save_dir: str

    features: List[str]
    target: str
    random_state: int
    test_size: float

    multiply_variables: List[str]
    multiply_value: int


def read_config():
    """Reads config file (yml) and returns it as text (str)"""

    if CONFIG_FILE.is_file():
        with open(CONFIG_FILE, "r") as f:
            config_txt = f.read()
        return config_txt
    else:
        raise Exception(f"yml file is not found at {CONFIG_FILE}!")


def create_config():
    """Creates and validates Config object based on config file"""
    config_txt = read_config()
    config_yml = load(config_txt)
    config_ = Config(**config_yml.data)
    return config_


config = create_config()
<file_sep>/requirements.txt
analytics-python==1.4.0
asgiref==3.4.1
backoff==1.10.0
bcrypt==3.2.0
certifi==2021.5.30
cffi==1.14.6
charset-normalizer==2.0.3
click==8.0.1
cryptography==3.4.7
cycler==0.10.0
fastapi==0.67.0
ffmpy==0.3.0
Flask==2.0.1
Flask-CacheBuster==1.0.0
Flask-Cors==3.0.10
Flask-Login==0.5.0
gradio==2.2.3
h11==0.12.0
idna==3.2
itsdangerous==2.0.1
Jinja2==3.0.1
joblib==1.0.1
kiwisolver==1.3.1
markdown2==2.4.0
MarkupSafe==2.0.1
matplotlib==3.4.2
monotonic==1.6
numpy==1.21.0
pandas==1.3.0
paramiko==2.7.2
Pillow==8.3.1
pycparser==2.20
pycryptodome==3.10.1
pydantic==1.8.2
PyNaCl==1.4.0
pyparsing==2.4.7
python-dateutil==2.8.2
python-multipart==0.0.5
pytz==2021.1
requests==2.26.0
scikit-learn==0.24.2
scipy==1.7.0
six==1.16.0
starlette==0.14.2
strictyaml==1.4.4
threadpoolctl==2.2.0
typing-extensions==3.10.0.0
urllib3==1.26.6
uvicorn==0.14.0
Werkzeug==2.0.1
