Upload 6 files

app.py

@@ -0,0 +1,398 @@
+# app_final.py (final debugged version)
+import streamlit as st
+import requests
+import yfinance as yf
+import pandas as pd
+import numpy as np
+import os
+from datetime import datetime, timedelta
+import joblib
+import re
+import time
+
+# ---------------------------- CONFIG ----------------------------
+HF_API_TOKEN = st.secrets["HF_API_TOKEN"]
+CRYPTO_NEWS_API_KEY = "of9jvyylshwcddtw0qsv16zswpi8k39lbr67qm97"
+FRED_API_KEY = "4c3fd5be0b1f052f5d1d0080261277b1"
+
+FINBERT_API = "https://api-inference.huggingface.co/models/ProsusAI/finbert"
+HEADERS = {"Authorization": f"Bearer {HF_API_TOKEN}"}
+
+TICKERS = {
+    "bitcoin": "BTC-USD",
+    "gold": "GC=F",
+    "sp500": "^GSPC",
+    "dxy": "DX-Y.NYB"
+}
+
+FRED_CODES = {
+    "interest_rate": "FEDFUNDS",
+    "inflation": "CPIAUCSL"
+}
+
+model = joblib.load("histgb_pca_model_clean.pkl")
+pca = joblib.load("pca.pkl")
+scaler = joblib.load("scaler.pkl")
+
+# ---------------------------- FUNCTIONS ----------------------------
+def fetch_news(source):
+    url = f"https://cryptonews-api.com/api/v1/category"
+    params = {
+        "section": "general",
+        "items": 10,
+        "page": 1,
+        "source": source,
+        "token": CRYPTO_NEWS_API_KEY
+    }
+    r = requests.get(url, params=params)
+    articles = r.json().get("data", [])
+    texts = []
+    for art in articles:
+        summary = art.get("text") or art.get("content", "").split(".")[0]
+        texts.append(summary)
+    return texts
+
+def call_finbert(news_list):
+    results_df = []
+    news_list = news_list[:5]
+    for idx, news in enumerate(news_list):
+        if not isinstance(news, str) or not news.strip():
+            results_df.append({"positive": 0.0, "neutral": 0.0, "negative": 0.0})
+            continue
+        payload = {"inputs": news}
+        for attempt in range(5):
+            try:
+                response = requests.post(FINBERT_API, headers=HEADERS, json=payload, timeout=30)
+                response.raise_for_status()
+                output = response.json()
+                
+                # Get raw scores
+                scores_raw = {item["label"].lower(): item["score"] for item in output[0]}
+                
+                # Ensure fixed column order
+                aligned_scores = {
+                    "positive": scores_raw.get("positive", 0.0),
+                    "neutral": scores_raw.get("neutral", 0.0),
+                    "negative": scores_raw.get("negative", 0.0)
+                }
+                
+                results_df.append(aligned_scores)
+                break
+            except requests.exceptions.RequestException as e:
+                st.warning(f"⚠️ FinBERT error on article {idx+1}, attempt {attempt+1}/5: {e}")
+                time.sleep(2)
+            except Exception as ex:
+                st.warning(f"❌ Failed to analyze article {idx+1}: {ex}")
+                results_df.append({"positive": 0.0, "neutral": 0.0, "negative": 0.0})
+                break
+    return pd.DataFrame(results_df)
+
+def aggregate_sentiments(sentiment_df):
+    scaled = sentiment_df.copy()
+    for col in scaled.columns:
+        scaled[col] = (scaled[col] - scaled[col].min()) / (scaled[col].max() - scaled[col].min() + 1e-8)
+    weighted = scaled.copy()
+    for col in ["positive", "negative"]:
+        weighted[col] = np.where(scaled[col] > 0.75, scaled[col] * 1.5, scaled[col])
+        weighted[col] = np.clip(weighted[col], 0, 1)
+    weighted["neutral"] = scaled["neutral"]
+    return weighted.mean().to_dict(), (scaled > 0.75).sum().to_dict()
+
+def fetch_yahoo_data(ticker, date):
+    data = yf.Ticker(ticker).history(start=date, end=date + timedelta(days=1))
+    if not data.empty:
+        return {
+            "open": round(data["Open"].iloc[0], 2),
+            "high": round(data["High"].iloc[0], 2),
+            "low": round(data["Low"].iloc[0], 2),
+            "close": round(data["Close"].iloc[0], 2),
+            "volume": int(data["Volume"].iloc[0]) if ticker != TICKERS["dxy"] else None,
+            "change_pct": round(((data["Close"].iloc[0] - data["Open"].iloc[0]) / data["Open"].iloc[0]) * 100, 2)
+        }
+    else:
+        st.warning(f"⚠️ No trading data for {ticker} on {date.strftime('%Y-%m-%d')}, using previous available data.")
+        return fetch_yahoo_data(ticker, date - timedelta(days=1))
+
+def fetch_fred(code, month):
+    url = f"https://api.stlouisfed.org/fred/series/observations"
+    params = {
+        "series_id": code,
+        "observation_start": f"{month}-01",
+        "api_key": FRED_API_KEY,
+        "file_type": "json"
+    }
+    res = requests.get(url, params=params).json()
+    try:
+        return float(res["observations"][0]["value"])
+    except:
+        prev_month = (datetime.strptime(month, "%Y-%m") - timedelta(days=30)).strftime("%Y-%m")
+        return fetch_fred(code, prev_month)
+
+def make_prediction(input_data):
+    expected_cols = list(scaler.feature_names_in_)
+
+    # SAFETY CHECK
+    if len(input_data) != len(expected_cols):
+        raise ValueError(f"❌ Input length mismatch! Got {len(input_data)}, expected {len(expected_cols)}")
+
+    # Align input values to expected column order
+    input_dict = dict(zip(expected_cols, input_data))
+    input_df = pd.DataFrame([input_dict])[expected_cols]
+
+    # DEBUG VIEW
+    st.write("📄 Aligned Input DataFrame:")
+    st.dataframe(input_df)
+
+    # Transform
+    x_scaled = scaler.transform(input_df)
+    x_pca = pca.transform(x_scaled)
+    proba = model.predict_proba(x_pca)[0][1]
+    prediction = "Increase" if proba >= 0.62 else "Decrease"
+    return prediction, round(proba, 4)
+
+
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
+def log_prediction(record):
+    try:
+        scope = ["https://spreadsheets.google.com/feeds",
+                 "https://www.googleapis.com/auth/drive"]
+
+        creds = ServiceAccountCredentials.from_json_keyfile_name("creds.json", scope)
+        client = gspread.authorize(creds)
+
+        sheet = client.open("BTC Predictions Log").sheet1  # Must match your actual Google Sheet name
+        sheet.append_row(list(record.values()))
+        st.success("✅ Logged to Google Sheet successfully.")
+    except Exception as e:
+        st.warning(f"⚠️ Logging to Google Sheets failed: {e}")
+
+# ---------------------------- STREAMLIT UI ----------------------------
+st.set_page_config(page_title="Next Day Bitcoin Price Movement", layout="wide")
+st.title("🔮 Next Day Bitcoin Price Movement Predictor")
+
+date = st.date_input("Select a date", datetime.today() - timedelta(days=1))
+month = date.strftime("%Y-%m")
+
+if "news_loaded" not in st.session_state:
+    st.session_state.news_loaded = False
+
+sentiment_features = []
+aggregated_display = {}
+news_by_source = {"CryptoNews": [], "CryptoPotato": []}
+edited_news_by_source = {}
+
+# ------------------------------------
+# STEP 1: FETCH NEWS + ENABLE EDITING
+# ------------------------------------
+if not st.session_state.news_loaded:
+    if st.button("📥 Fetch News"):
+        for src in ["CryptoNews", "CryptoPotato"]:
+            try:
+                news = fetch_news(src)
+                news_by_source[src] = news
+                st.session_state[src] = "\n\n".join(news)  # store for text_area default
+            except Exception as e:
+                st.warning(f"⚠️ Could not fetch {src}: {e}")
+                st.session_state[src] = ""
+        st.session_state.news_loaded = True
+        st.rerun()
+
+# ------------------------------------
+# STEP 2: SHOW TEXT BOXES + RUN PREDICTION
+# ------------------------------------
+if st.session_state.news_loaded:
+    st.subheader("📝 Edit News Articles")
+    for src in ["CryptoNews", "CryptoPotato"]:
+        default_text = st.session_state.get(src, "")
+        user_input = st.text_area(f"{src} Articles (5 max, one per paragraph)", default_text, height=300)
+        edited_news_by_source[src] = [para.strip() for para in user_input.split("\n\n") if para.strip()]
+
+    if st.button("🔮 Make Prediction"):
+        for src in ["CryptoNews", "CryptoPotato"]:
+            try:
+                news_by_source[src] = edited_news_by_source[src]
+                scores_df = call_finbert(news_by_source[src])
+                st.write(f"📊 FinBERT Scores for {src}:", scores_df)
+
+                weighted_avg, extreme_count = aggregate_sentiments(scores_df)
+                total_articles = len(scores_df)
+
+                pct_scores = {
+                    "positive_pct": extreme_count.get("positive", 0) / total_articles,
+                    "neutral_pct": extreme_count.get("neutral", 0) / total_articles,
+                    "negative_pct": extreme_count.get("negative", 0) / total_articles
+                }
+
+                sentiment_features.extend([
+                    weighted_avg["positive"],
+                    weighted_avg["neutral"],
+                    weighted_avg["negative"],
+                    pct_scores["positive_pct"],
+                    pct_scores["neutral_pct"],
+                    pct_scores["negative_pct"]
+                ])
+            except Exception as e:
+                st.warning(f"⚠️ Failed for {src}: {e}")
+                sentiment_features.extend([0.0] * 6)
+                news_by_source[src] = []
+
+        st.markdown("**Aggregated Sentiment**")
+        st.write("🔎 News by Source:", news_by_source)
+        sentiment_feature_labels = {
+	    "cryptonews_positive_weighted": sentiment_features[0],
+            "cryptonews_neutral_weighted": sentiment_features[1],
+            "cryptonews_negative_weighted": sentiment_features[2],
+            "cryptonews_positive_pct": sentiment_features[3],
+            "cryptonews_neutral_pct": sentiment_features[4],
+            "cryptonews_negative_pct": sentiment_features[5],
+            "cryptopotato_positive_weighted": sentiment_features[6],
+            "cryptopotato_neutral_weighted": sentiment_features[7],
+            "cryptopotato_negative_weighted": sentiment_features[8],
+            "cryptopotato_positive_pct": sentiment_features[9],
+            "cryptopotato_neutral_pct": sentiment_features[10],
+            "cryptopotato_negative_pct": sentiment_features[11],
+        }
+        st.markdown("### 🧠 Sentiment Features by Source")
+        st.json(sentiment_feature_labels)	
+
+        # Average across both sources
+        if len(sentiment_features) == 12:
+            aggregated_sentiments = [
+                (sentiment_features[0] + sentiment_features[6]) / 2,
+                (sentiment_features[1] + sentiment_features[7]) / 2,
+                (sentiment_features[2] + sentiment_features[8]) / 2,
+                (sentiment_features[3] + sentiment_features[9]) / 2,
+                (sentiment_features[4] + sentiment_features[10]) / 2,
+                (sentiment_features[5] + sentiment_features[11]) / 2
+            ]
+        elif len(sentiment_features) == 6:
+            aggregated_sentiments = sentiment_features
+        else:
+            st.warning("⚠️ Sentiment features incomplete. Defaulting to 0s.")
+            aggregated_sentiments = [0.0] * 6
+
+        # Fetch BTC + macro data
+        st.subheader("📈 Bitcoin Price Data")
+        btc = fetch_yahoo_data(TICKERS["bitcoin"], date)
+        st.json(btc)
+
+        st.subheader("📊 Macroeconomic Indicators")
+        macro = {}
+        for k, t in TICKERS.items():
+            if k != "bitcoin":
+                try:
+                    macro[k] = fetch_yahoo_data(t, date)
+                except Exception as e:
+                    st.warning(f"⚠️ Failed to fetch {k.upper()} data: {e}")
+                    macro[k] = {"open": 0, "high": 0, "low": 0, "close": 0, "volume": 0, "change_pct": 0}
+        st.json(macro)
+
+        st.subheader("🏩 Fed Indicators")
+        fed = {
+            "interest_rate": fetch_fred(FRED_CODES["interest_rate"], month),
+            "inflation": fetch_fred(FRED_CODES["inflation"], month)
+        }
+        st.json(fed)
+
+        # ========== BUILD FINAL INPUT DICT SAFELY ==========
+        final_input_dict = {
+            "S&P_500_Open": macro["sp500"].get("open", 0),
+            "S&P_500_High": macro["sp500"].get("high", 0),
+            "S&P_500_Low": macro["sp500"].get("low", 0),
+            "S&P_500_Close": macro["sp500"].get("close", 0),
+            "S&P_500_Volume": macro["sp500"].get("volume", 0),
+            "S&P_500_%_Change": macro["sp500"].get("change_pct", 0),
+
+            "Gold_Prices_Open": macro["gold"].get("open", 0),
+            "Gold_Prices_High": macro["gold"].get("high", 0),
+            "Gold_Prices_Low": macro["gold"].get("low", 0),
+            "Gold_Prices_Close": macro["gold"].get("close", 0),
+            "Gold_Prices_Volume": macro["gold"].get("volume", 0),
+            "Gold_Prices_%_Change": macro["gold"].get("change_pct", 0),
+
+            "US_Dollar_Index_DXY_Open": macro["dxy"].get("open", 0),
+            "US_Dollar_Index_DXY_High": macro["dxy"].get("high", 0),
+            "US_Dollar_Index_DXY_Low": macro["dxy"].get("low", 0),
+            "US_Dollar_Index_DXY_Close": macro["dxy"].get("close", 0),
+            "US_Dollar_Index_DXY_%_Change": macro["dxy"].get("change_pct", 0),
+
+            "Federal_Reserve_Interest_Rates_FEDFUNDS": fed.get("interest_rate", 0),
+            "Inflation_CPIAUCNS": fed.get("inflation", 0),
+
+            "Open": btc.get("open", 0),
+            "High": btc.get("high", 0),
+            "Low": btc.get("low", 0),
+            "Close": btc.get("close", 0),
+            "Volume": btc.get("volume", 0),
+            "Change %": btc.get("change_pct", 0),
+
+            "positive_weighted": aggregated_sentiments[0],
+            "neutral_weighted": aggregated_sentiments[1],
+            "negative_weighted": aggregated_sentiments[2],
+            "negative_pct": aggregated_sentiments[5],
+            "neutral_pct": aggregated_sentiments[4],
+            "positive_pct": aggregated_sentiments[3],
+        }
+
+        # ========== PREPARE & PREDICT ==========
+        expected_cols = list(scaler.feature_names_in_)
+        final_input = [final_input_dict[col] for col in expected_cols]
+
+        if any(pd.isna(x) for x in final_input):
+            st.error("❌ Missing or invalid input data. Please check news, market, or macro feeds.")
+        else:
+            # Prepare aligned input
+            input_df = pd.DataFrame([final_input_dict])[expected_cols]
+            x_scaled = scaler.transform(input_df)
+            x_pca = pca.transform(x_scaled)
+
+            # Model prediction
+            proba = model.predict_proba(x_pca)[0][1]
+            prediction = "Increase" if proba >= 0.62 else "Decrease"
+
+            # PCA features table
+            pca_df = pd.DataFrame(x_pca, columns=[f"PC{i+1}" for i in range(x_pca.shape[1])])
+            st.markdown("### 🧬 PCA-Transformed Features")
+            st.dataframe(pca_df.style.format("{:.4f}"))
+
+            # Prediction display
+            st.subheader("🔮 Prediction")
+            if prediction == "Decrease":
+                st.markdown(
+                    f"<div style='background-color:#fbeaea;color:#9e1c1c;padding:10px;border-radius:8px;'>"
+                    f"<b>Next Day BTC Price:</b> {prediction} (Prob: {proba:.2f})</div>",
+                    unsafe_allow_html=True
+                )
+            else:
+                st.success(f"Next Day BTC Price: **{prediction}** (Prob: {proba:.2f})")
+
+            # Log prediction
+            log = {
+                "fetch_date": datetime.today().strftime("%Y-%m-%d"),
+                "btc_open": btc["open"],
+                "btc_close": btc["close"],
+                "sent_pos": aggregated_sentiments[0],
+                "sent_neu": aggregated_sentiments[1],
+                "sent_neg": aggregated_sentiments[2],
+                "sent_pos_pct": aggregated_sentiments[3],
+                "sent_neu_pct": aggregated_sentiments[4],
+                "sent_neg_pct": aggregated_sentiments[5],
+                "macro_gold": macro["gold"]["close"],
+                "macro_sp500": macro["sp500"]["close"],
+                "macro_dxy": macro["dxy"]["close"],
+                "interest_rate": fed["interest_rate"],
+                "inflation": fed["inflation"],
+                "prediction": prediction,
+                "prob": proba,
+                "news_cryptonews": " || ".join(news_by_source["CryptoNews"]),
+                "news_cryptopotato": " || ".join(news_by_source["CryptoPotato"])
+            }
+
+            log_prediction(log)
+            st.success("✅ Logged to predictions_log.csv")
+
+
+
+

creds.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "bitcoin-predictor-log",
+  "private_key_id": "c66f0bb452bb5ffcaa9b23ac93eb42347000d94f",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCagBsCFFxFDHOJ\nuJGm7iQMU8+j/xnCpvep4/+lMvtCVlZsHD+C8SN23IIVC73oNHSMIOkUT7vjxlRP\nZ9bQSgLZ1KXspfH0PdjYJH1pxJdnElgKIUTK5NTcj33ZAKWa/XpkAsaZMHs7QUuQ\nlkhxsATVNOK7c/QtBMqqcMh14SHDUHBm8zncDoRg8Rmf0uaMEs+zXPPbOevBtEFo\nQH82k5jGSR8oevGfCFdOnXngfyyhX9Unm21C9TqpdL0ROGIj5V8yTMHTBKkM62MQ\nLfeKABufx+uGb4H4Paa2ZlJacFcXY7bsj6OsXxdeulNZLB5AQg6QQm1jtx40t0oK\nsxr2k+ItAgMBAAECggEAELFKGbuq62/otMbK/9LndKiCfOjFNv10sTeXyZi8QmLT\nJAuaRhK5HvC8ojr6SfIO7Ivqut3Rgk7NkaW5tRfl/nSF989HGLks6k9o+Go75HA8\nMF3/UX+PSwQ6190Ex33fARq2q9rr2Y9Ys3a1PYlDwGheHdwtk8aagfc9BVVtYS1u\n+CHntFZb8J6ozTDQREe3qL9v0YzEKaWeReBVE1d+j++EtpeFij9v/sl2d1oQMTLH\nPzrKeMzbMU3OqTWENhAIVjihuqZFZY99dAMdmyh1QtGrUV7Iic8ixM9fFB/aF9EQ\nsu7pmOQu2jPdB6Dkxwwy3S8yimOb2MyGPrBTAHx8wQKBgQDKiZ0ZcZbB7HsGLUAK\n0s7AJ3DoVh2KTxwsmmfoZIJgfu16JlSRWUznQdORJ/kTR4b6E6o36N2GrSjmIC5O\n0UKoaeoGoo3azjf+5C+dJ18nQ3wd18gDvMb2/yehsBPASwVnPie28ZN1664Tx8wP\ni0rJuadlqVzgY2n3d8ueHK8YwQKBgQDDSGcwNzLZMBAIeumA9rzmW+WESPlbgy3D\nYnkLP9jy9I1zazAbGW9S8AU4XdRc0tkHjCdzY7dC9hGUAEyTRS1n4C93U5HpJAzP\nQH/ob1Wsz2DCTR18dRZeTFKECHcHMjWjFJVMHvpjNGuyS4TLLoSJRhE4PnSEZOzJ\n8FVqzIRYbQKBgQDJKlfMPsLzR/OFVgpm9az+s+30Bhx/FEYykwYjjzjjqQ4sJcgX\nObAlfX8qjJ5apM+OsVt+/p6QtoqJz2rPRA9GATS9dFUa/3okg/Y6zDE5aVDsvzbd\nZ3HjP9jYQm/LrXfnbJe7oEPLetPCt86Zncshg3GditNB19wXPHgUSf8rwQKBgQCD\nql6AgMcU7rXwscacQEAO5SjzrywJSoHheZR5RDwnW0G/7yZJLzYC6nfqkEDtsO/J\nifLTdwkJ6dTiP+1hYkQCBIKcZsk7MyY72pYjBmXylQP9HXdjAaLqQ3VjNj3iqTTG\n1sruvvg9SQvP8+D+CUgtMgPMMzfmSBHq+dLtpiUZZQKBgQCYzwgY9k6xD1EEMljJ\n66UAEiQiIJOomatjmxc4MNrK1Zsows6f40qnJ22PhqNuKHqCGyj+qnQP9o1MWJGC\n7McvxUR16Z2oEtgRPUDMXe+I2PBaRL76m1SZ3cYV4Hm5HJXacoJU35bqo5NKQQi7\nlf6mgxpvZipUt/s3tChvhDpeeg==\n-----END PRIVATE KEY-----\n",
+  "client_email": "btc-predictor-logger@bitcoin-predictor-log.iam.gserviceaccount.com",
+  "client_id": "114024114053631353428",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/btc-predictor-logger%40bitcoin-predictor-log.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

histgb_pca_model_clean.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3465f30c06057d147bba5342686788c87fab93a67be6a158e5e5c93faeb8a1f9
+size 33944

pca.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ed7c470f4c88de93c41296fde8af77202271d070be7eab95ae4425dabb7f5ed
+size 4223

requirements.txt

@@ -0,0 +1,13 @@
+streamlit
+numpy==1.24.4
+pandas
+scikit-learn
+matplotlib
+seaborn
+joblib
+optuna
+shap
+yfinance
+requests
+gspread
+oauth2client

scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec8c4153f48ee2b3843dfc80bc967528a93bb152aeafe43fbfd4257fc90f46b
+size 2319