init

.env

@@ -0,0 +1,3 @@
+AIRFLOW_UID=1000
+URL_CMC=https://pro-api.coinmarketcap.com
+API_KEY_CMC=8057498e-ad35-465c-8359-8f6cc9d1ae1b

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+output/* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,136 @@
+#-------------------------------------libraries ----------------------------------
+
+import os
+import pandas as pd
+import streamlit as st
+import plotly.graph_objs as go
+import numpy as np
+import plotly.express as px 
+import logging
+# Set up logging basic configuration
+logging.basicConfig(level=logging.INFO)
+# Example of logging
+logging.info("Streamlit app has started")
+
+
+#-------------------------------------back ----------------------------------
+
+# etherscan
+## Load the data from the CSV files
+dataframes = []
+for filename in os.listdir('output'):
+    if filename.endswith('.csv'):
+        df_temp = pd.read_csv(os.path.join('output', filename), sep=';')
+        dataframes.append(df_temp)
+df_etherscan = pd.concat(dataframes)
+del df_temp
+
+# CMC
+## Load cmc data
+df_temp = pd.read_csv("output/top_100_update.csv", sep=',')
+df_cmc = df_temp[df_temp["last_updated"] == df_temp["last_updated"].max()]
+del df_temp
+#-------------------------------------streamlit ----------------------------------
+
+# Set the title and other page configurations
+st.title('Crypto Analysis')
+# Create two columns for the two plots
+col1, col2 = st.columns(2)
+
+with st.container():
+
+    with col1:
+        # etherscan
+        selected_token = st.selectbox('Select Token', df_etherscan['tokenSymbol'].unique(), index=0)
+        # Filter the data based on the selected token
+        filtered_df = df_etherscan[df_etherscan['tokenSymbol'] == selected_token]
+        # Plot the token value over time
+        st.plotly_chart(
+            go.Figure(
+                data=[
+                    go.Scatter(
+                        x=filtered_df['timeStamp'],
+                        y=filtered_df['value'],
+                        mode='lines',
+                        name='Value over time'
+                    )
+                ],
+                layout=go.Layout(
+                    title='Token Value Over Time',
+                    yaxis=dict(
+                        title=f'Value ({selected_token})',
+                    ),
+                    showlegend=True,
+                    legend=go.layout.Legend(x=0, y=1.0),
+                    margin=go.layout.Margin(l=40, r=0, t=40, b=30),
+                    width=500,
+                    height=500
+
+                )
+            )
+        )
+
+    with col2:
+        # cmc
+        selected_var = st.selectbox('Select Token', ["percent_change_24h","percent_change_7d","percent_change_90d"], index=0)
+        # Sort the DataFrame by the 'percent_change_24h' column in ascending order
+        df_sorted = df_cmc.sort_values(by=selected_var, ascending=False)
+        # Select the top 10 and worst 10 rows
+        top_10 = df_sorted.head(10)
+        worst_10 = df_sorted.tail(10)
+        # Combine the top and worst dataframes for plotting
+        combined_df = pd.concat([top_10, worst_10], axis=0)
+        max_abs_val = max(abs(combined_df[selected_var].min()), abs(combined_df[selected_var].max()))
+
+        # Create a bar plot for the top 10 with a green color scale
+        fig = go.Figure(data=[
+            go.Bar(
+                x=top_10["symbol"],
+                y=top_10[selected_var],
+                marker_color='rgb(0,100,0)',  # Green color for top 10
+                hovertext= "Name : "+top_10["name"].astype(str)+ '<br>' +
+                        selected_var + " : " + top_10["percent_tokens_circulation"].astype(str) + '<br>' +
+                        'Market Cap: ' + top_10["market_cap"].astype(str) + '<br>' +
+                        'Fully Diluted Market Cap: ' + top_10["fully_diluted_market_cap"].astype(str) + '<br>' +
+                        'Last Updated: ' + top_10["last_updated"].astype(str),
+                name="top_10"
+            )
+        ])
+
+        # Add the worst 10 to the same plot with a red color scale
+        fig.add_traces(go.Bar(
+                x=worst_10["symbol"],
+                y=worst_10[selected_var],
+                marker_color='rgb(255,0,0)',  # Red color for worst 10
+                hovertext="Name:"+worst_10["name"].astype(str)+ '<br>' +
+                        selected_var + " : " + worst_10["percent_tokens_circulation"].astype(str) + '<br>' +
+                        'Market Cap: ' + worst_10["market_cap"].astype(str) + '<br>' +
+                        'Fully Diluted Market Cap: ' + worst_10["fully_diluted_market_cap"].astype(str) + '<br>' +
+                        'Last Updated: ' + worst_10["last_updated"].astype(str),
+                name="worst_10"
+            )
+        )
+
+        # Customize aspect
+        fig.update_traces(marker_line_color='rgb(8,48,107)', marker_line_width=1.5, opacity=0.8)
+        fig.update_layout(title_text=f'Top 10 and Worst 10 by {selected_var.split("_")[-1]} Percentage Change')
+        fig.update_xaxes(categoryorder='total ascending')
+        fig.update_layout(
+            autosize=False,
+            width=500,
+            height=500,
+            margin=dict(
+                l=50,
+                r=50,
+                b=100,
+                t=100,
+                pad=4
+            ),
+            #paper_bgcolor="LightSteelBlue",
+        )
+        st.plotly_chart(fig)
+
+
+
+
+#-------------------------------------end ----------------------------------

app_dash.py

@@ -0,0 +1,73 @@
+import os
+import pandas as pd
+import dash
+from dash import dcc,html
+import dash_bootstrap_components as dbc
+from dash.dependencies import Input, Output
+import plotly.graph_objs as go
+
+# Load the data from the CSV files
+dataframes = []
+for filename in os.listdir('output'):
+    if filename.endswith('.csv'):
+        df = pd.read_csv(os.path.join('output', filename), sep=';')
+        dataframes.append(df)
+df = pd.concat(dataframes)
+
+
+# Create the Dash app
+app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+
+# Define the app layout
+app.layout = dbc.Container([
+    dbc.Row([
+        dbc.Col([
+            html.H1('Token Analysis'),
+            dcc.Dropdown(
+                id='token-dropdown',
+                options=[{'label': i, 'value': i} for i in df['tokenSymbol'].unique()],
+                value='MANA'
+            ),
+            # Add more filters here
+        ], width=5),
+        dbc.Col([
+            dcc.Graph(id='token-graph')
+        ], width=7)
+    ])
+])
+
+# Define the callback to update the graph
+@app.callback(
+    Output('token-graph', 'figure'),
+    [Input('token-dropdown', 'value')]
+)
+def update_graph(selected_token):
+    filtered_df = df[df['tokenSymbol'] == selected_token]
+    # filtered_df['timeStamp'] = pd.to_datetime(filtered_df['timeStamp'], unit='s')
+    # filtered_df['value'] = filtered_df['value'].astype(float) / 1e18
+    figure = go.Figure(
+        data=[
+            go.Scatter(
+                x=filtered_df['timeStamp'],
+                y=filtered_df['value'],
+                mode='lines',
+                name='Value over time'
+            )
+        ],
+        layout=go.Layout(
+            title='Token Value Over Time',
+            yaxis=dict(
+                title='Value ('+selected_token+')',  # Change this to 'Value (USD)' if the values are in USD
+            ),
+            showlegend=True,
+            legend=go.layout.Legend(
+                x=0,
+                y=1.0
+            ),
+            margin=go.layout.Margin(l=40, r=0, t=40, b=30)
+        )
+    )
+    return figure
+
+if __name__ == '__main__':
+    app.run_server(debug=True)

output/top_100_update.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ff89b933c5ee4694a0ec72fe7660677ed15012d95a0d6184767d05eb33fd397
+size 16258

output/transactions_APE.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c6094c453a4ae217cd7e6334ad0b92880e042d698db8b19978af61f42ceda1f
+size 25981544

output/transactions_AXIE.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21eddd1decbb2f70f6fe9102cf81f0a4309c10d838bc9a172255ff70a2461cb8
+size 7599371

output/transactions_GALA.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd264d44fff732f21b170dcd839968ecb0408fba89cc6a993fa0da8f20fa8e05
+size 32066355

output/transactions_ILV.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ce31d9e8d9c7b39bf1f73c2e05ac655f652ebffca121a6a662a0a89eaa62c9
+size 5552703

output/transactions_MANA.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8162afa63a588a222422d06a1f93508f92d225d124915c6ea51d5d051d4db1e
+size 12039331

output/transactions_PET.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa42451d8ab6696d44d5b648754791e1462f630de79439c580b8e92be5f016df
+size 885

output/transactions_WEAOPON.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35e366dc8930a78bd4f37409447da3c6b7f53f3b6a699c89a4c9d9d5740622f5
+size 537

requirements.txt

@@ -0,0 +1,12 @@
+beautifulsoup4
+pandas
+numpy
+requests
+lxml
+dash_bootstrap_components
+dash
+python-dotenv
+streamlit
+requests
+plotly
+nbformat

scrap_data_CMC.py

@@ -0,0 +1,88 @@
+#-------------------------------------libraries ----------------------------------
+
+from requests import Request, Session
+from requests.exceptions import ConnectionError, Timeout, TooManyRedirects
+import json
+import os
+import pandas as pd
+import numpy as np
+import logging
+from dotenv import load_dotenv
+load_dotenv()
+
+#-------------------------------------env vars----------------------------------
+
+url = os.getenv("URL_CMC")
+endpoints = ["v1/cryptocurrency/listings/latest",
+             "/v1/cryptocurrency/trending/latest",
+             ]
+start = "1"
+stop = "100"
+parameters = {
+  'start':start,
+  'limit':stop,
+  'convert':'USD'
+}
+headers = {
+  'Accepts': 'application/json',
+  'X-CMC_PRO_API_KEY': os.getenv("API_KEY_CMC"),
+}
+
+# Configure the logging settings
+log_folder = "./logs/scrapping/"
+os.makedirs(log_folder, exist_ok=True)  # Ensure the log folder exists
+log_file = os.path.join(log_folder, "scrapping.log")
+log_format = "%(asctime)s [%(levelname)s] - %(message)s"
+logging.basicConfig(filename=log_file, level=logging.INFO, format=log_format)
+
+#-------------------------------------api call----------------------------------
+
+session = Session()
+session.headers.update(headers)
+
+for endpoint in endpoints:
+  target = f"{url}/{endpoint}"
+  try:
+      response = session.get(target, params=parameters)
+      data = json.loads(response.text)
+      with open(f'output/cmc_data_{endpoint.replace("/", "_")}_{stop}.json', 'w') as f:
+          json.dump(data, f)
+      logging.info(f"Successfully fetched data from {target}")
+  except (ConnectionError, Timeout, TooManyRedirects) as e:
+      logging.error(f"Error while fetching data from {target}: {e}")
+
+#-------------------------------------process data----------------------------------
+
+# create data frame with chosen columns  
+df = pd.DataFrame(data["data"])[["name","symbol","circulating_supply","total_supply","quote"]]
+# explode column quote then chose columns 
+quote_df = pd.json_normalize(df['quote'].apply(lambda x: x['USD']))[["price","percent_change_24h","percent_change_7d","percent_change_90d","market_cap","fully_diluted_market_cap","last_updated"]]
+# drop quote
+df = df.drop("quote",axis=1)
+# create features
+df["percent_tokens_circulation"] = np.round((df["circulating_supply"]/df["total_supply"])*100,1)
+# merge dataframe
+df = df.join(quote_df)
+df["last_updated"] = pd.to_datetime(df["last_updated"])
+#df.to_csv(f"output/top_{stop}_update.csv")
+
+#-------------------------------------save data----------------------------------
+
+# Check if the file exists
+output_file = f"output/top_{stop}_update.csv"
+if os.path.isfile(output_file):
+    logging.info("Updating dataset"+f"top_{stop}_update"+". ")
+    # Read the existing data
+    existing_data = pd.read_csv(output_file)
+    # Concatenate the existing data with the new data vertically
+    updated_data = pd.concat([existing_data, df], axis=0, ignore_index=True)
+    # Remove duplicates (if any) based on a unique identifier column
+    updated_data.drop_duplicates(subset=["symbol", "last_updated"], inplace=True)
+    # Save the updated data back to the same file
+    updated_data.to_csv(output_file, index=False)
+else:
+    # If the file doesn't exist, save the current data to it
+    df.to_csv(output_file, index=False)
+logging.info("Script execution completed.")
+
+#-------------------------------------end----------------------------------

scrap_data_etherscan.py

@@ -0,0 +1,17 @@
+import requests
+import time
+import pandas as pd
+import json
+import os
+from utils.functions import update_and_save_csv
+
+# Create output folder
+if not os.path.exists("output"):
+    os.makedirs("output")
+
+# Load the JSON file into a dictionary
+print(os.getcwd())
+with open("ressources/dict_tokens_addr.json", "r") as file:
+    dict_addresses = json.load(file)
+
+update_and_save_csv(dict_addresses)