drafts

.gitignore

@@ -0,0 +1,10 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+.ipynb_checkpoints
+*.Rproj
+*.vrt
+*.tif
+*.sql
+__pycache__

gbif-app.py

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An example of showing geographic data."""
+
+import os
+
+# +
+import altair as alt
+import numpy as np
+import pandas as pd
+import pydeck as pdk
+import streamlit as st
+
+import ibis
+from ibis import _
+# -
+
+# SETTING PAGE CONFIG TO WIDE MODE AND ADDING A TITLE AND FAVICON
+st.set_page_config(layout="wide", page_title="GBIF Biodiversity Demo", page_icon=":butterfly:")
+
+
+
+# LOAD DATA ONCE
+@st.cache_resource
+def load_data():
+    con = ibis.duckdb.connect()
+    path = "butterflies_gb.parquet"
+    if os.path.isfile(path):
+        return con.read_parquet(path)
+    df = con.read_parquet("s3://anonymous@gbif-open-data-us-east-1/occurrence/2023-06-01/occurrence.parquet")
+    data = (df
+            .dropna(["decimallongitude", "decimallatitude", "year"], how="any")
+            .filter([_.order == "Lepidoptera", _.countrycode=="GB"])
+            .filter(_.year >= 2010)
+            .select(['year', 'decimallongitude', 'decimallatitude'])
+           )
+    return data
+
+
+def map(data, lat, lon, zoom):
+    st.write(
+        pdk.Deck(
+            map_style="mapbox://styles/mapbox/light-v9",
+            initial_view_state={
+                "latitude": lat,
+                "longitude": lon,
+                "zoom": zoom,
+                "pitch": 50,
+            },
+            layers=[
+                pdk.Layer(
+                    "HexagonLayer",
+                    data=data,
+                    get_position=["decimallongitude", "decimallatitude"],
+                    radius=1000,
+                    elevation_scale=100,
+                    pickable=True,
+                    extruded=True,
+                ),
+            ],
+        )
+    )
+
+
+# +
+# FILTER DATA FOR A SPECIFIC YEAR. ibis lazytable not cache-able..
+
+# @st.cache_data
+def filterdata(df, year):
+    return df.filter(_.year == year).to_pandas()
+
+
+# -
+
+# CALCULATE MIDPOINT FOR GIVEN SET OF DATA
+@st.cache_data
+def mpoint(lat, lon):
+    return (np.average(lat), np.average(lon))
+
+
+
+# STREAMLIT APP LAYOUT
+data = load_data()
+
+# LAYING OUT THE TOP SECTION OF THE APP
+row1_1, row1_2 = st.columns((2, 3))
+
+# SEE IF THERE'S A QUERY PARAM IN THE URL (e.g. ?pickup_hour=2)
+# THIS ALLOWS YOU TO PASS A STATEFUL URL TO SOMEONE WITH A SPECIFIC HOUR SELECTED,
+# E.G. https://share.streamlit.io/streamlit/demo-uber-nyc-pickups/main?pickup_hour=2
+if not st.session_state.get("url_synced", False):
+    try:
+        year = int(st.query_params["year"][0])
+        st.session_state["year"] = year
+        st.session_state["url_synced"] = True
+    except KeyError:
+        pass
+
+
+# IF THE SLIDER CHANGES, UPDATE THE QUERY PARAM
+def update_query_params():
+    year_selected = st.session_state["year"]
+    st.query_params["year"]=year_selected
+
+
+with row1_1:
+    st.title("GBIF Butterfly Occurances")
+    year_selected = st.slider(
+        "Select year", 2010, 2023, key="year", on_change=update_query_params
+    )
+
+
+with row1_2:
+    st.write(
+        """
+    ##
+    By sliding the slider on the left you can view different slices of time and explore different trends.
+    """
+    )
+
+# LAYING OUT THE MIDDLE SECTION OF THE APP WITH THE MAPS
+row2_1, row2_2, row2_3, row2_4 = st.columns((2, 1, 1, 1))
+
+# +
+# SETTING THE ZOOM LOCATIONS FOR THE AIRPORTS
+
+midpoint =  (52.0, -1.0)  #mpoint(data["lat"], data["lon"])
+# -
+
+
+
+with row2_1:
+    st.write(
+        f"""**Large Map**"""
+    )
+    map(filterdata(data, year_selected), midpoint[0], midpoint[1], 4)
+
+with row2_2:
+    st.write("**Panel 1**")
+    map(filterdata(data, year_selected), midpoint[0], midpoint[1], 3)
+
+with row2_3:
+    st.write("**Panel 2**")
+    map(filterdata(data, year_selected), midpoint[0], midpoint[1], 2)
+
+with row2_4:
+    st.write("**Panel 3**")
+    map(filterdata(data, year_selected), midpoint[0], midpoint[1], 1)
+
+
+
+
+
+
+
+

gbif.py

@@ -0,0 +1,162 @@
+import ibis
+from ibis import _
+import pydeck
+
+
+# +
+def connect_data():
+    con = ibis.duckdb.connect()
+    con.raw_sql('''
+    INSTALL httpfs;
+    LOAD httpfs;
+    LOAD 'build/release/extension/h3ext/h3ext.duckdb_extension';
+    SET s3_url_style='path';
+    SET s3_endpoint='minio.carlboettiger.info';
+    CREATE VIEW gb AS SELECT * FROM read_parquet('s3://shared-data/gbif_gb/**');
+    ''')
+    return con
+
+
+'''
+CREATE VIEW gbif AS SELECT * FROM read_parquet('s3://gbif/*');
+
+
+INSTALL httpfs;
+LOAD httpfs;
+SET s3_url_style='path';
+SET s3_endpoint='minio.carlboettiger.info';
+
+
+
+SET temp_directory='/tmp/duckdb';
+
+
+SET memory_limit = '150GB';
+SET max_memory = '150GB';
+COPY
+  (
+   SELECT *, 
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 1)) as h3z1,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 2)) as h3z2,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 3)) as h3z3,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 4)) as h3z4,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 5)) as h3z5,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 6)) as h3z6,
+   hex(h3_latlng_to_cell(gbif.decimallatitude, gbif.decimallongitude, 7)) as h3z7
+   FROM gbif
+   WHERE (NOT((decimallatitude IS NULL))) AND (NOT((decimallongitude IS NULL))) AND (countrycode = 'US')
+   ) TO 's3://shared-data/gbif/US' (FORMAT 'parquet', PARTITION_BY h3z1);
+
+'''
+
+
+
+# distinct species observations at h7 resolution
+def richness_data(con):
+    data =  (
+        con.table("gb").
+        filter(_.phylum == "Chordata").
+        select(_.genus, _.species, _["class"], _.h3z2, _.h3z3, _.h3z4, _.h3z5, _.h3z6, _.h3z7).
+        distinct().
+        to_parquet("gb-cache.parquet")
+    )
+    return data
+
+con = connect_data()
+richness_data(con)
+# -
+
+
+
+# +
+
+
+def zoom_data(zoom=6):
+    hzoom = "h3z" + str(zoom)
+    data =  (
+        con.read_parquet("gb-cache.parquet").
+        rename(h3 = hzoom).
+        group_by([_.h3, _["class"]]).
+        aggregate(n = _.count()).
+        to_csv("gbif-vert-gb-" + hzoom + ".csv")
+    )
+    return data
+
+
+def filterdata(df, year):
+    return df[df.year == year]
+
+zoom_data(4)
+zoom_data(5)
+zoom_data(6)
+zoom_data(7)
+
+
+# +
+def load_data(zoom=7):
+    con = ibis.duckdb.connect()
+    path = "gbif-vert-gb-h3z" + str(zoom) + ".csv"
+    df_all = (
+        con.
+        read_csv(path).
+        group_by(_.h3).
+        aggregate(n = _.n.sum()).
+        mutate(color = 255 * _.n / _.n.max()).
+        to_pandas()
+        )
+    return df_all
+
+def load_class(taxa="Amphibia", zoom=7):
+    con = ibis.duckdb.connect()
+    path = "gbif-vert-gb-h3z" + str(zoom) + ".csv"
+    df = (con.
+      read_csv(path).
+      filter(_['class']==taxa).
+      mutate(color = 255 * _.n / _.n.max()).
+      to_pandas()
+     )
+    return df
+
+df = load_data()
+df
+
+# +
+# Define a layer to display on a map
+
+import pydeck as pdk
+
+# Set the viewport location
+view_state = pdk.ViewState(
+    longitude=-1.415,
+    latitude=52.2323,
+    zoom=4,
+    min_zoom=1,
+    max_zoom=12,
+    pitch=40.5,
+    bearing=-27.36)
+
+def map(data):
+    layer = pdk.Layer(
+        "H3HexagonLayer",
+        data,
+        pickable=True,
+        stroked=True,
+        filled=True,
+        extruded=True,
+        elevation_scale=100,
+        get_elevation='color',
+        get_hexagon="h3",
+        get_fill_color="[color, 30, 255 - color, 160]",
+        get_line_color=[255, 255, 255],
+        line_width_min_pixels=2,
+    )
+    
+    # Render
+    r = pdk.Deck(layers=[layer], initial_view_state=view_state)
+    return r.to_html("hex_layer.html")
+
+
+
+
+
+map(df)

get_gbif.py

@@ -0,0 +1,42 @@
+import ibis
+from ibis import _
+import os
+
+
+
+# +
+
+import pydeck as pdk
+
+# Set the viewport location
+view_state = pdk.ViewState(
+    longitude=-1.415,
+    latitude=52.2323,
+    zoom=5,
+    min_zoom=1,
+    max_zoom=12,
+    pitch=40.5,
+    bearing=-27.36)
+
+def map(data):
+    layer = pdk.Layer(
+        "H3HexagonLayer",
+        data,
+        pickable=True,
+        stroked=True,
+        filled=True,
+        extruded=True,
+        elevation_range=[0,200],
+        elevation_scale=2000,
+        get_elevation='color',
+        get_hexagon="h3z7",
+        get_fill_color="[color, 30, color]",
+        get_line_color=[255, 255, 255],
+        line_width_min_pixels=2,
+    )
+    
+    # Render
+    r = pdk.Deck(layers=[layer], initial_view_state=view_state)
+    return r.to_html("hex_layer.html")
+
+map("https://huggingface.co/spaces/cboettig/hf-streamlit-demo/resolve/main/test.csv")