grouping

preprocess.py

@@ -5,21 +5,28 @@ import fiona
 import geopandas as gpd
 import rioxarray
 from shapely.geometry import box
+con = ibis.duckdb.connect()
+con.load_extension("spatial")
+threads = -1
+
+agency_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-name.parquet").select(manager_name_id = "Code", manager_name = "Dom")
+agency_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
+desig_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
+public_access = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
+state_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-state-name.parquet").select(state = "Code", state_name = "Dom")
+iucn = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-iucn.parquet").select(iucn_code = "CODE", iucn_category = "DOM")
 
 # +
 
 fgb = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.fgb"
 parquet = "https://data.source.coop/cboettig/pad-us-3/pad-us3-combined.parquet"
 # gdb = "https://data.source.coop/cboettig/pad-us-3/PADUS3/PAD_US3_0.gdb" # original, all tables
-
-con = ibis.duckdb.connect()
-con.load_extension("spatial")
-threads = -1
-
 # or read the fgb version, much slower
 # pad = con.read_geo(fgb)
 # pad = con.read_parquet(parquet)
 # Currently ibis doesn't detect that this is GeoParquet.  We need a SQL escape-hatch to cast the geometry
+
+
 con.raw_sql(f"CREATE OR REPLACE VIEW pad AS SELECT *, st_geomfromwkb(geometry) as geom from read_parquet('{parquet}')")
 pad = con.table("pad")
 # -
@@ -30,12 +37,9 @@ pad = con.table("pad")
 meta = fiona.open(fgb)
 crs = meta.crs
 
-# +
 ## optional getting bounds
-cog = "https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif"
-
 # extract bounds. (in this case these are already in the same projection actually so r.rio.bounds() would work)
-r = rioxarray.open_rasterio(cog)
+r = rioxarray.open_rasterio("https://data.source.coop/cboettig/mobi/species-richness-all/SpeciesRichness_All.tif")
 bounds = box(*r.rio.transform_bounds(crs))
 
 # +
@@ -89,18 +93,29 @@ pad_grouping = (
            )
     .mutate(bucket = case)
     .select(categorical_columns)
+    .rename(manager_name_id = "Mang_Name", 
+            manager_type_id = "Mang_Type", 
+            designation_type_id = "Des_Tp",
+            public_access_id = "Pub_Access",
+            category = "FeatClass",
+            iucn_code = "IUCN_Cat",
+            gap_code = "GAP_Sts",
+            state = "State_Nm",
+            easement_holder = "EsmtHldr",
+            date_established = "Date_Est",
+            area_name = "Unit_Nm")
+    .left_join(agency_name, "manager_name_id")
+    .left_join(agency_type, "manager_type_id")
+    .left_join(desig_type, "designation_type_id")
+    .left_join(public_access, "public_access_id")
+    .left_join(state_name, "state")
+    .left_join(iucn, "iucn_code")
+    .select(~s.contains("_right"))
 )
 
 pad_grouping.to_parquet("pad-groupings.parquet")
 # -
 
-agency_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-name.parquet").select(manager_name_id = "Code", manager_name = "Dom")
-agency_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-agency-type.parquet").select(manager_type_id = "Code", manager_type = "Dom")
-desig_type = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-desgination-type.parquet").select(designation_type_id = "Code", designation_type = "Dom")
-public_access = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-public-access.parquet").select(public_access_id = "Code", public_access = "Dom")
-state_name = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-state-name.parquet").select(state = "Code", state_name = "Dom")
-iucn = con.read_parquet("https://huggingface.co/datasets/boettiger-lab/pad-us-3/resolve/main/parquet/pad-iucn.parquet").select(iucn_code = "CODE", iucn_category = "DOM")
-
 (pad_parquet
     .rename(manager_name_id = "Mang_Name", 
             manager_type_id = "Mang_Type",