script for rounding data

preprocess_part2.ipynb

@@ -0,0 +1,152 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "6e29e27b-6baa-40ac-bbf8-230da2f94d17",
+   "metadata": {},
+   "source": [
+    "# Rounding data\n",
+    "\n",
+    "This should've been in the original preprocessing script, but the decision to round the data came much later so I'm including this as an addition."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81f3a57e-fa63-43b4-90f9-407f03abae79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "import ibis.selectors as s\n",
+    "parquet = \"https://huggingface.co/spaces/boettiger-lab/pad-us/resolve/575a4505f3eb1703070977d9d26f6a770045309c/pad-stats.parquet\"\n",
+    "con = ibis.duckdb.connect(extensions=[\"spatial\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b4745567-45db-406c-9b08-d72a97908d04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#rounding data with ibis\n",
+    "us = (con\n",
+    "      .read_parquet(parquet)\n",
+    "      .cast({\"geometry\": \"geometry\"})\n",
+    "      .mutate(geometry=_.geometry.convert(\n",
+    "          \"+proj=moll +lon_0=0 +datum=WGS84 +units=m +no_defs\",\n",
+    "          \"epsg:4326\"\n",
+    "      ))\n",
+    "      .mutate(richness=_.richness.round(3),\n",
+    "                rsr=_.rsr.round(3),\n",
+    "                all_species_rwr=_.all_species_rwr.round(3),\n",
+    "                all_species_richness=_.all_species_richness.round(3),\n",
+    "                manageable_carbon=_.manageable_carbon.round(3),\n",
+    "                irrecoverable_carbon = _.irrecoverable_carbon.round(3),\n",
+    "                human_impact=_.human_impact.round(3),\n",
+    "                deforest_carbon=_.deforest_carbon.round(3),\n",
+    "                biodiversity_intactness_loss=_.biodiversity_intactness_loss.round(3),\n",
+    "                forest_integrity_loss=_.forest_integrity_loss.round(3),\n",
+    "                crop_reduction =_.crop_reduction.round(3), \n",
+    "                crop_expansion =_.crop_expansion.round(3)\n",
+    "                )\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5985e893-ed21-487c-a609-e449edae9012",
+   "metadata": {},
+   "source": [
+    "# Save as PMTiles + Upload data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "100db9ae-e167-45ed-8c44-6205e5630923",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "from huggingface_hub import HfApi, login\n",
+    "import streamlit as st\n",
+    "\n",
+    "login(st.secrets[\"HF_TOKEN\"])\n",
+    "api = HfApi()\n",
+    "\n",
+    "def hf_upload(file, repo_id,repo_type):\n",
+    "    info = api.upload_file(\n",
+    "            path_or_fileobj=file,\n",
+    "            path_in_repo=file,\n",
+    "            repo_id=repo_id,\n",
+    "            repo_type=repo_type,\n",
+    "        )\n",
+    "def generate_pmtiles(input_file, output_file, max_zoom=12):\n",
+    "    # Ensure Tippecanoe is installed\n",
+    "    if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
+    "        raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
+    "\n",
+    "    # Construct the Tippecanoe command\n",
+    "    command = [\n",
+    "        \"tippecanoe\",\n",
+    "        \"-o\", output_file,\n",
+    "        \"-zg\",\n",
+    "        \"--extend-zooms-if-still-dropping\",\n",
+    "        \"--force\",\n",
+    "        \"--projection\", \"EPSG:4326\",  \n",
+    "        \"-L\",\"pad-stats:\"+input_file,\n",
+    "    ]\n",
+    "    # Run Tippecanoe\n",
+    "    try:\n",
+    "        subprocess.run(command, check=True)\n",
+    "        print(f\"Successfully generated PMTiles file: {output_file}\")\n",
+    "    except subprocess.CalledProcessError as e:\n",
+    "        print(f\"Error running Tippecanoe: {e}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eaf1a1cf-a5db-462c-a257-c68547b35d4d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gdf = us.execute().set_crs(\"EPSG:4326\")\n",
+    "\n",
+    "gdf.to_file(\"pad-stats.geojson\")\n",
+    "generate_pmtiles(\"pad-stats.geojson\", \"pad-stats.pmtiles\")\n",
+    "hf_upload(\"pad-stats.pmtiles\", \"boettiger-lab/pad-us-3\", \"dataset\")\n",
+    "\n",
+    "gdf.to_parquet(\"pad-stats.parquet\")\n",
+    "hf_upload(\"pad-stats.parquet\", \"boettiger-lab/pad-us-3\", \"dataset\")\n",
+    "hf_upload(\"pad-stats.parquet\", \"boettiger-lab/pad-us\", \"space\") # redundant but I want a local copy for testing\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}