Update pages/1_🖼️_VertBox.py

pages/1_🖼️_VertBox.py

@@ -1,195 +1,422 @@
 import streamlit as st
-import geopandas as gpd
 import folium
-from streamlit_folium import folium_static, st_folium
+from streamlit_folium import st_folium
 from folium.plugins import Draw
-import requests
-from shapely.geometry import box
-import json
-from PIL import Image
-import io
-import numpy as np
+import geopandas as gpd
 import tempfile
 import os
 import urllib.request
+import json
+from pathlib import Path
+import datetime
+from osgeo import gdal
+import io
 import zipfile
-
-st.set_page_config(layout="wide", page_title="Extracteur de données géospatiales Swisstopo")
+import base64
+import concurrent.futures
+import requests
+from functools import partial
 
 # Constants
+CATEGORIES = {
+    'Gebueschwald': 'Forêt buissonnante',
+    'Wald': 'Forêt',
+    'Wald offen': 'Forêt claisemée',
+    'Gehoelzflaeche': 'Zone boisée',
+}
+MERGE_CATEGORIES = True
 URL_STAC_SWISSTOPO_BASE = 'https://data.geo.admin.ch/api/stac/v0.9/collections/'
 DIC_LAYERS = {
     'ortho': 'ch.swisstopo.swissimage-dop10',
     'mnt': 'ch.swisstopo.swissalti3d',
     'mns': 'ch.swisstopo.swisssurface3d-raster',
-    'bati3D': 'ch.swisstopo.swissbuildings3d_2',
+    'bati3D_v2': 'ch.swisstopo.swissbuildings3d_2',
     'bati3D_v3': 'ch.swisstopo.swissbuildings3d_3_0',
 }
 
-FOLDER_NAME_SWISSTOPO = "swisstopo"
-NB_POLYGONES_MAX = 2000000
-
 # Helper functions
-@st.cache_data
-def lv95towgs84(x, y):
+def wgs84_to_lv95(lat, lon):
+    url = f'http://geodesy.geo.admin.ch/reframe/wgs84tolv95?easting={lon}&northing={lat}&format=json'
+    with urllib.request.urlopen(url) as response:
+        data = json.load(response)
+    return data['easting'], data['northing']
+
+def lv95_to_wgs84(x, y):
     url = f'http://geodesy.geo.admin.ch/reframe/lv95towgs84?easting={x}&northing={y}&format=json'
-    with urllib.request.urlopen(url) as f:
-        json_res = json.loads(f.read().decode('utf-8'))
-    return float(json_res['easting']), float(json_res['northing'])
+    with urllib.request.urlopen(url) as response:
+        data = json.load(response)
+    return data['northing'], data['easting']
 
-@st.cache_data
-def get_list_from_STAC_swisstopo(url, xmin, ymin, xmax, ymax, gdb=False):
-    est, sud = lv95towgs84(xmin, ymin)
-    ouest, nord = lv95towgs84(xmax, ymax)
-    suffix_url = f"/items?bbox={est},{sud},{ouest},{nord}"
-    url += suffix_url
-    res = []
+def detect_and_convert_bbox(bbox):
+    xmin, ymin, xmax, ymax = bbox
+    wgs84_margin = 0.9
+    wgs84_bounds = {
+        'xmin': 5.96 - wgs84_margin,
+        'ymin': 45.82 - wgs84_margin,
+        'xmax': 10.49 + wgs84_margin,
+        'ymax': 47.81 + wgs84_margin
+    }
+    lv95_margin = 100000
+    lv95_bounds = {
+        'xmin': 2485000 - lv95_margin,
+        'ymin': 1075000 - lv95_margin,
+        'xmax': 2834000 + lv95_margin,
+        'ymax': 1296000 + lv95_margin
+    }
 
+    if (wgs84_bounds['xmin'] <= xmin <= wgs84_bounds['xmax'] and
+        wgs84_bounds['ymin'] <= ymin <= wgs84_bounds['ymax'] and
+        wgs84_bounds['xmin'] <= xmax <= wgs84_bounds['xmax'] and
+        wgs84_bounds['ymin'] <= ymax <= wgs84_bounds['ymax']):
+        lv95_min = wgs84_to_lv95(ymin, xmin)
+        lv95_max = wgs84_to_lv95(ymax, xmax)
+        bbox_lv95 = (lv95_min[0], lv95_min[1], lv95_max[0], lv95_max[1])
+        return (bbox, bbox_lv95)
+
+    if (lv95_bounds['xmin'] <= xmin <= lv95_bounds['xmax'] and
+        lv95_bounds['ymin'] <= ymin <= lv95_bounds['ymax'] and
+        lv95_bounds['xmin'] <= xmax <= lv95_bounds['xmax'] and
+        lv95_bounds['ymin'] <= ymax <= lv95_bounds['ymax']):
+        wgs84_min = lv95_to_wgs84(xmin, ymin)
+        wgs84_max = lv95_to_wgs84(xmax, ymax)
+        bbox_wgs84 = (wgs84_min[1], wgs84_min[0], wgs84_max[1], wgs84_max[0])
+        return (bbox_wgs84, bbox)
+
+    return None
+
+def get_list_from_STAC_swisstopo(url, est, sud, ouest, nord, gdb=False):
+    lst_indesirables = [] if gdb else ['.xyz.zip', '.gdb.zip']
+    sufixe_url = f"/items?bbox={est},{sud},{ouest},{nord}"
+    url += sufixe_url
+    res = []
+    
     while url:
-        with urllib.request.urlopen(url) as f:
-            json_res = json.loads(f.read().decode('utf-8'))
+        with urllib.request.urlopen(url) as response:
+            json_res = json.load(response)
         
         url = None
-        for link in json_res.get('links', []):
-            if link['rel'] == 'next':
-                url = link['href']
+        links = json_res.get('links', None)
+        if links:
+            for link in links:
+                if link['rel'] == 'next':
+                    url = link['href']
         
         for item in json_res['features']:
             for k, dic in item['assets'].items():
                 href = dic['href']
                 if gdb:
-                    if href.endswith('.gdb.zip') and len(href.split('/')[-1].split('_')) == 7:
-                        res.append(href)
-                elif not href.endswith(('.xyz.zip', '.gdb.zip')):
-                    res.append(href)
+                    if href[-8:] == '.gdb.zip' and len(dic['href'].split('/')[-1].split('_')) == 7:
+                        res.append(dic['href'])
+                else:
+                    if href[-8:] not in lst_indesirables:
+                        res.append(dic['href'])
     
     return res
 
+def suppr_doublons_bati3D_v2(lst_url):
+    dico = {}
+    dxf_files = [url for url in lst_url if url[-8:] == '.dxf.zip']
+    for dxf in dxf_files:
+        *a, date, feuille = dxf.split('/')[-2].split('_')
+        dico.setdefault(feuille, []).append((date, dxf))
+    res = []
+    for k, liste in dico.items():
+        res.append(sorted(liste, reverse=True)[0][1])
+    return res
+
+def suppr_doublons_bati3D_v3(lst_url):
+    dico = {}
+    gdb_files = [url for url in lst_url if url[-8:] == '.gdb.zip']
+    for gdb in gdb_files:
+        *a, date, feuille = gdb.split('/')[-2].split('_')
+        dico.setdefault(feuille, []).append((date, gdb))
+    res = []
+    for k, liste in dico.items():
+        res.append(sorted(liste, reverse=True)[0][1])
+    return res
+
 def suppr_doublons_list_ortho(lst):
     dic = {}
     for url in lst:
         nom, an, noflle, taille_px, epsg = url.split('/')[-1][:-4].split('_')
         dic.setdefault((noflle, float(taille_px)), []).append((an, url))
-    return [sorted(lst, reverse=True)[0][1] for noflle, lst in dic.items()]
+    res = []
+    for noflle, lst in dic.items():
+        an, url = sorted(lst, reverse=True)[0]
+        res.append(url)
+    return res
 
-def suppr_doublons_bati3D(lst_url):
-    dico = {}
-    for dxf in [url for url in lst_url if url.endswith('.dxf.zip')]:
-        *a, date, feuille = dxf.split('/')[-2].split('_')
-        dico.setdefault(feuille, []).append((date, dxf))
-    return [sorted(liste, reverse=True)[0][1] for k, liste in dico.items()]
-
-def download_file(url, filename):
-    with urllib.request.urlopen(url) as response, open(filename, 'wb') as out_file:
-        out_file.write(response.read())
-
-def unzip_file(zip_file, extract_to):
-    with zipfile.ZipFile(zip_file, 'r') as zip_ref:
-        zip_ref.extractall(extract_to)
-
-# Main Streamlit App
-def main():
-    st.title("Extracteur de données Swisstopo")
-
-    # Sidebar for layer selection
-    st.sidebar.header("Sélection des couches")
-    mnt2m = st.sidebar.checkbox("MNT 2m")
-    mnt50cm = st.sidebar.checkbox("MNT 50cm")
-    mns = st.sidebar.checkbox("MNS (modèle numérique de surface)")
-    bati3D = st.sidebar.checkbox("Bâtiments 3D")
-    bati3D_v3 = st.sidebar.checkbox("Bâtiments 3D v3 (print 3D)")
-    ortho2m = st.sidebar.checkbox("Orthophoto 2m")
-    ortho10cm = st.sidebar.checkbox("Orthophoto 10cm")
-
-    # Main content
-    col1, col2 = st.columns([1, 2])
-
-    with col1:
-        st.header("Définir l'emprise")
-        xmin = st.number_input("X min", value=2600000.0)
-        ymin = st.number_input("Y min", value=1200000.0)
-        xmax = st.number_input("X max", value=2600100.0)
-        ymax = st.number_input("Y max", value=1200100.0)
-
-    with col2:
-        st.header("Carte")
-        m = folium.Map(location=[46.8, 8.2], zoom_start=8)
-        Draw(draw_options={'polyline': False, 'polygon': False, 'circle': False, 'marker': False, 'circlemarker': False},
-             edit_options={'edit': False}).add_to(m)
-        output = st_folium(m, width=700, height=500)
-
-        if output['last_active_drawing']:
-            coords = output['last_active_drawing']['geometry']['coordinates'][0]
-            xmin, ymin = min(c[0] for c in coords), min(c[1] for c in coords)
-            xmax, ymax = max(c[0] for c in coords), max(c[1] for c in coords)
-            st.success("Emprise sélectionnée!")
-
-    if st.button("Obtenir les données"):
-        bbox = (xmin, ymin, xmax, ymax)
-        urls = []
-
-        # MNT
-        if mnt2m or mnt50cm:
-            tri = '_2_' if mnt2m else '_0.5_'
-            url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS['mnt']
-            lst = [v for v in get_list_from_STAC_swisstopo(url, *bbox) if tri in v]
-            urls += lst
-
-        # MNS
-        if mns:
-            url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS['mns']
-            lst = get_list_from_STAC_swisstopo(url, *bbox)
-            urls += lst
-
-        # Bâtiments 3D
-        if bati3D:
-            url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS['bati3D']
-            lst = get_list_from_STAC_swisstopo(url, *bbox)
-            lst = suppr_doublons_bati3D(lst)
-            urls += lst
-
-        # Bâtiments 3D v3
-        if bati3D_v3:
-            url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS['bati3D_v3']
-            lst = get_list_from_STAC_swisstopo(url, *bbox, gdb=True)
-            urls += lst
-
-        # Orthophoto
-        if ortho2m or ortho10cm:
-            tri = '_2_' if ortho2m else '_0.1_'
-            url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS['ortho']
-            lst = [v for v in get_list_from_STAC_swisstopo(url, *bbox) if tri in v]
-            lst = suppr_doublons_list_ortho(lst)
-            urls += lst
-
-        if urls:
-            st.write(f"Nombre de fichiers à télécharger : {len(urls)}")
-            
-            # Create a temporary directory for downloads
-            with tempfile.TemporaryDirectory() as tmpdirname:
-                progress_bar = st.progress(0)
-                for i, url in enumerate(urls):
-                    filename = os.path.join(tmpdirname, url.split('/')[-1])
-                    download_file(url, filename)
-                    if filename.endswith('.zip'):
-                        unzip_file(filename, tmpdirname)
-                    progress_bar.progress((i + 1) / len(urls))
-                
-                st.success("Téléchargement terminé!")
-                
-                # Create a zip file of all downloaded content
-                zip_filename = "swisstopo_data.zip"
-                shutil.make_archive(zip_filename[:-4], 'zip', tmpdirname)
-                
-                with open(zip_filename, "rb") as fp:
-                    btn = st.download_button(
-                        label="Télécharger les données",
-                        data=fp,
-                        file_name=zip_filename,
-                        mime="application/zip"
-                    )
+def suppr_doublons_list_mnt(lst):
+    dic = {}
+    for url in lst:
+        nom, an, noflle, taille_px, epsg, inconnu = url.split('/')[-1][:-4].split('_')
+        dic.setdefault((noflle, float(taille_px)), []).append((an, url))
+    res = []
+    for noflle, lst in dic.items():
+        an, url = sorted(lst, reverse=True)[0]
+        res.append(url)
+    return res
+
+@st.cache_data
+def get_urls(bbox_wgs84, data_types, resolutions):
+    urls = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_data_type = {
+            executor.submit(
+                get_urls_for_data_type,
+                data_type,
+                bbox_wgs84,
+                resolutions.get(data_type)
+            ): data_type
+            for data_type, enabled in data_types.items() if enabled
+        }
+        for future in concurrent.futures.as_completed(future_to_data_type):
+            data_type = future_to_data_type[future]
+            try:
+                urls.extend(future.result())
+            except Exception as exc:
+                st.error(f"Error fetching URLs for {data_type}: {exc}")
+    return urls
+
+def get_urls_for_data_type(data_type, bbox_wgs84, resolution=None):
+    url = URL_STAC_SWISSTOPO_BASE + DIC_LAYERS[data_type]
+    if data_type in ['mnt', 'ortho']:
+        tri = f'_{resolution}_'
+        lst = [v for v in get_list_from_STAC_swisstopo(url, *bbox_wgs84) if tri in v]
+        if data_type == 'mnt':
+            return suppr_doublons_list_mnt(lst)
         else:
-            st.warning("Aucun fichier à télécharger pour les critères sélectionnés.")
+            return suppr_doublons_list_ortho(lst)
+    elif data_type == 'mns':
+        lst = [v for v in get_list_from_STAC_swisstopo(url, *bbox_wgs84) if 'raster' in v]
+        return suppr_doublons_list_mnt(lst)
+    elif data_type == 'bati3D_v2':
+        lst = get_list_from_STAC_swisstopo(url, *bbox_wgs84)
+        return suppr_doublons_bati3D_v2(lst)
+    elif data_type == 'bati3D_v3':
+        lst = get_list_from_STAC_swisstopo(url, *bbox_wgs84, gdb=True)
+        return suppr_doublons_bati3D_v3(lst)
+    return []
+
+def fetch_url(url):
+    response = requests.get(url)
+    return response.content
+
+def merge_ortho_images(urls, output_format='GTiff'):
+    try:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            local_files = []
+            with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+                future_to_url = {executor.submit(fetch_url, url): url for url in urls}
+                for i, future in enumerate(concurrent.futures.as_completed(future_to_url)):
+                    url = future_to_url[future]
+                    try:
+                        data = future.result()
+                        local_filename = os.path.join(temp_dir, f"ortho_{i}.tif")
+                        with open(local_filename, 'wb') as f:
+                            f.write(data)
+                        local_files.append(local_filename)
+                    except Exception as exc:
+                        st.error(f"Error downloading {url}: {exc}")
+
+            if not local_files:
+                st.error("No ortho images were successfully downloaded.")
+                return None
+
+            vrt_options = gdal.BuildVRTOptions(resampleAlg='nearest', addAlpha=False)
+            vrt_path = os.path.join(temp_dir, "merged.vrt")
+            vrt = gdal.BuildVRT(vrt_path, local_files, options=vrt_options)
+            vrt = None  # Close the dataset
+
+            output_path = os.path.join(temp_dir, f"merged.{output_format.lower()}")
+            if output_format == 'GTiff':
+                translate_options = gdal.TranslateOptions(format="GTiff", creationOptions=["COMPRESS=LZW", "TILED=YES"])
+            elif output_format == 'JPEG':
+                translate_options = gdal.TranslateOptions(format="JPEG", creationOptions=["QUALITY=85"])
+            elif output_format == 'PNG':
+                translate_options = gdal.TranslateOptions(format="PNG", creationOptions=["COMPRESS=DEFLATE"])
+            else:
+                st.error(f"Unsupported output format: {output_format}")
+                return None
+
+            gdal.Translate(output_path, vrt_path, options=translate_options)
+
+            if not os.path.exists(output_path):
+                st.error(f"Failed to create merged image: {output_path}")
+                return None
+
+            with open(output_path, 'rb') as f:
+                return f.read()
+    except Exception as e:
+        st.error(f"Error in merge_ortho_images: {e}")
+        return None
+
+def create_geojson_with_links(urls, bbox):
+    features = []
+    for url in urls:
+        feature = {
+            "type": "Feature",
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [bbox]
+            },
+            "properties": {
+                "url": url,
+                "type": url.split('/')[-2].split('_')[0]
+            }
+        }
+        features.append(feature)
+    
+    geojson = {
+        "type": "FeatureCollection",
+        "features": features
+    }
+    return json.dumps(geojson)
+
+@st.cache_data
+def prepare_download_package(urls, bbox, ortho_format):
+    geojson_data = create_geojson_with_links(urls, bbox)
+    ortho_urls = [url for url in urls if 'swissimage-dop10' in url]
+    ortho_data = merge_ortho_images(ortho_urls, ortho_format) if ortho_urls else None
+
+    zip_buffer = io.BytesIO()
+    with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+        zip_file.writestr('download_links.geojson', geojson_data)
+        if ortho_data:
+            zip_file.writestr(f'merged_ortho.{ortho_format.lower()}', ortho_data)
+        else:
+            st.warning("Failed to merge ortho images. Only download links will be included in the package.")
+
+    return zip_buffer.getvalue()
+
+def geojson_forest(bbox, fn_geojson):
+    xmin, ymin, xmax, ymax = bbox
+    url_base = 'https://hepiadata.hesge.ch/arcgis/rest/services/suisse/TLM_C4D_couverture_sol/FeatureServer/1/query?'
+    sql = ' OR '.join([f"OBJEKTART='{cat}'" for cat in CATEGORIES.keys()])
+    params = {
+        "geometry": f"{xmin},{ymin},{xmax},{ymax}",
+        "geometryType": "esriGeometryEnvelope",
+        "returnGeometry": "true",
+        "outFields": "OBJEKTART",
+        "orderByFields": "OBJEKTART",
+        "where": sql,
+        "returnZ": "true",
+        "outSR": '2056',
+        "spatialRel": "esriSpatialRelIntersects",
+        "f": "geojson"
+    }
+    query_string = urllib.parse.urlencode(params)
+    url = url_base + query_string
+    with urllib.request.urlopen(url) as response:
+        data = json.load(response)
+    with open(fn_geojson, 'w') as f:
+        json.dump(data, f)
+
+# Streamlit app
+st.set_page_config(page_title="Swiss Geospatial Data Downloader", layout="wide")
+st.title("Swiss Geospatial Data Downloader")
+
+# Sidebar for data selection
+st.sidebar.header("Data Selection")
+data_types = {
+    'mnt': st.sidebar.checkbox("Digital Terrain Model (MNT)", value=True),
+    'mns': st.sidebar.checkbox("Digital Surface Model (MNS)", value=True),
+    'bati3D_v2': st.sidebar.checkbox("3D Buildings v2", value=True),
+    'bati3D_v3': st.sidebar.checkbox("3D Buildings v3", value=True),
+    'ortho': st.sidebar.checkbox("Orthophotos", value=True),
+}
+resolutions = {
+    'mnt': st.sidebar.selectbox("MNT Resolution", [0.5, 2.0], index=0),
+    'ortho': st.sidebar.selectbox("Orthophoto Resolution", [0.1, 2.0], index=0),
+}
+ortho_format = st.sidebar.selectbox("Ortho Output Format", ['GTiff', 'JPEG', 'PNG'], index=0)
+
+# Main content area
+st.subheader("Select Bounding Box")
+
+# Create a map centered on Switzerland
+m = folium.Map(location=[46.8182, 8.2275], zoom_start=8)
+
+# Add rectangle draw control
+draw = Draw(
+    draw_options={
+        'rectangle': True,
+        'polygon': False,
+        'polyline': False,
+        'circle': False,
+        'marker': False,
+        'circlemarker': False
+    },
+    edit_options={'edit': False}
+)
+draw.add_to(m)
+
+# Use st_folium to render the map and get the drawn bbox
+output = st_folium(m, width=700, height=500)
+
+# Initialize session state for bbox
+if 'bbox' not in st.session_state:
+    st.session_state.bbox = [6.0, 46.0, 10.0, 47.0]  # Default values for Switzerland
+
+# Update bbox if a new one is drawn
+if output['last_active_drawing']:
+    coordinates = output['last_active_drawing']['geometry']['coordinates'][0]
+    st.session_state.bbox = [
+        min(coord[0] for coord in coordinates),
+        min(coord[1] for coord in coordinates),
+        max(coord[0] for coord in coordinates),
+        max(coord[1] for coord in coordinates)
+    ]
+
+# Display and allow editing of bounding box coordinates
+st.subheader("Enter Bounding Box Coordinates")
+col1, col2, col3, col4 = st.columns(4)
+with col1:
+    xmin = st.number_input("Min Longitude", value=st.session_state.bbox[0], format="%.4f", key="xmin")
+with col2:
+    ymin = st.number_input("Min Latitude", value=st.session_state.bbox[1], format="%.4f", key="ymin")
+with col3:
+    xmax = st.number_input("Max Longitude", value=st.session_state.bbox[2], format="%.4f", key="xmax")
+with col4:
+    ymax = st.number_input("Max Latitude", value=st.session_state.bbox[3], format="%.4f", key="ymax")
+
+# Update session state if coordinates are manually changed
+st.session_state.bbox = [xmin, ymin, xmax, ymax]
+
+if st.session_state.bbox:
+    st.write(f"Selected bounding box (WGS84): {st.session_state.bbox}")
+    bbox_results = detect_and_convert_bbox(st.session_state.bbox)
+    if bbox_results:
+        bbox_wgs84, bbox_lv95 = bbox_results
+        st.write(f"Converted bounding box (LV95): {bbox_lv95}")
+
+        if st.button("Get Download Package"):
+            with st.spinner("Preparing download package..."):
+                urls = get_urls(bbox_wgs84, data_types, resolutions)
+                if urls:
+                    zip_data = prepare_download_package(urls, bbox_wgs84, ortho_format)
+                    b64 = base64.b64encode(zip_data).decode()
+                    href = f'<a href="data:application/zip;base64,{b64}" download="swiss_geospatial_data.zip">Download All Data</a>'
+                    st.markdown(href, unsafe_allow_html=True)
+                    st.success("Download package prepared. Click the link above to download.")
+                else:
+                    st.warning("No files found for the selected area and options.")
+
+        if st.button("Download Forest Data"):
+            with st.spinner("Downloading forest data..."):
+                with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.geojson') as tmp:
+                    geojson_forest(bbox_lv95, tmp.name)
+                    gdf = gpd.read_file(tmp.name)
+                    st.write(gdf)
+                    # Provide download link for forest data
+                    with open(tmp.name, 'r') as f:
+                        forest_data = f.read()
+                    b64 = base64.b64encode(forest_data.encode()).decode()
+                    href = f'<a href="data:application/json;base64,{b64}" download="forest_data.geojson">Download Forest Data</a>'
+                    st.markdown(href, unsafe_allow_html=True)
+                os.unlink(tmp.name)
+                st.success("Forest data prepared. Click the link above to download.")
+    else:
+        st.error("Selected area is outside Switzerland. Please select an area within Switzerland.")
 
-if __name__ == "__main__":
-    main()
+st.sidebar.info("This application allows you to download various types of geospatial data for Switzerland. Select the data types you want, draw a bounding box on the map, and click 'Get Download Package' to prepare all data for download.")