application_vnd.jupyter.stderr_output_1.png
from datetime import datetime

from usgs_scraper import extra_data_scraper

import pandas as pd

import re

import requests

import sys

"""
Get all the monitoring locations for a state from the USGS Water Services API.
Input:
    The state we want data from (Arizona, New York, etc.)
Output:
    A CSV of all monitoring locations, including:
        - the agency monitoring them (usually USGS)
        - monitoring location ID number
        - name of monitoring location
        - latitude
        - longitude
        - lat/long type
        - county
        - hydrologic unit
        - drainage area (square miles)
        - datum of gage (AKA elevation)
    Example CSV:
        agency
Data:
    USGS National Water Dashboard:
        https://dashboard.waterdata.usgs.gov/app/nwd/en/?aoi=default
    USGS Water Services API:
        https://waterdata.usgs.gov/nwis/rt
    USGS Water Services API URL builder tool:
        https://waterservices.usgs.gov/rest/IV-Test-Tool.html
"""
import csv
import numpy as np
import pandas as pd
import re
import requests
import sys
from datetime import datetime
from usgs_scraper import extra_data_scraper
def add_location_details(row):
    """
    Scrapes the following from the USGS website and adds it to the dataframe for the current row:
        - latitude
        - longitude
        - lat/long type
        - county
        - hydrologic unit
        - drainage area (square miles)
        - datum of gage (AKA elevation)
    """
    location_id = row['location_id']
    scraped_data = extra_data_scraper(location_id)
    row['latitude'] = scraped_data['latitude']
    row['longitude'] = scraped_data['longitude']
    row['lat_long_type'] = scraped_data['lat_long_type']
    row['county'] = scraped_data['county']
    row['hydrologic_unit'] = scraped_data['hydrologic_unit']
    row['drainage_area'] = scraped_data['drainage_area']
    row['datum_of_gage'] = scraped_data['datum_of_gage']
    row['datum_type'] = scraped_data['datum_type']
    return row
def main(arg):
    state = arg
    print(f'Getting USGS streamgage data for state: {state}')
    timestamp = datetime.now().strftime('%Y%m%d')
    outfile = f'usgs_streamgages_{state}_{timestamp}.csv'
    data_type = 'rdb'
    water_api_url = f'https://nwis.waterservices.usgs.gov/nwis/iv/?format={data_type}&stateCd={state}&parameterCd=00060,00065&siteStatus=all'
    response = requests.get(water_api_url)
    text = response.text
    lines = text.split('\n')
    lines = lines[17:]
    monitoring_locations = []
    for line in lines:
        if line[:5] == '# ---':
            break
        else:
            regex = '#    ([a-zA-Z]+) ([0-9]+) ([A-Za-z0-9\\.\\,\\s\\-\\@\\(\\)]+)'
            extraction = re.search(regex, line)
            agency = extraction.group(1)
            location_id = extraction.group(2)
            name = extraction.group(3)
            monitoring_locations.append({'agency': agency, 'location_id': location_id, 'name': name})
    df = pd.DataFrame(monitoring_locations)
    df = df.apply(add_location_details, axis=1)
    print(df)
    df.to_csv(outfile)
if __name__ == '__main__':
    if sys.argv[1]:
        state = sys.argv[1]
        main(state)
    else:
        default_state = 'az'
        main(default_state)