text_plain_output_1.png
from collections import defaultdict

from functools import lru_cache

from geopy.geocoders import Nominatim

from tqdm.auto import tqdm

import numpy as np

import pandas as pd

import pycountry

import pycountry_convert as pc

import re

import spacy

countries_only = pd.read_json('../input/precise-location/countries.json')
country_ex = pd.read_csv('../input/precise-location/country_iso_codes_expanded.csv')
country_ex = country_ex.fillna(' ')
contriesSet = []
for i in tqdm(range(len(countries_only))):
    contriesSet.append(countries_only['name'][i])
    contriesSet += list(countries_only['translations'][i].values())
    contriesSet = list(set(contriesSet))
dictMultiLangCountry = {}
for i in tqdm(range(len(countries_only))):
    dictMultiLangCountry[countries_only['name'][i]] = countries_only['name'][i]
    mc = list(countries_only['translations'][i].values())
    for c in mc:
        dictMultiLangCountry[c] = countries_only['name'][i]
for i in tqdm(range(len(country_ex))):
    for col in range(26):
        if country_ex[f'alternative_country_name_{col}'][i] != ' ':
            dictMultiLangCountry[country_ex[f'alternative_country_name_{col}'][i]] = country_ex['country'][i]
dictMultiLangCountry['China'] = 'China'
dictMultiLangCountry['SIngapore'] = 'Singapore'
dictMultiLangCountry.pop('')

geoname_cities = pd.read_csv('../input/precise-location/geonames-all-cities-with-a-population-1000.csv', sep=';')
dictAllUnicodeCities = dict()
for i in tqdm(range(len(geoname_cities))):
    if geoname_cities.Name[i] in dictAllUnicodeCities:
        dictAllUnicodeCities[geoname_cities.Name[i]].append(geoname_cities['Country name EN'][i])
        dictAllUnicodeCities[geoname_cities.Name[i]] = list(set(dictAllUnicodeCities[geoname_cities.Name[i]]))
    else:
        dictAllUnicodeCities[geoname_cities.Name[i]] = [geoname_cities['Country name EN'][i]]
    if geoname_cities['Alternate Names'][i] is not np.NaN:
        alt_ = geoname_cities['Alternate Names'][i].split(',')
        for city in alt_:
            if city in dictAllUnicodeCities:
                dictAllUnicodeCities[city].append(geoname_cities['Country name EN'][i])
                dictAllUnicodeCities[city] = list(set(dictAllUnicodeCities[city]))
            else:
                dictAllUnicodeCities[city] = [geoname_cities['Country name EN'][i]]
dictAllUnicodeCities['Dhaka'] = ['Bangladesh']
dictAllUnicodeCities['Changi'] = ['Singapore']
dictAllUnicodeCities['Admiralty'] = ['Singapore']
dictAllUnicodeCities['North Bridge'] = ['Singapore']
dictAllUnicodeCities['Greater Bay'] = ['China']
dictAllUnicodeCities['Philadelphia'] = ['United States']
dictAllUnicodeCities['Manila'] = ['Philippines']
dictAllUnicodeCities['Irving'] = ['Singapore']
dictAllUnicodeCities['HDB Hub'] = ['Singapore']
dictAllUnicodeCities['s Heerenberg'] = ['Netherlands']
dictAllUnicodeCities['S Heerenberg'] = ['Netherlands']
dictAllUnicodeCities['Durham'] = ['United States']
dictAllUnicodeCities['dhabi'] = ['United Arab Emirates']
dictAllUnicodeCities['Scotland'] = ['United Kingdom']
dictAllUnicodeCities['Middle East'] = ['United Arab Emirates']
dictAllUnicodeCities['Alberta'] = ['Canada']
dictAllUnicodeCities['Colombo'] = ['Sri Lanka']
dictAllUnicodeCities['Phoenix'] = ['United States']
dictAllUnicodeCities.pop('Ahmad')
dictAllUnicodeCities.pop('Road')
dictAllUnicodeCities.pop('Street')
dictAllUnicodeCities.pop('30')
dictAllUnicodeCities.pop('North')
dictAllUnicodeCities.pop('West')
dictAllUnicodeCities.pop('Bridge')
dictAllUnicodeCities.pop('Model')
dictAllUnicodeCities.pop('Spa')
dictAllUnicodeCities.pop('Park')
dictAllUnicodeCities.pop('Bay')
dictAllUnicodeCities.pop('Home')
dictAllUnicodeCities.pop('List')
dictAllUnicodeCities.pop('China')
dictAllUnicodeCities.pop('Court')
dictAllUnicodeCities.pop('Wing')
dictAllUnicodeCities.pop('HDB')
dictAllUnicodeCities.pop('Al')
dictAllUnicodeCities.pop('Bin')
dictAllUnicodeCities.pop('A')
dictAllUnicodeCities.pop('Vista')
dictAllUnicodeCities.pop('Aria')
dictAllUnicodeCities.pop('No')
dictAllUnicodeCities.pop('I')

countries_cities = pd.read_json('../input/precise-location/countriescities.json')
citiesList = []
for i in range(len(countries_cities)):
    for j in range(len(countries_cities['cities'][i])):
        citiesList.append(countries_cities['cities'][i][j].get('name'))

states_only = pd.read_json('../input/precise-location/states.json')
statesCountriesDict = {}
for i in range(len(states_only)):
    statesCountriesDict[states_only['name'][i]] = states_only['country_name'][i]

from collections import defaultdict
countries_states = pd.read_json('../input/precise-location/countriesstates.json')
statesList = []
statesNameToStateCode = defaultdict(list)
statesCodeToStatesName = defaultdict(list)
for i in range(len(countries_states)):
    for j in range(len(countries_states['states'][i])):
        statesList.append(countries_states['states'][i][j].get('name'))
        statesNameToStateCode[countries_states['states'][i][j].get('name')].append(countries_states['states'][i][j].get('state_code'))
        statesCodeToStatesName[countries_states['states'][i][j].get('state_code')].append(countries_states['states'][i][j].get('name'))

def get_ngram(text, WordsToCombine=2):
    text = text.replace(',', '')
    words = text.split()
    output = []
    for i in range(len(words) - WordsToCombine + 1):
        output.append(' '.join(words[i:i + WordsToCombine]))
    return output

from re import T
def detect_language(text):
    nlp = spacy.blank('xx')
    nlp.add_pipe('language_detector')
    doc = nlp(text)
    return doc._.language
cid_mapper = {country.name: country.alpha_2 for country in pycountry.countries}
@lru_cache(maxsize=128)
def preproc(locstr):
    pattern = re.compile('#[0-9]+\\-[0-9]+|Headquarters|HQ|Town|Court|Access|via|City|Head|Bank|Center|Remote|Building|Office|City|Of| And|Transportation|D.C|feild|Metro|Tn.|Health Care|Health|Care|STE 100|Sector|Tower|[-&/|;:\\"]+|Area|Surrounding|[\\(\\)\\{\\}\\[\\]]|📍 |🌎', flags=re.IGNORECASE)
    return pattern.sub(' ', locstr).strip().rstrip('.')
road_re = re.compile('^.*?(road|street)(?!\\w)', flags=re.IGNORECASE)
def get_road(locstr):
    m = road_re.match(locstr)
    if m:
        return (m.group(), m.group(1))
    else:
        return (locstr, '')
def get_road_idn(locstr):
    if locstr.split(' ')[0] in ('Jl', 'Jl.', 'JL', 'Jalan', 'jl', 'Jln', 'Jln.'):
        return True
    for i in locstr.split(' '):
        for j in ['Jl', 'Jl.', 'JL', 'jl']:
            if i == j:
                return True
@lru_cache(maxsize=128)
def get_all_geoname(locstr):
    try:
        geolocator = Nominatim(user_agent='geopy25', timeout=3)
        location = geolocator.geocode(locstr)
        location = geolocator.reverse('{}, {}'.format(str(location.raw['lat']), str(location.raw['lon'])), exactly_one=True)
        address = location.raw['address']
        city = address.get('city', '')
        state = address.get('state', '')
        country = address.get('country', '')
        return (city, state, country)
    except:
        pass
@lru_cache(maxsize=128)
def get_geoname_from_road(roadstr):
    try:
        geolocator = Nominatim(user_agent='geopy2', timeout=3)
        location = geolocator.geocode(roadstr)
        addr = location.address
        addr = addr.split(',')
        country = addr[-1]
        state = addr[-3]
        city = addr[-4]
        return (city, state, country)
    except:
        pass
def get_result(countries):
    countries = str(countries)
    if countries is not '':
        country_id = cid_mapper.get(countries.strip())
    else:
        country_id = ''
    if country_id:
        try:
            continent_code = pc.country_alpha2_to_continent_code(country_id.strip())
        except:
            continent_code = ''
    else:
        continent_code = ''
    if continent_code:
        continent_name = pc.convert_continent_code_to_continent_name(continent_code.strip())
    else:
        continent_name = ''
    return {'country': countries, 'country_code': country_id, 'region': continent_name, 'region_code': continent_code}

from itertools import count

def get_locations(locstr):

    if (locstr == ""):
        return {
            "city":"",
            "state":"",
            "country":"",
            "country_code": "",
            "region":"",
            "region_code":""}

    states, cities, countries, country_id, continent_code, continent_name = "", "", "", "", "", ""
    
    if detect_language(locstr) not in ['ja', 'ko', 'zh']:
        locstr = preproc(locstr)
        locstr = re.sub(r' , ', ', ', locstr)
        locstr = re.sub(r',', ' ', locstr)
        locstr = re.sub(r'\s\s+', ' ', locstr)
        # print(locstr)
        if locstr.isupper() or locstr.islower():
            locstr = locstr.title()
        locstr = ' '.join(re.sub(r"([A-Z]+[a-z])", r" \1", locstr).split())

#     print(locstr)

    for i in locstr.replace(',', '').split(' ')[::-1]:
        if i in dictMultiLangCountry:
            countries = dictMultiLangCountry[i]
            if (countries != "") or (countries is not None):
                # print('5 ' + countries)
                return get_result(countries)
    
    for i in locstr.replace(',', '').split(' ')[::-1]:
        if i in dictAllUnicodeCities:
            countries = dictAllUnicodeCities[i]
            if len(countries) == 1:
                countries = countries[0]
                if (countries != "") or (countries is not None):
                    # print('6 ' + countries)
                    return get_result(countries)
            else:
                countries = ""
                    
    if (countries == "") :
        trigram = get_ngram(locstr, 3)
        if countries is "":
            for gram in trigram:
                if gram in dictMultiLangCountry:
                    countries = dictMultiLangCountry[gram]
                    if (countries != "") or (countries is not None):
                        # print('1 ' + countries)
                        return get_result(countries)
                if gram in dictAllUnicodeCities:
                    countries = dictAllUnicodeCities[gram]
                    if len(countries) == 1:
                        countries = countries[0]
                        if (countries != "") or (countries is not None):
                            # print('2 ' + countries)
                            return get_result(countries)
                    else:
                        countries = ""

        bigram = get_ngram(locstr)
        if countries is "":
            for gram in bigram:
                if gram in dictMultiLangCountry:
                    countries = dictMultiLangCountry[gram]
                    if (countries != "") or (countries is not None):
                        # print('3 ' + countries)
                        return get_result(countries)
                if gram in dictAllUnicodeCities:
                    countries = dictAllUnicodeCities[gram]
                    if len(countries) == 1:
                        countries = countries[0]
                        if (countries != "") or (countries is not None):
                            # print('4 ' + countries)
                            return get_result(countries)
                    else:
                        countries = ""

    ############################################################################
    # ROAD REFORMATING (ex: 159, Sin Ming Road # 07-02 Lobby 2 Amtech Building
    # --> 159, Sin Ming Road)
    ############################################################################
    locstr, road = get_road(locstr)
    
    if len(locstr.split(" ")[0]) == 1:
        locstr = locstr[2:]

    if get_road_idn(locstr):
        countries = "Indonesia"
        return get_result(countries)
    ############################################################################
    # ROAD
    ############################################################################
    if re.findall('[0-9]+', locstr) or road:
        try:
            cities, states, countries = get_geoname_from_road(locstr)
            countries = countries.strip()
            if countries is not "":
                if countries in dictMultiLangCountry:
                    return get_result(countries)
            if cities:
                countries = dictAllUnicodeCities[cities]
                if len(countries) == 1:
                    countries = countries[0]
                else: 
                    countries = ""
                if countries is not "":
                    if countries in dictMultiLangCountry:
                        return get_result(countries)
            if states:
                countries = statesCountriesDict[states]
                if countries is not "":
                    if countries in dictMultiLangCountry:
                        return get_result(countries)
        except:
            pass
    
    ############################################################################
    ## THIS CODE TO SOLVE CITY - STATE CODE FORMAT (ex: Bonney Lake, WA) #######
    ############################################################################ 
    loc_split_by_comma = locstr.split(",")
    if (len(loc_split_by_comma) == 2):
        if len(loc_split_by_comma[-1].strip()):
            if statesCodeToStatesName.get(loc_split_by_comma[-1].strip()) and len(statesCodeToStatesName.get(loc_split_by_comma[-1].strip())) > 1:
                if loc_split_by_comma[0] in set(citiesList):
                    cities = loc_split_by_comma[0]
                try:
                    cities, states, countries = get_all_geoname(locstr)
                    if countries is not "":
                        if countries in dictMultiLangCountry:
                            countries = dictMultiLangCountry[countries]
                            return get_result(countries)
                except:
                    states, countries = "", ""
                if type(states) == list:
                    states = states[0]
                    if states:
                        countries = statesCountriesDict[states]
                        if countries is not "":
                            if countries in dictMultiLangCountry:
                                countries = dictMultiLangCountry[countries]
                                return get_result(countries)

            elif statesCodeToStatesName.get(loc_split_by_comma[-1].strip()) and len(statesCodeToStatesName.get(loc_split_by_comma[-1].strip())) == 1:
                cities = loc_split_by_comma[0]
                states = statesCodeToStatesName.get(loc_split_by_comma[-1].strip())
                if type(states) == list:
                    states = states[0]
                    if states:
                        countries = statesCountriesDict[states]
                        if countries is not "":
                            if countries in dictMultiLangCountry:
                                countries = dictMultiLangCountry[countries]
                                return get_result(countries)
        else:
            locstr = locstr

    #########################################################################
    #### IF CURRENT RESULT JUST HAS A CITY
    #########################################################################
    try:
        if (cities != '') and (states == '') and (countries == ''):
            _, states, countries = get_all_geoname(cities)
            if countries is not "":
                if countries in dictMultiLangCountry:
                    countries = dictMultiLangCountry[countries]
                    return get_result(countries)
    except:
        country_id, countries = "", ""
    #########################################################################
    #### IF ALL KEY IS NULL / EMPTY STRING 
    #########################################################################
    try:
        if (cities is "") and (states is "") and (countries != '') or (countries == ''):
            _, _, countries = get_all_geoname(locstr)
            if countries is not "":
                if countries in dictMultiLangCountry:
                    countries = dictMultiLangCountry[countries]
                    return get_result(countries)
    except:
        country_id, countries = "", ""

    return {
                "country":"",
                "country_code":"",
                "region":"",
                "region_code":""
            }    

sample = 'City of Fond du Lac'
loc = get_locations(sample)

sample = '3501 NW Lowell Street Suite 202, Silverdale, WA 98383'
loc = get_locations(sample)
print(loc)