# Suppress compatibility warnings
import warnings
warnings.filterwarnings('ignore')


import pandas as pd
import numpy as np
from nltk.corpus.reader.plaintext import PlaintextCorpusReader

text_dir = '../Data/Texts/'
corpus = PlaintextCorpusReader(text_dir, '.*\.txt')

# A function to turn fileids into a table of metadata
def parse_fileids(fileids):
    '''Takes a list of file names formatted like A-Cather-Antonia-1918-F.txt.
       Returns a pandas dataframe of derived metadata.'''
    import pandas as pd
    meta = {}
    for fileid in fileids:
        file = fileid.strip('.txt') # Get rid of file suffix
        fields = file.split('-') # Split on dashes
        fields[2] = fields[2].replace('_', ' ') # Remove underscore from titles
        fields[3] = int(fields[3])
        meta[file] = fields
    metadata = pd.DataFrame.from_dict(meta, orient='index') # Build dataframe
    metadata.columns = ['nation', 'author', 'title', 'pubdate', 'gender'] # Col names
    return metadata.sort_index() # Note we need to sort b/c datframe built from dictionary

def collect_stats(corpus):
    '''Takes an NLTK corpus as input. 
       Returns a pandas dataframe of stats indexed to fileid.'''
    import nltk
    import pandas as pd
    stats = {}
    for fileid in corpus.fileids():
        word_count = len(corpus.words(fileid))
        stats[fileid.strip('.txt')] = {'wordcount':word_count}
    statistics = pd.DataFrame.from_dict(stats, orient='index')
    return statistics.sort_index()

books = parse_fileids(corpus.fileids())
stats = collect_stats(corpus)
books = books.join(stats)
books.index.set_names('file', inplace=True)
books.head()


def get_loc(line):
    '''Takes a string of NER output. 
       Returns a location if found, else None.'''
    line = line.split('\t')
    try:
        if line[1] == 'LOCATION':
            return line[0]
        else:
            return None
    except:
        return None

def ingest_ner(f):
    '''Take a file handle for an NER output file.
       Uses get_loc() to extract locations.
       Returns a dict of locations and counts.'''
    from collections import defaultdict
    locations = defaultdict(lambda: 0)
    for line in f:
        loc = get_loc(line)
        if loc:
            locations[loc] += 1
    return locations
    
    
    import os # Used to create OS-appropriate file paths
ner_dir = '../Data/NER/' # Where we keep the NER files

files_list =[]   # Lists to hold our data
locs_list = []
occurs_list = []

for fileid in sorted(corpus.fileids()): # Loop over fileids in corpus
    file = os.path.join(ner_dir, fileid)
    with open(file) as f:
        locations = ingest_ner(f) # Get a dict of locations and counts in this file
        for loc in sorted(locations, key=locations.get, reverse=True): # Loop over keys in dict
            files_list.append(fileid.strip('.txt'))
            locs_list.append(loc)
            occurs_list.append(locations[loc])
            
            
# Turn the lists into a Pandas dataframe
d = {'file': files_list, 
     'location': locs_list,
     'occurs': occurs_list}
geo = pd.DataFrame(d)
print(geo.describe())
geo.head()


# Total number of named location occurrences in corpus
geo.occurs.sum()


%matplotlib inline

import matplotlib as mpl
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

sns.set_style("whitegrid")
sns.set_context("talk")

plt.hist(np.log10(geo.occurs), bins=50)
plt.show()


# Total location counts for full corpus
places = geo.groupby('location')
places.occurs.aggregate(np.sum).head()


# How many different volumes contain at least one mention of each place?
places.occurs.size().head()


place_counts = pd.DataFrame(places.occurs.aggregate(np.sum))
place_counts['volumes'] = places.occurs.size()
print(place_counts.describe())
place_counts.head()


lookups = place_counts[(place_counts['occurs']&gt;5) &amp; (place_counts['volumes']&gt;1)]
lookups.describe()


lookups.occurs.sum()


lookups.head()


# Parameters for geocoding clients
# NOTE: Per-second query rates will quickly use up daily API quota.
#   Need to be careful not to exceed daily quota

import googlemaps

gc_rate  =     50 # Geocoding queries per second
pl_rate  =      5 # Places queries per second
api_key_file = '[INSERT PATH TO GOOGLE API KEY FILE HERE]/google-geo-api-key.txt'

# Get API key from file
try:
    api_key = open(api_key_file).read().strip()
except:
    sys.exit('Cannot get Google geocoding API key. Exiting.')

gc_client = googlemaps.Client(key=api_key, queries_per_second=gc_rate) # For Geocoding API
pl_client = googlemaps.Client(key=api_key, queries_per_second=pl_rate) # For Places API


def get_placeid(string, api_client):
    '''Takes a string and an established googlemaps places API client.
       Returns first place_id associated with that string.
       If not place_id found, returns "ZERO_RESULTS" or None, depending on result status code.'''
    try:
        place = api_client.places(string)
        status = place['status']
        if status == 'OK':
            place_id = place['results'][0]['place_id']
        elif status == 'ZERO_RESULTS':
            place_id = None
        else:
            place_id = None
    except:
        place_id = None
    return place_id

def process_id(placeid, api_client):
    '''Takes a Google place_id and an established googlemaps geocoding API client.
        Looks up and parses geo data for placeid.
        Returns int code on error, else dictionary of geo data.
    '''
    # Define all variables, initial to None
    result = {
    'formatted_address' : None,
    'location_type' : None,
    'country' : None,
    'admin_1' : None,
    'admin_2' : None,
    'locality' : None,
    'colloquial_area' : None,
    'continent' : None,
    'natural_feature' : None,
    'point_of_interest' : None,
    'lat' : None,
    'lon' : None,
    'partial' : None,
    }
    # Perform reverse geocode. Note this needs googlemaps v 2.4.3 or higher
    try:
        data = gc_client.reverse_geocode(placeid)
    except:
        return 1 # Problem with geocoding API call
    
    # Use the first result. Should only be one when reverse geocoding with place_id.
    try:
        data = data[0]
        result['formatted_address'] = data['formatted_address']
        result['location_type'] = data['types'][0]
        result['lat'] = data['geometry']['location']['lat']
        result['lon'] = data['geometry']['location']['lng']
        try:
            result['partial'] = result['partial_match']
        except:
            result['partial'] = False
    except:
        print("   Bad geocode for place_id %s" % (placeid))
        return 2 # Problem with basic geocode result
    
    try:
        for addr_comp in data['address_components']:
            comp_type = addr_comp['types'][0]
            if comp_type == 'locality':
                result['locality'] = addr_comp['long_name']
            elif comp_type == 'country':
                result['country'] = addr_comp['long_name']
            elif comp_type == 'administrative_area_level_1':
                result['admin_1'] = addr_comp['long_name']
            elif comp_type == 'administrative_area_level_2':
                result['admin_2'] = addr_comp['long_name']
            elif comp_type == 'colloquial_area':
                result['colloquial_area'] = addr_comp['long_name']
            elif comp_type == 'natural_feature':
                result['natural_feature'] = addr_comp['long_name']
            elif comp_type == 'point_of_interest':
                result['point_of_interest'] = addr_comp['long_name']
            elif comp_type == 'continent':
                result['continent'] = addr_comp['long_name']
    except:
        return 3 # Problem with address components
    
    return result
    
    
%%time

placeids = {} # Store results ina dictionary keyed to placeid

for loc in lookups.index:
    plid = get_placeid(loc, pl_client)
    placeids[loc] = plid
    
placeids = pd.DataFrame.from_dict(placeids, orient='index')
placeids.columns = ['placeid']
placeids.index.set_names('location', inplace=True)




#placeids = pd.read_csv('../Results/placeids.csv', index_col=0)






placeids.to_csv('../Results/placeids.csv')
placeids.head()




%%time

geodata = {} # Again, store results in a dict keyed to placeid

for plid in placeids['placeid']:
    if plid:
        geodata[plid] = process_id(plid, gc_client)
        
        
geodata_df = pd.DataFrame.from_dict(geodata)
geodata_df = geodata_df.transpose()
geodata_df.index.set_names('placeid', inplace=True)
geodata_df.head()


#geodata_df = pd.read_csv('../Results/geodata_df.csv', index_col=0)


# Write geo data to disk
geodata_df.to_csv('../Results/geodata_df.csv')


# Put all the data together
placeids_geo = placeids.join(geodata_df, on='placeid')
placeids_geo.head()


geo_all = geo.join(placeids_geo, on='location')
geo_all = geo_all[geo_all.placeid.notnull()]
geo_all = geo_all[geo_all['locality']!='Charlotte']
print(geo_all.shape)
geo_all.head()


