#!/usr/bin/env python3

import operator
import os
import re
import argparse
import csv
import sys

# Parse the input files and create an associative array of frequencies for each item
def parse(args):
    items = {}
    total_items = 0

    for filename in os.listdir(args.path):
        if re.search(args.string, filename) and ".csv" in filename:
            f = open
            print("parsing", filename)
            with f(args.path + filename, 'rb') as data_file:
                for line in data_file:
                    line = line.strip()
                    if line == "":
                        continue
                    if args.data == "hashtags" or args.data == "urls" or args.data == "mentions":
                        line = line.lower()
                        if b'na' in line and line in b'na':
                            continue
                        if b';' in line:
                            multi_tags = line.split(b';')
                            for tag in multi_tags:
                                total_items += 1
                                try:
                                    items[tag]
                                except KeyError:
                                    items[tag] = 1
                                else:
                                    items[tag] += 1
                            continue
                    total_items += 1
                    try:
                        items[line]
                    except KeyError:
                        items[line] = 1
                    else:
                        items[line] += 1

    label1 = b'(Total Occurences)'    
    if args.data == "screen_name":
        label1 = b'(Total Tweets)'
    elif args.data == "mentions":
        label1 = b'(Total Mentions)'
    items[label1] = total_items
    return items

# Sort the list by most frequent to least.
# Also collapses any items that don't make the top cut into an "other" category.
def sort_list(list):
    sorted_list = sorted(list.items(), key=operator.itemgetter(1), reverse=True)

    i = 0
    rest = 0
    short_list = []
    for item in sorted_list:
        i += 1
        if i 

    list = parse(args)
    sorted_list = sort_list(list)
    output_csv(args, sorted_list)
