
import os
import json
import gzip
import pymongo
import datetime
import multiprocessing as mp

def git_line_json(git_line):
    JSON = {}
    JSON["actor"] = git_line["actor"]["id"]
    JSON["date"] = datetime.datetime.strptime(git_line["created_at"],
                                              '%Y-%m-%dT%H:%M:%SZ')
    JSON["type"] = git_line["type"]
    JSON["repo"] = git_line["repo"]["id"]
    if JSON["type"] == "IssuesEvent":
        JSON["action"] = git_line["payload"]["action"]
        JSON["issue"] = git_line["payload"]["issue"]["id"]
        JSON["issue_url"] = git_line["payload"]["issue"]["url"]
    if JSON["type"] == "PushEvent":
        if "issue_events_url" in git_line["payload"]:
            JSON["issue_url"] = git_line["payload"]["issue_events_url"]
    return JSON

def set_collection(collection,data_dir,gz_files_names):
    d0 = datetime.datetime.now()
    for gz_file in gz_files_names:
        print gz_file
        with gzip.open(data_dir + gz_file) as f:
            file_content = f.read()
        file_content_lines = file_content.split("\n")
        ALL_JSONS = []
        for json_line in file_content_lines:
            try:
                ALL_JSONS.append(git_line_json(json.loads(json_line)))
            except:
                pass
        collection.insert_many(ALL_JSONS)
    df = datetime.datetime.now()
    print "Collections took ",(df-d0).total_seconds()

def set_month(parameters):
    date0 = parameters[0]
    datef = parameters[1]
    data_dir = parameters[2]
    month_string = parameters[3]

    collection = db[month_string]

    gz_files_names = [a for a in os.listdir(data_dir) if "gz" in a]
    print date0
    set_collection(collection, data_dir, gz_files_names[:5])


if __name__=="__main__":
    #data_dir_all_github = "/home/anon/Desktop/Doctorado/Projects/RecurrentNeuralPointProcess/Queues/RawData/GitHub/"

    all_data_dir = "/home/anon/Desktop/Doctorado/Projects/RecurrentPointProcesses/Queues/raw_data/github/all_months/"
    data_dir = "/home/anon/Desktop/Doctorado/Projects/RecurrentNeuralPointProcess/Queues/RawData/GitHub/2015-01/"


    client = pymongo.MongoClient()
    db = client["GitHub"]
    number_of_process = 6

    Pool = mp.Pool(number_of_process)

    #client = pymongo.MongoClient("129.26.78.40:27017")
    #db = client["github"]
    all_months = [a for a in os.listdir(all_data_dir)]
    PARAMETERS = []
    for month_string in all_months:
        year = int(month_string.split("-")[0])
        month = int(month_string.split("-")[1])
        if month < 12:
            date0 = datetime.datetime(year,month,1)
            datef = datetime.datetime(year, month+1, 1)
        else:
            date0 = datetime.datetime(year,month,1)
            datef = datetime.datetime(year+1,1, 1)

        collection = db[month_string]
        data_dir = all_data_dir+month_string+"/"
        PARAMETERS.append((date0,datef,data_dir,month_string))

    Pool.map(set_month,PARAMETERS)


"""
def git_line_json(git_line):
    JSON = {}
    JSON["actor"] = git_line["actor"]["id"]
    JSON["date"] = parser.parse(git_line['created_at'])
    JSON["type"] = git_line["type"]
    JSON["repo"] = git_line["repo"]["id"]
    if JSON["type"] == "IssuesEvent":
        JSON["action"] = git_line["payload"]["action"]
        JSON["issue"] = git_line["payload"]["issue"]["id"]
    return JSON
"""