# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script collects statistics from DRAT run on multiple repositories.
# Please see help() method to understand the usage

# author: karanjeets


import sys
import os
import subprocess
import time
import shutil
import datetime
import csv
import urllib2
import json
import xmlrpclib
import getopt

# Check for environment variables
def check_env_var():
	if os.getenv("DRAT_HOME") == None:
		print "Environment variable $DRAT_HOME is not set."
		sys.exit(1)
	if os.getenv("JAVA_HOME") == None:
		print "Environment variable $JAVA_HOME is not set."
		sys.exit(1)
	if os.getenv("OPSUI_URL") == None:
		print "Environment variable $OPSUI_URL is not set."
		sys.exit(1)
	if os.getenv("SOLR_URL") == None:
		print "Environment variable $SOLR_URL is not set."
		sys.exit(1)
	if os.getenv("WORKFLOW_URL") == None:
		print "Environment variable $WORKFLOW_URL is not set."
		sys.exit(1)


# Returns Current Date Time
def current_datetime():
	dt = datetime.datetime.now()
	return dt.strftime("%Y-%m-%dT%H:%M:%SZ")


# Returns a normalized path. 
# Removes the first "/" character and replaces remaining "/" with "_"
def normalize_path(repository):
	tmp = repository[repository.index("/") + 1:]
	tmp = tmp.replace("/", "_")
	tmp = tmp + "_" + current_datetime()
	return tmp


# Count the number of files in a directory recursively
# Leverages a basic utility to exclude some files as well
def count_num_files(path, exclude):
	count = 0
	for root, dirs, files in os.walk(path):
		for filename in files:
			if exclude not in os.path.join(root, filename):
				count += 1
	return count


# Prints usage of this script
def help():
	print >>sys.stderr, "\n\nUsage: python dratstats.py <path to list of repository URLs> <path to output directory>\n"


# Printing out on Console
def printnow(string):
	print string
	sys.stdout.flush()


# Parsing RAT log files
def parseFile(filepath):
   f = open(filepath, 'r')
   lines = f.readlines()
   notes = 0
   binaries = 0
   archives = 0
   standards = 0
   apachelicensed = 0
   generated = 0
   unknown = 0

   for line in lines:
      if line.startswith('Notes:'):
         notes = notes + int(line.split(':')[1].strip())
      if line.startswith('Binaries:'):
         binaries = binaries + int(line.split(':')[1].strip())
      if line.startswith('Archives:'):
         archives = archives + int(line.split(':')[1].strip())
      if line.startswith('Standards:'):
         standards = standards + int(line.split(':')[1].strip())
      if line.startswith('Apache Licensed:'):
         apachelicensed = apachelicensed + int(line.split(':')[1].strip())
      if line.startswith('Generated:'):
         generated = generated + int(line.split(':')[1].strip())
      if line.find('Unknown Licenses') != -1:
         unknown = unknown + int(line.split(' ')[0].strip())
         return (notes, binaries,archives,standards,apachelicensed,generated,unknown)
         
   return (-1,-1,-1,-1,-1,-1,-1)


# OODT Process (start, stop)
def oodt_process(command):
	try:
		retcode = subprocess.call("${DRAT_HOME}/bin/oodt" + " " + command, shell=True)
		if retcode < 0:
			print >>sys.stderr, "ODDT process was terminated by signal", -retcode, ". OODT failed to " + command + ". Aborting..."
			sys.exit(1)
		elif retcode > 0:
			print >>sys.stderr, "OODT process returned", retcode, ". OODT failed to " + command + ". Aborting..."
			sys.exit(1)
	except OSError as e:
		print >>sys.stderr, "OODT execution failed:", e, ". OODT failed to " + command + ". Aborting..."
		sys.exit(1)


# DRAT process (crawl, index, map, reduce)
def drat_process(command, repository):
	retval = True
	try:
		retcode = 0
		if command == "crawl" or command == "index":
			retcode = subprocess.call("${DRAT_HOME}/bin/drat" + " " + command + " " + repository, shell=True)
		elif command == "map" or command == "reduce":
			retcode = subprocess.call("nohup ${DRAT_HOME}/bin/drat" + " " + command + " &", shell=True)
		if retcode < 0:
			print >>sys.stderr, "DRAT " + command + " process was terminated by signal", -retcode, ". Aborting..."
			retval = False
		elif retcode > 0:
			print >>sys.stderr, "DRAT " + command + " process returned", retcode, ". Aborting..."
			retval = False
	except OSError as e:
		print >>sys.stderr, "DRAT " + command + " execution failed:", e, ". Aborting..."
		retval = False
	return retval


# Reset DRAT
def drat_reset():
	printnow ("Removing  " + os.getenv("DRAT_HOME") + "/data/workflow")
	shutil.rmtree(os.getenv("DRAT_HOME") + "/data/workflow")
	printnow ("Removing  " + os.getenv("DRAT_HOME") + "/filemgr/catalog")
	shutil.rmtree(os.getenv("DRAT_HOME") + "/filemgr/catalog")
	printnow ("Removing  " + os.getenv("DRAT_HOME") + "/solr/drat/data")
	shutil.rmtree(os.getenv("DRAT_HOME") + "/solr/drat/data")
	printnow ("Removing  " + os.getenv("DRAT_HOME") + "/data/archive")
	shutil.rmtree(os.getenv("DRAT_HOME") + "/data/archive")
	os.mkdir(os.getenv("DRAT_HOME") + "/data/archive")
	printnow ("Removing  " + os.getenv("DRAT_HOME") + "/data/jobs")
	shutil.rmtree(os.getenv("DRAT_HOME") + "/data/jobs")
	os.mkdir(os.getenv("DRAT_HOME") + "/data/jobs")


# Check if there are any pending PGE jobs in the queue
def job_in_queue(job_name):
	status = "PGE EXEC"
	server = xmlrpclib.ServerProxy(os.getenv("WORKFLOW_URL"), verbose=False)
	response = server.workflowmgr.getWorkflowInstancesByStatus(status)

	for i in range(0, len(response)):
		#print response[i]["sharedContext"]["TaskId"]
		if response[i]["sharedContext"]["TaskId"][0] == job_name:
			return True

	return False


# Wait for job to complete
def wait_for_job(job_name):
	while job_in_queue(job_name):
		for i in range(1, 11):
			sys.stdout.write('.')
			sys.stdout.flush()
			time.sleep(2)


# Run DRAT and collect statistics
def run(repos_list, output_dir):
	with open(repos_list) as repositories:
		for repository in repositories:
			repository = repository.strip()
			printnow ("\nVerifying repository path...\n")
			if not os.path.exists(repository):
				printnow ("\nPath " + repository + "is not valid. Skipping and moving on...\n")
				continue
			printnow ("\nRepository Path: OK\n")

			printnow ("\nStarting OODT...\n")
			oodt_process("start")
			time.sleep(20)
			printnow ("\nOODT Started: OK\n")

			printnow ("\nRunning DRAT on " + repository + " ...\n")
			
			retval = True
			stats = {}
			stats['id'] = repository

			stats['crawl_start'] = current_datetime()
			retval = drat_process("crawl", repository)
			stats['crawl_end'] = current_datetime()

			if retval: