#!/usr/bin/env python
'''
harvest.py

A set of modules to support downloading and synchronizing a WAF
'''
from catalog_harvesting.waf_parser import WAFParser
from catalog_harvesting.erddap_waf_parser import ERDDAPWAFParser
from catalog_harvesting.csw import download_csw
from catalog_harvesting import get_logger, get_redis_connection
from catalog_harvesting.records import parse_records
from catalog_harvesting.ckan_api import get_harvest_info, create_harvest_job
from catalog_harvesting.notify import Mail, Message, MAIL_DEFAULT_SENDER
from hashlib import sha1
from pymongo import MongoClient
from datetime import datetime
from base64 import b64encode
import requests
import os
import time
import redis


def download_from_db(conn_string, dest):
    '''
    Download several WAFs using collections from MongoDB as a source

    :param str conn_string: MongoDB connection string
    :param str db_name: The name of the MongoDB database to connect to
    :param str dest: Write directory destination
    '''

    tokens = conn_string.split('/')
    if len(tokens) > 3:
        db_name = tokens[3]
    else:
        db_name = 'default'

    db = MongoClient(conn_string)[db_name]
    for harvest in list(db.Harvests.find({"publish": True})):
        try:
            download_harvest(db, harvest, dest)
        except KeyboardInterrupt:
            # exit on SIGINT
            raise
        except:
            get_logger().exception("Failed to harvest")