<file_sep>__all__ = ["BaseWorker"]
from ApiRepl.BaseWorker import BaseWorker
<file_sep>/*queue*/
CREATE TABLE queue
  (
    `id`          INT auto_increment NOT NULL,
    `priority`    INT DEFAULT 0,
    `type`        VARCHAR(64) NOT NULL,
    `min`         VARCHAR(64),
    `max`         VARCHAR(64),
    `started` DATETIME,
    `finished` DATETIME,
    PRIMARY KEY (id)
  );

CREATE TABLE errors
  (
    `id`     INT auto_increment NOT NULL,
    `state`  VARCHAR(64) NOT NULL,
    `error`  TEXT,
    `source` INT NOT NULL,
    PRIMARY KEY (id)
  );

/*FetchLog*/
CREATE TABLE fetchlog
  (
    `id`           INT auto_increment NOT NULL,
    `recordsadded` INT NOT NULL DEFAULT 0,
    `timestamp` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    `source`       INT NOT NULL,
    FOREIGN KEY (source) references queue(id) on delete cascade,
    PRIMARY KEY (id)
  );
<file_sep>insert into queue(type, min, max) values ("undefined","a","z");
<file_sep># ApiRepl

## What it is
If you want API data copied to your database, the workers should fetch for you elastically.

## Setup
### Database Requirement
use schema.sql to create a database, and load it with the types of information you need
### Subclass BaseWorker
see samples/usage.py for an example.
Create an api method and save method for your workflow by subclassing BaseWorker.

## Functionality (requirements)
* Splits update task into pieces
* Prioritizes pieces
* Can be paused and resumed
* Can scale across hosts easily using DB
* Good test coverage (soon :) )

## Usage
See Sample folder
<file_sep>"""
To demonstrate how to use this class, here is one way to use this tool.
"""

from ApiRepl import BaseWorker
import random

class PersonWorker(BaseWorker):
    """ A fictional worker for a person api. """
    def __init__(self, *args, **kwargs):
        BaseWorker.__init__(self, *args, **kwargs)

    def api(self):
        """ Specalization for this api. """
        self.finished=True
        return {'TEST':'Success'}


def saveobj(item):
    return item

def throttle():
    """Simulate a kill condition."""
    return random.random() > 0.95

def test_specalization():
    worker = PersonWorker(type="undefined")
    for item in worker:
        if item is 0 or throttle():
            break
        saveobj(item)
    del worker
<file_sep>"""
To demonstrate how to use this class, here is one way to use this tool.
"""
import json
import urllib2
import random
import pymysql

from ApiRepl import BaseWorker

"""
Sample Schema:
Person:
- Name
- DOB
- Height
"""

class PersonWorker(BaseWorker):
    """ A fictional worker for a person api. """
    def __init__(self, *args, **kwargs):
        BaseWorker.__init__(self, *args, **kwargs)

    def api(self):
        """ Specalization for this api. """
        # get a list of names
        people = json.load(urllib2.urlopen("api.people.com/"))
        # toss out those out of range
        lookup = [p for p in people.keys() if
                  p > self.last and p <= self.maximum]
        if len(lookup) == 0:
            # need to know when finished, so that it can resume or stop
            self.finished = True
            return 0
        who = sorted(lookup)[0]
        self.last = who
        person = json.load(urllib2.urlopen(
            "api.people.com/{item}".format(item=who)))
        return person


def saveobj(item):
    """Save each object from the worker."""
    # connect to database
    cursor = pymysql.connect(host='localhost', db='fetch').cursor()
    cursor.execute('insert into Person (Name, DOB, Height) values (%s %s %s)',
                   (item['Name'], item['DOB'], item['Height']))


def throttle():
    """Simulate a kill condition."""
    return random.random() > 0.95

if __name__ == "__main__":
    worker = PersonWorker(type="person")
    for item in worker:
        if item is 0 or throttle():
            del worker
        saveobj(item)
