text:
var FIREBASE_LOVE_URL = 'https://my-cappstone.firebaseio.com/'
var ref = new Firebase('https://my-cappstone.firebaseio.com/quotes');
var loveBtn;
var truthBtn;
var beautyBtn;

var app = angular.module('myApp', []);

// $('#myModal').modal('show');

$( ".quoteBox" ).hide();

// Button click code

loveBtn = $(".loveBtn");
truthBtn = $(".truthBtn");
beautyBtn = $(".beautyBtn");

$(".loveBtn").on("click", function() {

    getQuotes(getLoveQuote);

  });

$(".truthBtn").on("click", function() {

    getQuotes(getTruthQuote);

  });

$(".beautyBtn").on("click", function() {

    getQuotes(getBeautyQuote);

  });

// creates empty array variables to hold objects
loveArr = [];
truthArr = [];
beautyArr = [];

// loops through array
function getQuotes(cb){
  $.get('https://my-cappstone.firebaseio.com/quotes.json')
            .success(function(data) {
                quotes = data;
                for( var i=0; i < Object.keys(quotes).length; i++) {
                    if( quotes[Object.keys(quotes)[i]].category == 'love' ) {
                        loveArr.push({
                          quote: quotes[Object.keys(quotes)[i]].quote,
                          author: quotes[Object.keys(quotes)[i]].author
                        });
                    }
                    else if ( quotes[Object.keys(quotes)[i]].category == 'truth' ) {
                        truthArr.push({
                          quote: quotes[Object.keys(quotes)[i]].quote,
                          author: quotes[Object.keys(quotes)[i]].author
                        });
                    }
                    else if ( quotes[Object.keys(quotes)[i]].category == 'beauty' ) {
                        beautyArr.push({
                          quote: quotes[Object.keys(quotes)[i]].quote,
                          author: quotes[Object.keys(quotes)[i]].author
                        });
                    }
                }
                cb();
            });
}

function getTruthQuote (){
    var rand = Math.floor(Math.random() * truthArr.length);

    var quote = truthArr[rand].quote;
    var author = truthArr[rand].author;

    $("#quote").html(quote)
    $("#author").html(author)
}

function getLoveQuote (){
    var rand = Math.floor(Math.random() * loveArr.length);

    var quote = loveArr[rand].quote;
    var author = loveArr[rand].author;

    $("#quote").html(quote)
    $("#author").html(author)
}

function getBeautyQuote (){
    var rand = Math.floor(Math.random() * beautyArr.length);

    var quote = beautyArr[rand].quote;
    var author = beautyArr[rand].author;

    $("#quote").html(quote)
    $("#author").html(author)
}

//shows & hides quoteBox on button click

$( ".loveBtn" ).click(function() {
  $( ".quoteBox" ).fadeIn( "fast", function() {
    // Animation complete.
  });
});

$( ".truthBtn" ).click(function() {
  $( ".quoteBox" ).fadeIn( "fast", function() {
    // Animation complete.
  });
});

$( ".beautyBtn" ).click(function() {
  $( ".quoteBox" ).fadeIn( "fast", function() {
    // Animation complete.
  });
});

// always run this function
function getJSON(url, cb) {
  var xhr = new XMLHttpRequest();

  xhr.open('GET', url);
  xhr.onload = function () {
    if (this.status >= 200 && this.status < 400) {
      cb(JSON.parse(this.response));
     }
    };

    xhr.send();
}

// // main.js for ZeroClipboard
// var client = new ZeroClipboard( document.getElementById("copy-button") );

// client.on( "ready", function( readyEvent ) {
//   // alert( "ZeroClipboard SWF is ready!" );

//   client.on( "aftercopy", function( event ) {
//     // `this` === `client`
//     // `event.target` === the element that was clicked
//     event.target.style.display = "none";
//     alert("Copied text to clipboard: " + event.data["text/plain"] );
//   } );
// } );

// find template and compile it
// var templateSource = document.getElementById('results-template').innerHTML,
//     template = Handlebars.compile(templateSource),
//     resultsPlaceholder = document.getElementById('results'),
//     playingCssClass = 'playing',
//     audioObject = null;

// var fetchTracks = function (albumId, callback) {
//     $.ajax({
//         url: 'https://api.spotify.com/v1/albums/' + albumId,
//         success: function (response) {
//             callback(response);
//         }
//     });
// };

// var searchAlbums = function (query) {
//     $.ajax({
//         url: 'https://api.spotify.com/v1/search',
//         data: {
//             q: 'artist:' + query,
//             type: 'album',
//             market: "US"
//         },
//         success: function (response) {
//             resultsPlaceholder.innerHTML = template(response);
//         }
//     });
// };

// results.addEventListener('click', function(e) {
//     var target = e.target;
//     if (target !== null && target.classList.contains('cover')) {
//         if (target.classList.contains(playingCssClass)) {
//             audioObject.pause();
//         } else {
//             if (audioObject) {
//                 audioObject.pause();
//             }
//             fetchTracks(target.getAttribute('data-album-id'), function(data) {
//                 audioObject = new Audio(data.tracks.items[0].preview_url);
//                 audioObject.play();
//                 target.classList.add(playingCssClass);
//                 audioObject.addEventListener('ended', function() {
//                     target.classList.remove(playingCssClass);
//                 });
//                 audioObject.addEventListener('pause', function() {
//                     target.classList.remove(playingCssClass);
//                });
//             });
//         }
//     }
// });

// searchAlbums('Beastie Boys');


text: SELECT func1_nosql_stb(func2_mod_int_stb(a)) FROM foo order by 1;

redpajama_set_name: RedPajamaGithub


text:
from os import walk
from datetime import datetime
import re
import pickle
import sys
import time
import warnings
import collections # needed to check for type of variable (collections.Iterable) and exclude duplicates
from bs4 import BeautifulSoup

# 3rd party
import requests
import numpy as np
from numpy import nanmean
from sklearn import linear_model
import pandas as pd

# custom
from flatten import flatten

class Race:
    """Race Class for:
        a) storing all data for a given horse-race
        a) extract that data from a raw race.html
        b) extract that data from horseforms.html
        d) allow to retrieve the stored data as a pd.table"""

    starters=float('nan')
    nonrunners=float('nan')
    raceID=float('nan')
    rtype=float('nan')
    rday=float('nan')
    rtime=float('nan')
    country=float('nan')
    rname=float('nan')
    rnum=float('nan')
    dist=float('nan')
    stakes=float('nan')
    stakes=float('nan')
    weights=float('nan')
    ages=float('nan')
    sex=float('nan')
    starters=float('nan')
    meanodds=float('nan')
    trendodds=float('nan')
    horsenames=float('nan')
    jockeynames=float('nan')
    jockeytrainers=float('nan')
    heritages1=float('nan')
    heritages2=float('nan')
    owners=float('nan')
    odds=float('nan')
    fillfinish=float('nan')
    finishers=float('nan')
    pastpl1=float('nan')
    pastpl2=float('nan')
    pastpl3=float('nan')
    pastpl4=float('nan')
    pastpl5=float('nan')
    lofi_meanodds=float('nan')
    lofi_trendodds=float('nan')
    lofi_nracesrun=float('nan')
    lofi_places=float('nan')
    lofi_pastpl1=float('nan')
    lofi_pastpl2=float('nan')
    lofi_pastpl3=float('nan')
    lofi_pastpl4=float('nan')
    lofi_pastpl5=float('nan')

    def __init__(self, textraw,formen,verbose=False):
        #self.textraw = textraw
        self.description = "Race from Germantote"
        self.author = "MZ"
        self.extract(textraw,verbose) # FILLS ALL VARIABLES ABOVE
        self.getformen(formen) # FILLS ALL VARIABLES ABOVE

    def extract(self,textraw,verbose=False):
        """extract function is called on start and extracts race-specific data"""
        textraw2=re.split(
            '<div id="racecardTopContainer"',
            textraw)[1] # throw away top part of html
        racedaylist,raceslist=re.split(
            '
',
            textraw2) # split rest into race day and single race info
        self.starters=int(re.findall(
                '
(\d+?) Starter
',
                raceslist)[0])
        if not self.starters: self.starters=float('nan')
        self.nonrunners=[bool(x)
                           for x
                           in  re.findall('
',
                           raceslist)]
        if not self.nonrunners: # strange "mix" races
            self.nonrunners=[bool(x)
                               for x
                               in  re.findall('
',
                               raceslist)]
        if len(self.nonrunners)!=self.starters:
            print('Error only '+str(len(self.nonrunners))+ ' nonrunners.')
            self.nonrunners=[bool(0)]*self.starters
        self.runners=[not i for i in self.nonrunners]
        self.ridraw=re.findall('data-raceid="(\d+?)">',racedaylist)[0]
        self.raceID=int(self.ridraw)#racedaylist or raceslist???
        if not self.raceID: self.raceID=float('nan')
        if verbose:
            print(self.raceID)
        self.rtype=re.findall(
            '',
            racedaylist)
        if self.rtype[0] is 'T':
            self.rtype='Flach' # Oddly, Flachrennen get a 'T' by germantote
        elif self.rtype[0] is 'H':
            self.rtype='Trab' # Oddly, Trabrennen get a 'H' by germantote
        else:
            self.rtype=2 # "Other"
        self.rday=datetime.strptime(re.findall(
                'Rennen am (.+?)',
                racedaylist)[0],
                '%d.%m.%Y')
        if not self.rday: self.rday=float('nan')
        self.rtime=datetime.strptime(re.findall(
                '
(\d\d\:\d\d)
',
                raceslist)[0],
                '%H:%M')
        if not self.rtime: self.rtime=float('nan')
        self.country=re.findall(
            'a href="/races\?country=(...)&date=',
            racedaylist)
        if not self.rtime: self.rtime=float('nan')
        self.rname=re.findall(
            ' (.+?)\n',
            racedaylist)
        if not self.rname: self.rname='nan'
        self.rnum=int(re.findall(
                '
\s*(\d+)\s*
',
                raceslist)[0])
        if not self.rnum: self.rname=float('nan')
        self.dist=float(re.findall(
                '
(.+?) m
',
                raceslist)[0])
        if not self.dist: self.dist=float('nan')
        self.stakes=re.findall(
            '
(.+?) (.+?)
',
            raceslist)
        if not self.stakes[0][0]: self.stakes[0][0]=float('nan')
        if not self.stakes[0][0]: self.stakes[0][0]='nan'
        #Extract horse-specific data from racesheet
        #Get starternumbers (str since replacementhorses often get something like 1a 1b)
        self.starterNo1=re.findall(
            '(.+?)',
            raceslist)
        self.starterNo2=re.findall(
            '(.+?)',
            raceslist)
        #Horsenames
        self.horsenames=re.findall(
            '\s*\n\s*?\s*(.+?)\s*?<.a>\n\s*?',
            raceslist)
        if len(self.horsenames)!=self.starters:
            print('Error only '+str(len(self.horsenames))+ ' horsenames.')
        #Extract finish-data
        finish_raw=re.findall(
            '
.*?
',
            textraw,re.S)#get finishtable
        #self.finish_raw=finish_raw
        if finish_raw:
            trows=re.findall(
                '
(.*?)
',
                finish_raw[0],
                re.S) #gets all table rows
            self.finishall=[re.findall('<(?:th|td) .*?>(.*?)</(?:th|td)>',
                                row,re.S)
                            for row
                            in trows] #gets all table headers or table cells for each row
            self.finisherNames=([x[2]
                             for x
                             in self.finishall[1:]]) # gets finisher's starter numbers from second row on in second column
            self.finishersRaw=([x[1]
                             for x
                             in self.finishall[1:]]) # gets finisher's starter numbers from second row on in second column
            self.placementsRaw=([int(x[0])
                             for x
                             in self.finishall[1:]])

            self.finishers=[float('nan')] * self.starters
            #VERSION 1:
            #fill all fields with mean finish of all starters not placed
            #PROBLEM: will systematically affect relationship between starters and finish >> starter only useful as a predictor of winners
            self.fillfinish=[np.mean([self.starters-sum(self.nonrunners),
                              1+len(self.finishersRaw)])]* self.starters
            #VERSION 2:
            #fill all fields with nan
            #PROBLEM: more horses will be dropped
            #self.fillfinish=[float('nan')]* self.starters
            #VERSION 3:
            #fill all fields with random numbers
            #PROBLEM: random variance added
            #IMPLEMENTED IN ANALYSIS-PRE-PROCESSING
            if self.finisherNames: # in case finisher list has names
                for i,finny in enumerate(self.finisherNames):
                    finindex=self.horsenames.index(finny)#get index of finisher
                    self.finishers[finindex]=self.placementsRaw[i]
                    self.fillfinish[finindex]=self.placementsRaw[i]
            elif self.finishersRaw:
                for i,finny in enumerate(self.finishersRaw):
                    finindex=self.starterNo1.index(finny)#get index of finisher
                    self.finishers[finindex]=self.placementsRaw[i]
                    self.fillfinish[finindex]=self.placementsRaw[i]
        else:
            self.finishall=[]
            self.finishersRaw=[]
            self.finishers=float('nan')
            self.fillfinish=float('nan')

        self.jockeynames=re.findall(
            '\s*(.*?)(?:|)'
            ,raceslist)
        if len(self.jockeynames)!=self.starters:
            print('Error only '+str(len(self.jockeynames))+ ' jockeynames.')
        self.jockeytrainers=re.findall(
            '
(.*?)
',
            raceslist)
        if len(self.jockeytrainers)!=self.starters:
            print('Error only '+str(len(self.jockeynames))+ ' jockeytrainers.')
        self.owners=re.findall(
            'Besitzer:\s(.+?)\s+?</div',
            raceslist)
        if len(self.owners)!=self.starters:
            print('Error only '+str(len(self.owners))+ ' owners.')
            self.owners='nan'
        self.heritages=re.findall(
            'Abstammung: (.+?)â\x80\x93 (.+?)\s+Besitzer:',
            raceslist,
            re.S)
        self.heritages1=[heir[0] for heir in self.heritages]
        if len(self.heritages1)!=self.starters:
            print('Error only '+str(len(self.heritages1))+ ' heritages1.')
        self.heritages2=[heir[1] for heir in self.heritages]
        if len(self.heritages2)!=self.starters:
            print('Error only '+str(len(self.heritages2))+ ' heritages2.')
        weight=re.findall(
            '(.+?) kg',#(\d+,*\d*)
            raceslist)
        if weight:
            try:
                self.weights= [float(x.replace(',','.'))
                               for x in weight]
                if len(self.weights)!=self.starters:
                    print('Error only '+str(len(self.weights))+ ' weights in Race: '+str(self.raceID))
            except:
                self.weights= float('nan')
                print('Error extracting weights in Race: '+str(self.raceID))

        else:
            self.weights= float('nan')

        age_and_sex=re.findall(
            '(.+?)j\.(.+?)',
            raceslist)
        self.ages=[float(x[0])
                  for x
                  in age_and_sex]
        if len(self.ages)!=self.starters:
            print('Error only '+str(len(self.ages))+ ' ages.')
        self.sex=[x[1]
                  for x
                  in age_and_sex]
        if len(self.sex)!=self.starters:
            print('Error only '+str(len(self.sex))+ ' sex.')

        # ODDS
        oddsraw=re.findall(
                '
\s+(?:|)(.+?)\s+
',
                raceslist)
        oddsraw=[float(x.replace(',','.').replace('-','nan'))
                       for x
                       in oddsraw]
        self.odds=[float('nan')]*self.starters
        if len(oddsraw)==self.starters:
            self.odds=oddsraw
        elif len(oddsraw)==sum(self.runners):
            irunners=[i for i,runner in enumerate(self.runners) if runner]
            for j,i in enumerate(irunners):
                self.odds[i]=oddsraw[j]
        else:
            self.odds=oddsraw
            print('Error in '+ str(self.raceID) +' only '+str(len(oddsraw))+' / '+
                  str(self.starters) +' odds, but ' + str(sum(self.runners)) + ' runners.')

        #Low-Fidelity ("lofi") extract of horse history from "Formen" shown on racesheet.(formen-tables in the HiFi "Formen"-Subpages were empty from July 2016 on...)
        self.lofi_formentables=re.findall(
                            '(
\s+.+?\s
)|(
\s+.+?\s
)|(
.+?
)',
                            raceslist,re.S)

        lofi_formenrows=[re.findall('
(.*?)
',formtable[0],re.S)
                        for formtable in self.lofi_formentables]
        lofi_formencells=[[re.findall('<(?:th|td).+?>(.*?)	',row,re.S)
                        for row in horse]
                        for horse in lofi_formenrows]
        self.lofi_pastfinish=lofi_formencells

        self.lofi_places=[[float(row[1].replace('-','nan')) if self.isnumber(row[1]) else float('nan')
                            for row in horse]
                            for horse in lofi_formencells]
        if len(self.lofi_places)!=self.starters:
            print('Error only '+str(len(self.lofi_places))+ ' lofi_places.')

        self.lofi_nracesrun=[float(len(horse)) if len(np.array(horse)) > 0 else float(0)
                for horse in self.lofi_places]
        if len(self.lofi_nracesrun)!=self.starters:
            print('Error only '+str(len(self.lofi_nracesrun))+ ' lofi_nracesrun.')
        with warnings.catch_warnings(): #Just to suppress NaN warning in case only NaN data are available for a given race
             warnings.simplefilter("ignore", category=RuntimeWarning) #Just to suppress NaN warning in case only NaN data are available for a given race

             self.lofi_meanplace=[float(nanmean(horse)) if len(np.array(horse)) > 0 else float('nan')
                    for horse in self.lofi_places]
             if len(self.lofi_meanplace)!=self.starters:
                 print('Error only '+str(len(self.lofi_meanplace))+ ' lofi_meanplace.')

             self.lofi_pastodds=[[float(row[6].replace(',','.').replace('-','nan')) if self.isnumber(row[6]) else float('nan')
                    for row in horse]
                    for horse in lofi_formencells]
             if len(self.lofi_pastodds)!=self.starters:
                 print('Error only '+str(len(self.lofi_pastodds))+ ' lofi_pastodds.')

             self.lofi_meanodds=[float(nanmean(horse)) if len(np.array(horse)) > 0 else float('nan')
                    for horse in self.lofi_pastodds]
             if len(self.lofi_meanodds)!=self.starters:
                 print('Error only '+str(len(self.lofi_meanodds))+ ' lofi_meanodds.')
             lofi_mdl=linear_model.LinearRegression()
             self.lofi_trendodds=[lofi_mdl.fit(np.arange(0,
                                              len(np.array(horse)[~np.isnan(horse)]),
                                                1).reshape(-1,1),np.array(horse)[~np.isnan(horse)].reshape(-1, 1)).coef_[0][0]
                                        if len(np.array(horse)[~np.isnan(horse)])>1
                                        else float('nan')
                                    for horse
                                    in self.lofi_pastodds]
             if len(self.lofi_trendodds)!=self.starters:
                 print('Error only '+str(len(self.lofi_trendodds))+ ' lofi_trendodds.')

    def getformen(self,formen):
        formentables=[re.findall(
                        '(
\s+Form\s+
\s+
\s+
\s+
.*?
)',
                        form.text,re.S)
                      for form
                      in formen]
        #self.formentables=formentables
        formenrows=[re.findall('
(.*?)
',formtable[0],re.S) for formtable in formentables]
        self.pastfinish=[[re.findall(
                    '<(?:th|td).*?>(?:)?(.*?)(?:)?</(?:th|td)>',
                    row,re.S)
                         for row in rows]
                         for rows in formenrows]
        self.nracesrun=[len(horse)-1 for horse in self.pastfinish]  #counts previous races, subtracts headerline
        if len(self.nracesrun)!=self.starters:
            print('Error only '+str(len(self.nracesrun))+ ' nracesrun.')
        # Get places as float
        self.places=[[float(row[2])
                                if self.isnumber(row[2])
                                else float('nan')
                            for row in horse[1:]]
                        for horse in self.pastfinish]
        with warnings.catch_warnings(): #Just to suppress NaN warning in case only NaN data are available for a given race
             warnings.simplefilter("ignore", category=RuntimeWarning) #Just to suppress NaN warning in case only NaN data are available for a given race

             # Calculate mean if not empty
             self.meanplace=[nanmean(horse)
                                if len(np.array(horse)[~np.isnan(horse)]) > 0
                                else float('nan')
                            for horse in self.places]
             if len(self.meanplace)!=self.starters:
                 print('Error only '+str(len(self.meanplace))+ ' meanplace.')

             self.pastodds=[[float(row[6].replace(',','.').replace('-','nan'))
                                    if self.isnumber(row[2])
                                    else float('nan')
                                for row in horse[1:]]
                            for horse in self.pastfinish]
             self.meanodds=[nanmean(horse)
                           if len(np.array(horse)[~np.isnan(horse)]) > 0
                           else float('nan')
                           for horse in self.pastodds]
             if len(self.meanodds)!=self.starters:
                 print('Error only '+str(len(self.meanodds))+ ' meanodds.')

             mdl=linear_model.LinearRegression()
             self.trendodds=[
                            mdl.fit(
                                    np.arange(0,
                                              len(np.array(horse)[~np.isnan(horse)]),
                                              1).reshape(-1,1),np.array(horse)[~np.isnan(horse)].reshape(-1, 1)).coef_[0][0]
                                if len(np.array(horse)[~np.isnan(horse)])>1
                                else float('nan')
                            for horse
                            in self.pastodds]
             if len(self.trendodds)!=self.starters:
                 print('Error only '+str(len(self.trendodds))+ ' trendodds.')
        #Macht noch trouble:
        #self.days_since_last_race=[
        #    (self.rday-datetime.strptime(horse[1][0],
        #                                '%d.%m.%Y')).days
        #        if (horse[1][0])
        #        else float('nan')
        #    for horse
        #    in self.pastfinish]

    def dfexport(self,verbose=False):
        if len(self.odds)!=self.starters:
            self.odds= float('nan')
        if len(self.starterNo1)!=self.starters:
            self.starterNo1= float('nan')
        if len(self.starterNo2)!=self.starters:
            self.starterNo2= float('nan')

         # Add past race placements post-hoc (afraid to mess up import otherwise)
        self.pastpl1=[float(horse[0])
                            if len(np.array(horse)[~np.isnan(horse)]) > 0
                            else float('nan')
                            for horse in self.places]
        self.pastpl2=[float(horse[1])
                            if len(np.array(horse)[~np.isnan(horse)]) > 1
                            else float('nan')
                            for horse in self.places]
        self.pastpl3=[float(horse[2])
                            if len(np.array(horse)[~np.isnan(horse)]) > 2
                            else float('nan')
                            for horse in self.places]
        self.pastpl4=[float(horse[3])
                            if len(np.array(horse)[~np.isnan(horse)]) > 3
                            else float('nan')
                            for horse in self.places]
        self.pastpl5=[float(horse[4])
                            if len(np.array(horse)[~np.isnan(horse)]) > 4
                            else float('nan')
                            for horse in self.places]
        #Add a simple numeric version of starterNo (StarterNo 1 and 2 often contain starter numbers like 2A 3B etc...)
        self.starterNo=list(range(1, self.starters+1))

        # Same for lofi history
        if isinstance(self.lofi_places, (list, tuple)): # Check if lofi_places contains list, otherwise don't do the stuff and keep the default nans.
            self.lofi_pastpl1=[float(horse[0])
                                if len(np.array(horse)[~np.isnan(horse)]) > 0
                                else float('nan')
                                for horse in self.lofi_places]
            self.lofi_pastpl2=[float(horse[1])
                                if len(np.array(horse)[~np.isnan(horse)]) > 1
                                else float('nan')
                                for horse in self.lofi_places]
            self.lofi_pastpl3=[float(horse[2])
                                if len(np.array(horse)[~np.isnan(horse)]) > 2
                                else float('nan')
                                for horse in self.lofi_places]
            self.lofi_pastpl4=[float(horse[3])
                                if len(np.array(horse)[~np.isnan(horse)]) > 3
                                else float('nan')
                                for horse in self.lofi_places]
            self.lofi_pastpl5=[float(horse[4])
                                if len(np.array(horse)[~np.isnan(horse)]) > 4
                                else float('nan')
                                for horse in self.lofi_places]

        d = {
            'nonrunners' :  self.nonrunners,
            'n_nonrunners' : sum(self.nonrunners),
            'raceID'  :   self.raceID,
            'rtype'  :   self.rtype,
            'rday'  :   self.rday,
            'rtime'  :   self.rtime,
            'country'  :   self.country[0],
            'rname'  :   self.rname[0],
            'rnum'  :   self.rnum,
            'dist'  :   self.dist,
            'stake'  :   self.stakes[0][0],
            'stake_curr'  : self.stakes[0][1],
            'weights'  :   self.weights,
            'ages'  :   self.ages,
            'sex'  :   self.sex,
            'starters' : self.starters,
            'starterNo1' : self.starterNo1, #String from racesheet
            'starterNo2' : self.starterNo2, #String from racesheet
            'starterNo' : self.starterNo,   #Simply starters counted (int)
            'real_starters' : self.starters-sum(self.nonrunners),
            'meanodds' : self.meanodds,
            'trendodds' : self.trendodds,
            'horsenames'  :   self.horsenames, # HIER WEITER, STING-LISTEN LASSEN SICH NICHT EINFACH EINFÜGEN!!
            'jockeynames'  :   self.jockeynames,
            'jockeytrainers'  :   self.jockeytrainers,
            'heritages1'  :   self.heritages1,
            'heritages2'  :   self.heritages2,
            'owners'  :   self.owners,
            'odds'    : self.odds,
            'finish' : self.fillfinish,
            'nanfinish' : self.finishers,
            'pastpl1' : self.pastpl1,
            'pastpl2' : self.pastpl2,
            'pastpl3' : self.pastpl3,
            'pastpl4' : self.pastpl4,
            'pastpl5' : self.pastpl5,
            'lofi_meanodds' : self.lofi_meanodds,
            'lofi_trendodds' : self.lofi_trendodds,
            'lofi_nracesrun' : self.lofi_nracesrun,
            'lofi_pastpl1' : self.lofi_pastpl1,
            'lofi_pastpl2' : self.lofi_pastpl2,
            'lofi_pastpl3' : self.lofi_pastpl3,
            'lofi_pastpl4' : self.lofi_pastpl4,
            'lofi_pastpl5' : self.lofi_pastpl5
        }

        if verbose:
            print({key: (len(value) if isinstance(value, list) else 's:'+str(value)) for (key, value) in d.items()})

        return pd.DataFrame(d)

    def isnumber(self,s): # Function checks if string can be converted to float and is >0
        try:
            f=float(s.replace(',','.'))
            if f > 0:
                return True
            else:
                return False
        except ValueError:
            return False

class racePredictorSet:
    varlist=[]
    PredImputer=[]
    Xscaler=[]
    Predictors=[]

    def __init__(self, varlist,PredImputer,Xscaler,Predictors):
    #self.textraw = textraw
        self.description = "Set of"
        self.author = "MZ"

        self.varlist=varlist
        self.PredImputer=PredImputer
        self.Xscaler=Xscaler
        self.Predictors=Predictors

def downloadSingleRace(raceID):
    baseurl='https://wettstar.de'
    downurl=baseurl+'/race?id='+str(raceID)
    #Print today's date
    print("Start: ImportnPredic Single Race: "
           +downurl)

    #Define Header for Login (required in order to get Formen)
    oriheader={
    'Host':
        'wettstar.de', #non-crucial
    'User-Agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:45.0)\
        Gecko/20100101 Firefox/45.0', #non-crucial
    'Accept':
        'application/json, text/javascript, */*; q=0.01', #non-crucial
    'Accept-Language':
        'de,en-US;q=0.7,en;q=0.3', #non-crucial
    'Accept-Encoding':
        'gzip, deflate, br', #non-crucial
    'Content-Type':
        'application/x-www-form-urlencoded; charset=UTF-8',#, #crucial
    'Connection':
        'keep-alive' #non-crucial
    }
    payload ={
        'username':'Zunhias',
        'password':'Paranoia23#'
    }
    #Download race and associated formen
    racesheets=[]
    formen=[]
    formPTRN= "window.open\(\'(.+?)', \'Formguide\'";
    #For each race location...
    try:
        #Open new session...
        with requests.Session() as s:
            p = s.post('https://wettstar.de/auth/validatepostajax',
                       headers = oriheader,
                       data=payload) #, data=json.dumps(payload2)
        # Check if login was successful
            if not re.search('"login":true',p.text):
                #print(p.text) #Login-response received
                sys.exit("LOGIN FAILED")
        #For each single race at race location...
            #Get current racesheet
            racesheets.append(s.get(downurl,
                                    headers = oriheader,
                                    cookies=s.cookies))
            #Get horseformen urls for that race
            horseform_urls=(re.findall(formPTRN,
                                       racesheets[-1].text))
            curr_formen=[]
            #Get horseformen-sheets for that race
            for (k, horseform_url) in enumerate(horseform_urls):
                curr_formen.append(s.get(baseurl+horseform_url,
                                         headers = oriheader,
                                         cookies=s.cookies))
            formen.append(curr_formen)
            print("Time: "
                  +str(time.monotonic())
                  +" "+downurl
                  +"   n Formen: "+str(len(curr_formen)))
    #Exception of Request
    except requests.exceptions.RequestException as e:
        print(e)
    ### END OF DOWNLOAD START OF EXTRACT
    print("End of Download start of extract")
    if len(racesheets)==len(formen):
        racelist=[Race(r.text,f) for r,f in zip(racesheets,formen)]
    else:
        print('RACESHEETS AND FORMEN HAVE DIFFERENT LENGTH')
    return racelist,racesheets,formen

def loadSingleRace(raceID,racetime='past'):
    if racetime is 'past':
        racelistFolder='/Users/matthiaszunhammer/Dropbox/PY/HorseRacing/pastraces/'
    elif racetime is 'future':
        racelistFolder='/Users/matthiaszunhammer/Dropbox/PY/HorseRacing/futureraces/'
    else:
        print('Error: racetime not set as ''past'' or ''future''. Please select whether you want past or future racepickle loaded')

    with open(racelistFolder+raceID+'.pickle','rb') as rf:
            r_raw=pickle.load(rf)
    return r_raw

def loadRacePickleAsDF(racetime='past',racetype = 'Flachrennen'):
    # Function that
    # a) loads the pickle containing tomorrows race objects and
    # b) extracts a table from the race objects
    # c) Returns either Flachrennen (default) or Trabrennen
    if racetime is 'past':
        racelistFolder='/Users/matthiaszunhammer/Dropbox/PY/HorseRacing/pastraces/'
    elif racetime is 'future':
        racelistFolder='/Users/matthiaszunhammer/Dropbox/PY/HorseRacing/futureraces/'
    else:
        print('Error: racetime not set as ''past'' or ''future''. Please select whether you want past or future racepickle loaded')

    allfiles = []
    for (dirpath, dirnames, filenames) in walk(racelistFolder):
        allfiles.append(filenames)
        break
    allfiles = allfiles[0]

    r_raw=[]
    for file in allfiles:
        with open(racelistFolder+file,'rb') as rf:
            r_raw.append(pickle.load(rf))

    # BRING RACE CLASSES IN SHAPE / COUNT
    # Flatten list
    print("Aufgezeichnete Renntage: " + str(len(r_raw)))
    r=flatten(r_raw)
    print("Aufgezeichnete Rennen: " + str(len(r)))
    # Extract basic info ahead
    rNos=[races.raceID for races in r]
    # Identify duplicate races
    dupli_r=[k for k,v in collections.Counter(rNos).items() if v>1]
    print("Duplicate Entries: "+str(len(dupli_r)))
    # Count number of Flach- and Trabrennen

    #HERE ARE THE TRABRENNEN

    #EXPORT TO PANDAS DATAFRAME
    drf=[]
    if racetype is 'Flachrennen':
            #HERE ARE THE FLACHRENNEN
        rf=[races for races in r if races.rtype=="Flach"]
        print("Aufgezeichnete Flachrennen: " + str(len(rf)))
        # drf=[races.dfexport() for races in rf] # Faster but failing if race instances are erronenous
        for races in rf:
            #print(races.raceID)
            try:
                drf.append(races.dfexport())
            except:
                print('Error in Df Export of '+ str(races.raceID))
    elif racetype is 'Trabrennen':
            #HERE ARE THE FLACHRENNEN
        rf=[races for races in r if races.rtype=="Trab"]
        print("Aufgezeichnete Trabrennen: " + str(len(rf)))
        # drf=[races.dfexport() for races in rf]
        for races in rf:
            #print(races.raceID)
            try:
                drf.append(races.dfexport())
            except:
                print('Error in Df Export of '+ str(races.raceID))
    else:
        print("Input variable racetype specified incorrectly. Either enter ""Flachrennen"" (default) or ""Trabrennen""")
        drf=[]

    drf=pd.concat(drf,ignore_index=True)
    df=drf.drop_duplicates(keep='last')
    return df

def preproRaceDF(df,racetime='past'):
    df = df.drop(df[df.nonrunners==1].index)
    df = df.drop(df[df.raceID==1].index) #Drop races with invalid raceID)

    # Derive categorical variables to creaty dummy variables
    # Clean sex ;)
    df.sex=df.sex.str.strip()
    df.loc[(df.sex!='Hengst')&(df.sex!='Stute')&(df.sex!='Wallach'),'sex']=np.nan#'NaN' #First clean all odd sexes
    df.sex=df.sex.astype('category',categories=["Hengst","Stute","Wallach"])
    dd=pd.get_dummies(df[['sex']], prefix='dummy', prefix_sep='_', dummy_na=False, sparse=False, drop_first=False)
    # Country
    df.country=df.country.astype('category')

    #add dummy variables to df
    dd=pd.get_dummies(df[['sex','country']], prefix='dummy', prefix_sep='_', dummy_na=False, sparse=False, drop_first=False)
    df = pd.concat([df,dd], axis=1)

    # Calculate uninformed and informed probabilities based on starters and odds
    #Wettstar provides decimal (European) odds, i.e. potential winnings (net returns) + the stake (e.g. 6/5 or 1.2 plus 1 = 2.2). " See: https://en.wikipedia.org/wiki/Odds#Odds_against "
    df=df.assign(p_starters = 1/df.real_starters)
    df=df.assign(p_real_starters = 1/df.real_starters)
    df=df.assign(p_odds = 1/(df.odds))

    # Replace "HiFi" Horse-Forms with "LoFi" Horse-Forms if NaN
    df.loc[np.isnan(df['meanodds']),'meanodds']= df.loc[np.isnan(df['meanodds']),'lofi_meanodds']
    df.loc[np.isnan(df['trendodds']),'trendodds']= df.loc[np.isnan(df['trendodds']),'lofi_trendodds']
    df.loc[np.isnan(df['pastpl1']),'pastpl1']= df.loc[np.isnan(df['pastpl1']),'lofi_pastpl1']
    df.loc[np.isnan(df['pastpl2']),'pastpl2']= df.loc[np.isnan(df['pastpl2']),'lofi_pastpl2']
    df.loc[np.isnan(df['pastpl3']),'pastpl3']= df.loc[np.isnan(df['pastpl3']),'lofi_pastpl3']
    df.loc[np.isnan(df['pastpl4']),'pastpl4']= df.loc[np.isnan(df['pastpl4']),'lofi_pastpl4']
    df.loc[np.isnan(df['pastpl5']),'pastpl5']= df.loc[np.isnan(df['pastpl5']),'lofi_pastpl5']

    #Clean absurd weights
    df.loc[(df.weights<10),'weights']=np.nan#'NaN' #First clean all odd sexes
    df.loc[(df.weights>150),'weights']=np.nan#'NaN' #First clean all odd sexes

    #Clean absurd ages
    df.loc[(df.ages>25),'ages']=np.nan#'NaN' #First clean all odd sexes
    df.loc[(df.ages<0),'ages']=np.nan#'NaN' #First clean all odd sexes

    #Clean absurd past-places
    df.loc[(df.pastpl1>35),'pastpl1']=35#
    df.loc[(df.pastpl2>35),'pastpl2']=35#
    df.loc[(df.pastpl3>35),'pastpl3']=35#
    df.loc[(df.pastpl4>35),'pastpl4']=35#
    df.loc[(df.pastpl5>35),'pastpl5']=35#

    #Clean absurd starternumbers
    df.loc[df.real_starters<3,'real_starters']=np.nan

    #Normalize Age
    #df.logages=np.log(df.ages+1)
    df=df.assign(logages = np.log(df.ages+1))

    #Normalize meanodds
    df=df.assign(logmeanodds = np.log(df.meanodds+1))

    #Normalize Trendodds
    df=df.assign(sqrttrendodds = np.nan)
    #df=df.assign(sqrttrendodds = np.log(1/np.sqrt(df.trendodds-100)))

    with warnings.catch_warnings(): #Just to suppress NaN warning in case only NaN data are available for a given race
        warnings.simplefilter("ignore", category=RuntimeWarning) #Just to suppress NaN warning in case only NaN data are available for a given race
        byID=df.groupby('raceID')
        df=df.assign(WIages=byID['ages'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WIlogages=byID['logages'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WImeanodds=byID['meanodds'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WItrendodds=byID['trendodds'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WIweights=byID['weights'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WImeanodds=byID['meanodds'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(WIpastpl1=byID['pastpl1'].transform(lambda x: x-np.nanmean(x)))
        df=df.assign(bookieMargin=byID['p_odds'].transform(lambda x: (np.sum(x)-1) if np.sum(x)>1 else 'nan'))
        df=df.assign(knwn_finish=byID['finish'].transform(lambda x: np.sum(~np.isnan(x))))
        df=df.assign(mssng_finish=byID['finish'].transform(lambda x: np.sum(np.isnan(x))))

    # DROP RACES WHERE THERE IS A NEGATIVE BOOKIE-MARGIN (indicative of erronenous odds)
    df = df.drop(df[df.bookieMargin<0].index) #Drop races with invalid raceID)

    # DEFINE TARGET VARIABLE!!!
    # For Classification
    df=df.assign(winners = 0) # Usually using one unit per bet
    df.loc[df.finish==1,'winners']= 1
    # To predict winners
    df=df.assign(winnings = -1) # Usually using one unit per bet
    df.loc[df.finish==1,'winnings']= df.odds[df.finish==1]-1 # ATTENTIONE: Winning horses get you A NET WIN OF ODDS MINUS YOUR BETSUM (! YOU HAVE TO SPEND MONEY FIRST, BEFORE GETTING THE RETURN)

    return df

def elastic_predict(df,predset):
    Xraw=df[predset.varlist].as_matrix()
    nanrowsX=np.isnan(Xraw).any(axis=1)
    print('For Xpure, ', str(sum(nanrowsX)), 'rows()', str(sum(nanrowsX)/len(nanrowsX)*100),'%) out of', str(len(nanrowsX)), 'were dropped.')
    df=df.drop(df[nanrowsX].index)
    X=df[predset.varlist].as_matrix()
    mdl=linear_model.ElasticNetCV(l1_ratio=[1], #[.1, .5, .9, .99, .999, 1]
                          fit_intercept=True,
                          normalize=False)
    mdl.intercept_=predset.Predictors[0]
    mdl.coef_=predset.Predictors[1]
    predicted_values=mdl.predict(X)
    df=df.assign(pred_y=predicted_values)
    return df

def elastic_predict_odds(df,predset):
    # Odds have to be represented by first coefficient
    X=df[predset.varlist[1:]].as_matrix() #exclude odds from varlist
    mdl=linear_model.ElasticNetCV()
    mdl.intercept_=predset.Predictors[0]
    mdl.coef_=predset.Predictors[1][1:] #exclude odds from prediction
    predicted_odds=(mdl.predict(X)/(predset.Predictors[1][0]))*1 #scale by predicted odds
    return predicted_odds

def rescale_coefs(intercept,coefficient_list,scaler): # Used to rescale estimated coefs AND intercept
    intercept = intercept - np.dot(coefficient_list, scaler.mean_/scaler.scale_)
    coefficient_list=np.true_divide(coefficient_list,  scaler.scale_)
    return intercept, coefficient_list

def betting_suggestion(df):
    #For each race get horse with best prediction
    with warnings.catch_warnings(): #Just to suppress NaN warning in case only NaN data are available for a given race
        warnings.simplefilter("ignore", category=RuntimeWarning) #Just to suppress NaN warning in case only NaN data are available for a given race
        byID=df.groupby('raceID')
        df=df.assign(bestpred=byID['pred_y'].transform(lambda x: x==max(x)))
    #For each race decide to bet if best prediction > 0 (indicating gain)
    df=df.assign(bets=False)
    df.loc[(df.pred_y>0)&(df.bestpred==1),'bets']=True
    #For each race calculate the suggested betsum in money units
    df=df.assign(betsum=df.pred_y[df.bets]*10)
    df.loc[np.isnan(df.betsum),'betsum',]=0
    df.loc[(df.betsum>0)&(df.betsum<1),'betsum']=1 #account for the fact that minimum betsum is 1
    return df

def cols_from_html_tbl(tbl):  #Helper function for extracting columns from html-table tbl (table must be passed as Results-Object from BeautifulSoup)
    rows = tbl.find_all('tr')
    for row in rows:
        cols = row.find_all('td')
        for i,cell in enumerate(cols):
            if not'col_list' in locals():
                col_list=[[] for x in range(len(cols))]
            col_list[i].append(cell.text)
    return col_list

def isnumber(s): # Function checks if string can be converted to float and is >0
    try:
        f=float(s.replace(',','.'))
        if f > 0:
            return True
        else:
            return False
    except ValueError:
        return False

def extract(requests_instance,verbose=False):
    html = BeautifulSoup(requests_instance[0].content,'html5lib')

    #Basic html segments
    top_container=html.find('div',{'id':'racecardTopContainer'})
    race_info_box=top_container.find('div',{'class':'racesList'})
    racecard=html.find('div',{'class':'racecardList'}) #single horse containers

    #Race-specific data
    raceID=int(top_container["data-raceid"]) #or get directly from import script?

    starter=race_info_box.find('li',{'class':'starter'}).text
    starter=int(starter.split()[0])

    dateraw=top_container.h1.text.split()[-1]
    rday=datetime.strptime(dateraw,'%d.%m.%Y')

    timeraw=race_info_box.find('li',{'class':'time'}).text
    rtime=datetime.strptime(timeraw,'%H:%M')

    country_raw=top_container.h2.a['href']
    country=re.findall('/races\?country=(...)&date=',country_raw)
    country=country[0]

    rname=top_container.h2.text.strip()

    dist=race_info_box.find('li',{'class':'distance'}).text
    dist=float(dist.split()[0])

    stakes=race_info_box.find('li',{'class':'stakes'}).text
    stakes=float(stakes.split()[0])

    ground=race_info_box.find('li',{'class':'raceGround'}).text
    ground=str(ground)

    race_type_allowance=race_info_box.find('div',{'class':'raceTypeAllowance'}).text
    race_type_allowance=str(race_type_allowance)   # Verbose version of race_type written in box

    race_type_symbol=top_container.i["class"]     # Short version of race_type from symbol (may miss steeplechase with hurdles)
    if race_type_symbol[0] is 'T':
        rtype='Flach' # Oddly, Flachrennen get a 'T' by germantote
    elif race_type_symbol[0] is 'H':
        rtype='Trab' # Oddly, Trabrennen get a 'H' by germantote
    else:
        rtype=2 # "Other"

    rnum=int(race_info_box('div',{'class':'counter'})[0].text.strip())

    stakes_raw=race_info_box.find('li',{'class':'stakes'}).text
    stakes=float(stakes_raw.split()[0])
    stakes_currency=stakes_raw.split()[1]

    #Extract horse-specific data from racesheet
    horse_clearfixes=racecard.find_all('li',{'class','clearfix'})
    nonrunners=[True if 'nonrunner' in i['class'] else False for i in horse_clearfixes]

    starterNo1=html.find_all('span',{'class':'count1'})
    starterNo1=[i.text for i in starterNo1]

    starterNo2=html.find_all('span',{'class':'count2'})
    starterNo2=[i.text for i in starterNo2]

    horsenames=html.find_all('span',{'class':'runnername'})
    horsenames=[i.text.strip() for i in horsenames]

    jockeynames=html.find_all('span',{'class':'jockeyname'})
    jockeynames=[i.text.strip() for i in jockeynames]

    jockeytrainers=html.find_all('div',{'class':'jockeytrainer'})
    jockeytrainers=[i.text.strip() for i in jockeytrainers]

    statInfo=html.find_all('div',{'class':'statInfo'})
    heritage=[]
    owners=[]
    for i in statInfo:
        heritage.append(i.span.nextSibling)
        owners.append(i.span.nextSibling.nextSibling.nextSibling)
    heritage=[re.split('–',i) for i in heritage]
    heritages1=[i[0].strip() for i in heritage]
    heritages2=[i[1].strip() for i in heritage]

    weights=html.find_all('span',{'class':'weight'})
    weights=[i.text.split()[0].strip() for i in weights]
    weights=[float(i.replace(',','.')) for i in weights]

    age_and_sex=html.find_all('span',{'class':'horseage'})
    age_and_sex=[x.text.split('j. ') for x in age_and_sex]
    age=[float(i[0]) for i in age_and_sex]
    sex=[i[1] for i in age_and_sex]

    odds=html.find_all('span',{'class':'odds'})
    odds=[float(i.text.replace(',','.').replace('-','NaN')) for i in odds]

    #Low-Fidelity ("lofi") horse history from "Formen" shown on racesheet.(formen-tables in the HiFi "Formen"-Subpages were empty from July 2016 on...)

    past_dates=[]
    lofi_pastfinish=[]
    past_racedates=[]
    past_race_courses=[]
    past_dist=[]
    past_stakes=[]
    past_jockey=[]
    past_odds=[]

    for horse in horse_clearfixes:
        currtable=(cols_from_html_tbl(horse.table))

        raw_past_date=[datetime.strptime(i,'%d.%m.%y') for i in currtable[0]]
        past_dates.append(raw_past_date)

        raw_pastfinish=[float(i.strip('.')) if isnumber(i.strip('.')) else float('nan') for i in currtable[1]]
        lofi_pastfinish.append(raw_pastfinish)
        lofi_places=lofi_pastfinish #for backwards-compatibility

        past_race_courses.append(currtable[2])

        raw_pastdist=[float(i.strip(' m')) if isnumber(i.strip(' m')) else float('nan') for i in currtable[3]]
        past_dist.append(raw_pastdist)

        raw_paststakes=[float(i) if isnumber(i) else 'nan' for i in currtable[4]]
        past_stakes.append(raw_paststakes)

        past_jockey.append(currtable[5])

        raw_pastodds=[float(i.replace(',','.')) if isnumber(i) else 'nan' for i in currtable[6]]
        past_odds.append(raw_pastodds)

        lofi_nracesrun=len(currtable[0])

        lofi_meanplace=[float(nanmean(horse))
                             if len(horse) > 0 else float('nan')
                             for horse in lofi_places]

#
#             if len(self.lofi_meanplace)!=self.starters:
#                 print('Error only '+str(len(self.lofi_meanplace))+ ' lofi_meanplace.')
#
#             self.lofi_pastodds=[[float(row[6].replace(',','.').replace('-','nan')) if self.isnumber(row[6]) else float('nan')
#                    for row in horse]
#                    for horse in lofi_formencells]
#             if len(self.lofi_pastodds)!=self.starters:
#                 print('Error only '+str(len(self.lofi_pastodds))+ ' lofi_pastodds.')
#
#             self.lofi_meanodds=[float(nanmean(horse)) if len(np.array(horse)) > 0 else float('nan')
#                    for horse in self.lofi_pastodds]
#             if len(self.lofi_meanodds)!=self.starters:
#                 print('Error only '+str(len(self.lofi_meanodds))+ ' lofi_meanodds.')
#             lofi_mdl=linear_model.LinearRegression()
#             self.lofi_trendodds=[lofi_mdl.fit(np.arange(0,
#                                              len(np.array(horse)[~np.isnan(horse)]),
#                                                1).reshape(-1,1),np.array(horse)[~np.isnan(horse)].reshape(-1, 1)).coef_[0][0]
#                                        if len(np.array(horse)[~np.isnan(horse)])>1
#                                        else float('nan')
#                                    for horse
#                                    in self.lofi_pastodds]
#             if len(self.lofi_trendodds)!=self.starters:
#                 print('Error only '+str(len(self.lofi_trendodds))+ ' lofi_trendodds.')
#
#    def getformen(self,formen):
#        formentables=[re.findall(
#                        '(
\s+Form\s+
\s+
\s+
\s+
.*?
)',
#                        form.text,re.S)
#                      for form
#                      in formen]
#        #self.formentables=formentables
#        formenrows=[re.findall('
(.*?)
',formtable[0],re.S) for formtable in formentables]
#        self.pastfinish=[[re.findall(
#                    '<(?:th|td).*?>(?:)?(.*?)(?:)?</(?:th|td)>',
#                    row,re.S)
#                         for row in rows]
#                         for rows in formenrows]
#        self.nracesrun=[len(horse)-1 for horse in self.pastfinish]  #counts previous races, subtracts headerline
#        if len(self.nracesrun)!=self.starters:
#            print('Error only '+str(len(self.nracesrun))+ ' nracesrun.')
#        # Get places as float
#        self.places=[[float(row[2])
#                                if self.isnumber(row[2])
#                                else float('nan')
#                            for row in horse[1:]]
#                        for horse in self.pastfinish]
#        with warnings.catch_warnings(): #Just to suppress NaN warning in case only NaN data are available for a given race
#             warnings.simplefilter("ignore", category=RuntimeWarning) #Just to suppress NaN warning in case only NaN data are available for a given race
#
#             # Calculate mean if not empty
#             self.meanplace=[nanmean(horse)
#                                if len(np.array(horse)[~np.isnan(horse)]) > 0
#                                else float('nan')
#                            for horse in self.places]
#             if len(self.meanplace)!=self.starters:
#                 print('Error only '+str(len(self.meanplace))+ ' meanplace.')
#
#             self.pastodds=[[float(row[6].replace(',','.').replace('-','nan'))
#                                    if self.isnumber(row[2])
#                                    else float('nan')
#                                for row in horse[1:]]
#                            for horse in self.pastfinish]
#             self.meanodds=[nanmean(horse)
#                           if len(np.array(horse)[~np.isnan(horse)]) > 0
#                           else float('nan')
#                           for horse in self.pastodds]
#             if len(self.meanodds)!=self.starters:
#                 print('Error only '+str(len(self.meanodds))+ ' meanodds.')
#
#             mdl=linear_model.LinearRegression()
#             self.trendodds=[
#                            mdl.fit(
#                                    np.arange(0,
#                                              len(np.array(horse)[~np.isnan(horse)]),
#                                              1).reshape(-1,1),np.array(horse)[~np.isnan(horse)].reshape(-1, 1)).coef_[0][0]
#                                if len(np.array(horse)[~np.isnan(horse)])>1
#                                else float('nan')
#                            for horse
#                            in self.pastodds]
#             if len(self.trendodds)!=self.starters:
#                 print('Error only '+str(len(self.trendodds))+ ' trendodds.')
#        #Macht noch trouble:
#        #self.days_since_last_race=[
#        #    (self.rday-datetime.strptime(horse[1][0],
#        #                                '%d.%m.%Y')).days
#        #        if (horse[1][0])
#        #        else float('nan')
#        #    for horse
#        #    in self.pastfinish]

#        #Extract finish-data
#        finish_raw=re.findall(
#            '
.*?
',
#            textraw,re.S)#get finishtable
#        #self.finish_raw=finish_raw
#        if finish_raw:
#            trows=re.findall(
#                '
(.*?)
',
#                finish_raw[0],
#                re.S) #gets all table rows
#            self.finishall=[re.findall('<(?:th|td) .*?>(.*?)</(?:th|td)>',
#                                row,re.S)
#                            for row
#                            in trows] #gets all table headers or table cells for each row
#            self.finisherNames=([x[2]
#                             for x
#                             in self.finishall[1:]]) # gets finisher's starter numbers from second row on in second column
#            self.finishersRaw=([x[1]
#                             for x
#                             in self.finishall[1:]]) # gets finisher's starter numbers from second row on in second column
#            self.placementsRaw=([int(x[0])
#                             for x
#                             in self.finishall[1:]])
#
#            self.finishers=[float('nan')] * self.starters
#            #VERSION 1:
#            #fill all fields with mean finish of all starters not placed
#            #PROBLEM: will systematically affect relationship between starters and finish >> starter only useful as a predictor of winners
#            self.fillfinish=[np.mean([self.starters-sum(self.nonrunners),
#                              1+len(self.finishersRaw)])]* self.starters
#            #VERSION 2:
#            #fill all fields with nan
#            #PROBLEM: more horses will be dropped
#            #self.fillfinish=[float('nan')]* self.starters
#            #VERSION 3:
#            #fill all fields with random numbers
#            #PROBLEM: random variance added
#            #IMPLEMENTED IN ANALYSIS-PRE-PROCESSING
#            if self.finisherNames: # in case finisher list has names
#                for i,finny in enumerate(self.finisherNames):
#                    finindex=self.horsenames.index(finny)#get index of finisher
#                    self.finishers[finindex]=self.placementsRaw[i]
#                    self.fillfinish[finindex]=self.placementsRaw[i]
#            elif self.finishersRaw:
#                for i,finny in enumerate(self.finishersRaw):
#                    finindex=self.starterNo1.index(finny)#get index of finisher
#                    self.finishers[finindex]=self.placementsRaw[i]
#                    self.fillfinish[finindex]=self.placementsRaw[i]
#        else:
#            self.finishall=[]
#            self.finishersRaw=[]
#            self.finishers=float('nan')
#            self.fillfinish=float('nan')


text:
import networkx as nx
import editdistance

from collections import OrderedDict
from clint.textui.progress import bar
from scipy.spatial import distance
from textplot.helpers import build_graph
from textplot.utils import sort_dict
from tdiff.text import Text
from tdiff.utils import dijkstra

class Diff:

    @classmethod
    def from_paths(cls, path1, path2):

        """
        Build a matrix from a pair of file paths.

        Args:
            path1 (str)
            path2 (str)

        Returns: Matrix
        """

        return cls(Text.from_file(path1), Text.from_file(path2))

    def __init__(self, text1, text2):

        """
        Wrap a pair of text instances.

        Args:
            text1 (Text)
            text2 (Text)
        """

        self.text1 = text1
        self.text2 = text2

    def kde_best_match(self, n=500, show_matches=False, **kwargs):

        """
        For each term in text 1, find the term in text 2 with the most similar
        pattern of distribution.

        Args:
            n (int): Consider N most-frequent words.
            show_matches (bool): Show identity (A -> A) matches.

        Returns:
            list: Tuples of (t1 term, t2 term, weight).
        """

        mft1 = self.text1.most_frequent_terms(n)
        mft2 = self.text2.most_frequent_terms(n)

        # For each term in text 1.
        links = []
        for t1 in mft1:

            # Score against each term in text 2.
            scores = []
            for t2 in mft2:

                t1_kde = self.text1.kde(t1, **kwargs)
                t2_kde = self.text2.kde(t2, **kwargs)

                score = 1-distance.braycurtis(t1_kde, t2_kde)
                scores.append((t2, score))

            # Get the nearest neighbor.
            scores = sorted(scores, key=lambda x: x[1], reverse=True)
            t2 = scores[0][0]

            if show_matches or t1 != t2:
                links.append((
                    self.text1.unstem(t1),
                    self.text2.unstem(t2),
                    scores[0][1]
                ))

        # Sort strongest -> weakest.
        links = sorted(links, key=lambda x: x[2], reverse=True)

        return links

    def topn_edit_distances(self, term_depth=500, n=10, **kwargs):

        """
        For each term in text 1, find the term in text 2 with the most similar
        set of nearest-neighbors, in terms of path distance.

        Args:
            n (int): The number of neighbors to consider.

        Returns:
            list: Tuples of (t1 term, t2 term, distance)
        """

        # Build graphs.
        g1 = self.text1.build_graph(term_depth=term_depth, **kwargs)
        g2 = self.text2.build_graph(term_depth=term_depth, **kwargs)

        # Get lengths between all pairs.
        dj1 = dijkstra(g1.graph, cutoff=n)
        dj2 = dijkstra(g2.graph, cutoff=n)

        # For each term in text 1.
        links = []
        for s1, t1 in bar(dj1.items()):

            # Score against each term in text 2.
            scores = OrderedDict()
            for s2, t2 in dj2.items():

                nn1 = list(t1.keys())[:n]
                nn2 = list(t2.keys())[:n]
                scores[s2] = editdistance.eval(nn1, nn2)

            # Get the closest neighbor.
            scores = sort_dict(scores, desc=False)
            winner = list(scores.items())[0]

            # Register the match.
            links.append((s1, winner[0], winner[1]))

        # Sort strongest -> weakest.
        links = sorted(links, key=lambda x: x[2])

        return links

    def topn_digraph(self, term_depth=500, skim_depth=5, **kwargs):

        """
        For each term in text 1, find the N most similar terms in text 2 and
        register the connections as edges in a digraph. Then, do the same thing
        in the opposite direction, linking text 2 "back" onto text 1.

        Args:
            n (int): The number of neighbors to consider.

        Returns: nx.DiGraph
        """

        pass


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-Type" content="text/html" charset="UTF-8">
<title>IScontoVenditaStrategy</title>
<meta name="date" content="2015-02-13">
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">

JavaScript is disabled on your browser.

Overview

Package

Class

Use

Tree

Deprecated

Index

Help

Prev Class

Next Class

Frames

No Frames

All Classes

Summary:

Nested |

Field |

Constr |

Method

Detail:

Field |

Constr |

Method

gameshop.advance.interfaces

Interface IScontoVenditaStrategy

All Known Implementing Classes:

ScontoPercentualeVenditaStrategy, ScontoTotaleVenditaStrategy, ScontoVenditaMigliorePerClienteStrategyComposite, ScontoVenditaStrategyComposite

public interface IScontoVenditaStrategy

Interfaccia della strategia di sconto sulle vendite

Method Summary

Methods

Modifier and Type
Method and Description

Money
getTotal(ITransazione vendita)

boolean
isValid(DateTime period)

Method Detail

getTotal

Money getTotal(ITransazione vendita)
               throws java.rmi.RemoteException

Throws:

java.rmi.RemoteException

isValid

boolean isValid(DateTime period)

Overview

Package

Class

Use

Tree

Deprecated

Index

Help

Prev Class

Next Class

Frames

No Frames

All Classes

Summary:

Nested |

Field |

Constr |

Method

Detail:

Field |

Constr |

Method


text:
if node['bcpc']['enabled']['monitoring'] then

    include_recipe "bcpc::default"

    cookbook_file "/tmp/zabbix-agent.tar.gz" do
        source "bins/zabbix-agent.tar.gz"
        owner "root"
        mode 00444
    end

    bash "install-zabbix-agent" do
        code <<-EOH
            tar zxf /tmp/zabbix-agent.tar.gz -C /usr/local/
        EOH
        not_if "test -f /usr/local/sbin/zabbix_agentd"
    end

    user node['bcpc']['zabbix']['user'] do
        shell "/bin/false"
        home "/var/log"
        gid node['bcpc']['zabbix']['group']
        system true
    end

    directory "/var/log/zabbix" do
        user node['bcpc']['zabbix']['user']
        group node['bcpc']['zabbix']['group']
        mode 00755
    end

    template "/etc/init/zabbix-agent.conf" do
        source "upstart-zabbix-agent.conf.erb"
        owner "root"
        group "root"
        mode 00644
        notifies :restart, "service[zabbix-agent]", :delayed
    end

    template "/usr/local/etc/zabbix_agent.conf" do
        source "zabbix_agent.conf.erb"
        owner node['bcpc']['zabbix']['user']
        group "root"
        mode 00600
        notifies :restart, "service[zabbix-agent]", :delayed
    end

    template "/usr/local/etc/zabbix_agentd.conf" do
        source "zabbix_agentd.conf.erb"
        owner node['bcpc']['zabbix']['user']
        group "root"
        mode 00600
        notifies :restart, "service[zabbix-agent]", :delayed
    end

    service "zabbix-agent" do
        provider Chef::Provider::Service::Upstart
        action [:enable, :start]
    end

    cookbook_file "/usr/local/bin/zabbix_discover_buckets" do
        source "zabbix_discover_buckets"
        owner "root"
        mode "00755"
    end

end


text:
package org.apache.sling.scripting.sightly.impl.plugin;

import java.util.HashMap;
import java.util.Map;

import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Properties;
import org.apache.felix.scr.annotations.Property;
import org.apache.felix.scr.annotations.Service;
import org.apache.sling.scripting.sightly.impl.compiler.expression.Expression;
import org.apache.sling.scripting.sightly.impl.compiler.expression.ExpressionNode;
import org.apache.sling.scripting.sightly.impl.compiler.expression.node.MapLiteral;
import org.apache.sling.scripting.sightly.impl.compiler.expression.node.RuntimeCall;
import org.apache.sling.scripting.sightly.impl.compiler.ris.command.OutVariable;
import org.apache.sling.scripting.sightly.impl.compiler.ris.command.Patterns;
import org.apache.sling.scripting.sightly.impl.compiler.ris.command.VariableBinding;
import org.apache.sling.scripting.sightly.impl.compiler.common.DefaultPluginInvoke;
import org.apache.sling.scripting.sightly.impl.compiler.frontend.CompilerContext;
import org.apache.sling.scripting.sightly.impl.compiler.util.stream.PushStream;

/**
 * The resource plugin
 */
@Component
@Service(Plugin.class)
@Properties({
        @Property(name = "service.description", value = "Sightly Resource Block Plugin"),
        @Property(name = Plugin.SCR_PROP_NAME_BLOCK_NAME, value = "resource"),
        @Property(name = Plugin.SCR_PROP_NAME_PRIORITY, intValue = PluginComponent.DEFAULT_PRIORITY)
})
public class ResourcePlugin extends PluginComponent {

    public static final String FUNCTION = "includeResource";

    @Override
    public PluginInvoke invoke(final Expression expression, final PluginCallInfo callInfo, final CompilerContext compilerContext) {

        return new DefaultPluginInvoke() {

            private Map<String, ExpressionNode> expressionOptions = new HashMap<String, ExpressionNode>(expression.getOptions());

            @Override
            public void beforeChildren(PushStream stream) {
                String resourceVar = compilerContext.generateVariable("resourceContent");
                stream.emit(new VariableBinding.Start(resourceVar,
                        new RuntimeCall(FUNCTION,
                                expression.getRoot(), new MapLiteral(expressionOptions))));
                stream.emit(new OutVariable(resourceVar));
                stream.emit(VariableBinding.END);
                Patterns.beginStreamIgnore(stream);
            }

            @Override
            public void afterChildren(PushStream stream) {
                Patterns.endStreamIgnore(stream);
            }

        };
    }
}


text:
package org.aikodi.chameleon.oo.method.exception;

import org.aikodi.chameleon.core.element.ElementImpl;
import org.aikodi.chameleon.core.lookup.LookupException;
import org.aikodi.chameleon.oo.expression.MethodInvocation;

import java.util.Set;

/**
 * @author Marko van Dooren
 */

public abstract class ExceptionDeclaration extends ElementImpl {

  public ExceptionDeclaration() {
	}

  public abstract boolean compatibleWith(ExceptionClause clause) throws LookupException;

  public abstract Set getExceptionTypes(MethodInvocation invocation) throws LookupException;

	/**
	 *
	 * @uml.property name="worstCaseExceptionTypes"
	 */
	public abstract Set getWorstCaseExceptionTypes()
		throws LookupException;

  public abstract boolean hasValidAccessibility() throws LookupException;

}


text:
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text;

namespace Xceed.Wpf.DataGrid
{
  // We use an intermediate liste to always acces the source with the index
  // and never directly by keeping a variable that contains the source item directly.
  // ( We do that to be able to work correctly with a DataView has source )
  //
  // RawItem is always pointing at the same item.
  // The dataIndex may be changing to always represent the same item if the item is moving in the source.
  internal class RawItem
  {
    public RawItem( int dataIndex, object dataItem )
    {
      m_dataIndex = dataIndex;
      m_dataItem = dataItem;
      m_sortedIndex = -1;
    }

    public int Index
    {
      get
      {
        return m_dataIndex;
      }
    }

    public int SortedIndex
    {
      get
      {
        return m_sortedIndex;
      }
    }

    public object DataItem
    {
      get
      {
        return m_dataItem;
      }
    }

    public DataGridCollectionViewGroup ParentGroup
    {
      get
      {
        return m_parentGroup;
      }
    }

    public int GetGlobalSortedIndex()
    {
      if( m_parentGroup == null )
        return -1;

      return m_sortedIndex + m_parentGroup.GetFirstRawItemGlobalSortedIndex();
    }

    internal void SetIndex( int newIndex )
    {
      m_dataIndex = newIndex;
    }

    internal void SetParentGroup( DataGridCollectionViewGroup parentGroup )
    {
      m_parentGroup = parentGroup;
    }

    internal void SetSortedIndex( int newIndex )
    {
      m_sortedIndex = newIndex;
    }

    internal void SetDataItem( object dataItem )
    {
      m_dataItem = dataItem;
    }

    private object m_dataItem;
    private int m_dataIndex;
    private int m_sortedIndex;
    private DataGridCollectionViewGroup m_parentGroup;
  }
}


text:
#ifndef GRPC_C_CODEGEN_METHOD_H
#define GRPC_C_CODEGEN_METHOD_H

#include <grpc_c/grpc_c.h>

typedef enum GRPC_rpc_type {
  GRPC_NORMAL_RPC = 0,
  GRPC_CLIENT_STREAMING, /* request streaming */
  GRPC_SERVER_STREAMING, /* response streaming */
  GRPC_BIDI_STREAMING
} GRPC_rpc_type;

struct GRPC_method {
  GRPC_rpc_type type;
  const char* name;
};

typedef struct GRPC_method GRPC_method;

#endif /* GRPC_C_CODEGEN_METHOD_H */


text:
layout: proyect
title: J&TCC
description: Branding y arte para EP
cover: jack_portada.jpg
images:
  - jack_01final.svg
  - jack_02.jpg
  - jack_03.jpg
  - jack_04.jpg
  - jack_05.jpg
  - jack_06.jpg
  - jack_07.jpg
order: 6
---

Jack & The Coward Committee es una banda de la escena de *rock&roll* y psicodelia mexicana; se rige bajo la idea de aceptarse cobarde como un acto de valentía. Puede consultarse su material en [cowardcommittee.bandcamp.com](https://cowardcommittee.bandcamp.com/).

Este proyecto fue realizado en 2011 y consistió en el branding general de la banda y el diseño/arte de su EP *Songs From The Cave*, esto último incluye el booklet para CD y las portadas individuales para cada single; se colaboró con fotógrafos morelianos.


text:
"""Helper object to read and modify Shared Preferences from Android apps.

See e.g.:
  http://developer.android.com/reference/android/content/SharedPreferences.html
"""

import logging
import posixpath
from xml.etree import ElementTree

from devil.android import device_errors
from devil.android.sdk import version_codes

logger = logging.getLogger(__name__)

_XML_DECLARATION = "<?xml version='1.0' encoding='utf-8' standalone='yes' ?>\n"

class BasePref(object):
  """Base class for getting/setting the value of a specific preference type.

  Should not be instantiated directly. The SharedPrefs collection will
  instantiate the appropriate subclasses, which directly manipulate the
  underlying xml document, to parse and serialize values according to their
  type.

  Args:
    elem: An xml ElementTree object holding the preference data.

  Properties:
    tag_name: A string with the tag that must be used for this preference type.
  """
  tag_name = None

  def __init__(self, elem):
    if elem.tag != type(self).tag_name:
      raise TypeError('Property %r has type %r, but trying to access as %r' %
                      (elem.get('name'), elem.tag, type(self).tag_name))
    self._elem = elem

  def __str__(self):
    """Get the underlying xml element as a string."""
    return ElementTree.tostring(self._elem)

  def get(self):
    """Get the value of this preference."""
    return self._elem.get('value')

  def set(self, value):
    """Set from a value casted as a string."""
    self._elem.set('value', str(value))

  @property
  def has_value(self):
    """Check whether the element has a value."""
    return self._elem.get('value') is not None

class BooleanPref(BasePref):
  """Class for getting/setting a preference with a boolean value.

  The underlying xml element has the form, e.g.:
      <boolean name="featureEnabled" value="false" />
  """
  tag_name = 'boolean'
  VALUES = {'true': True, 'false': False}

  def get(self):
    """Get the value as a Python bool."""
    return type(self).VALUES[super(BooleanPref, self).get()]

  def set(self, value):
    """Set from a value casted as a bool."""
    super(BooleanPref, self).set('true' if value else 'false')

class FloatPref(BasePref):
  """Class for getting/setting a preference with a float value.

  The underlying xml element has the form, e.g.:
      <float name="someMetric" value="4.7" />
  """
  tag_name = 'float'

  def get(self):
    """Get the value as a Python float."""
    return float(super(FloatPref, self).get())

class IntPref(BasePref):
  """Class for getting/setting a preference with an int value.

  The underlying xml element has the form, e.g.:
      <int name="aCounter" value="1234" />
  """
  tag_name = 'int'

  def get(self):
    """Get the value as a Python int."""
    return int(super(IntPref, self).get())

class LongPref(IntPref):
  """Class for getting/setting a preference with a long value.

  The underlying xml element has the form, e.g.:
      <long name="aLongCounter" value="1234" />

  We use the same implementation from IntPref.
  """
  tag_name = 'long'

class StringPref(BasePref):
  """Class for getting/setting a preference with a string value.

  The underlying xml element has the form, e.g.:
      <string name="someHashValue">249b3e5af13d4db2</string>
  """
  tag_name = 'string'

  def get(self):
    """Get the value as a Python string."""
    return self._elem.text

  def set(self, value):
    """Set from a value casted as a string."""
    self._elem.text = str(value)

class StringSetPref(StringPref):
  """Class for getting/setting a preference with a set of string values.

  The underlying xml element has the form, e.g.:
      <set name="managed_apps">
          <string>com.mine.app1</string>
          <string>com.mine.app2</string>
          <string>com.mine.app3</string>
      </set>
  """
  tag_name = 'set'

  def get(self):
    """Get a list with the string values contained."""
    value = []
    for child in self._elem:
      assert child.tag == 'string'
      value.append(child.text)
    return value

  def set(self, value):
    """Set from a sequence of values, each casted as a string."""
    for child in list(self._elem):
      self._elem.remove(child)
    for item in value:
      ElementTree.SubElement(self._elem, 'string').text = str(item)

_PREF_TYPES = {
    c.tag_name: c
    for c in
    [BooleanPref, FloatPref, IntPref, LongPref, StringPref, StringSetPref]
}

class SharedPrefs(object):
  def __init__(self, device, package, filename, use_encrypted_path=False):
    """Helper object to read and update "Shared Prefs" of Android apps.

    Such files typically look like, e.g.:

        <?xml version='1.0' encoding='utf-8' standalone='yes' ?>
        <map>
          <int name="databaseVersion" value="107" />
          <boolean name="featureEnabled" value="false" />
          <string name="someHashValue">249b3e5af13d4db2</string>
        </map>

    Example usage:

        prefs = shared_prefs.SharedPrefs(device, 'com.my.app', 'my_prefs.xml')
        prefs.Load()
        prefs.GetString('someHashValue') # => '249b3e5af13d4db2'
        prefs.SetInt('databaseVersion', 42)
        prefs.Remove('featureEnabled')
        prefs.Commit()

    The object may also be used as a context manager to automatically load and
    commit, respectively, upon entering and leaving the context.

    Args:
      device: A DeviceUtils object.
      package: A string with the package name of the app that owns the shared
        preferences file.
      filename: A string with the name of the preferences file to read/write.
      use_encrypted_path: Whether to read and write to the shared prefs location
        in the device-encrypted path (/data/user_de) instead of the older,
        unencrypted path (/data/data). Only supported on N+, but falls back to
        the unencrypted path if the encrypted path is not supported on the given
        device.
    """
    self._device = device
    self._xml = None
    self._package = package
    self._filename = filename
    self._unencrypted_path = '/data/data/%s/shared_prefs/%s' % (package,
                                                                filename)
    self._encrypted_path = '/data/user_de/0/%s/shared_prefs/%s' % (package,
                                                                   filename)
    self._path = self._unencrypted_path
    self._encrypted = use_encrypted_path
    if use_encrypted_path:
      if self._device.build_version_sdk < version_codes.NOUGAT:
        logging.info('SharedPrefs set to use encrypted path, but given device '
                     'is not running N+. Falling back to unencrypted path')
        self._encrypted = False
      else:
        self._path = self._encrypted_path
    self._changed = False

  def __repr__(self):
    """Get a useful printable representation of the object."""
    return '<{cls} file {filename} for {package} on {device}>'.format(
        cls=type(self).__name__,
        filename=self.filename,
        package=self.package,
        device=str(self._device))

  def __str__(self):
    """Get the underlying xml document as a string."""
    return _XML_DECLARATION + ElementTree.tostring(self.xml)

  @property
  def package(self):
    """Get the package name of the app that owns the shared preferences."""
    return self._package

  @property
  def filename(self):
    """Get the filename of the shared preferences file."""
    return self._filename

  @property
  def path(self):
    """Get the full path to the shared preferences file on the device."""
    return self._path

  @property
  def changed(self):
    """True if properties have changed and a commit would be needed."""
    return self._changed

  @property
  def xml(self):
    """Get the underlying xml document as an ElementTree object."""
    if self._xml is None:
      self._xml = ElementTree.Element('map')
    return self._xml

  def Load(self):
    """Load the shared preferences file from the device.

    A empty xml document, which may be modified and saved on |commit|, is
    created if the file does not already exist.
    """
    if self._device.FileExists(self.path):
      self._xml = ElementTree.fromstring(
          self._device.ReadFile(self.path, as_root=True))
      assert self._xml.tag == 'map'
    else:
      self._xml = None
    self._changed = False

  def Clear(self):
    """Clear all of the preferences contained in this object."""
    if self._xml is not None and len(self):  # only clear if not already empty
      self._xml = None
      self._changed = True

  def Commit(self, force_commit=False):
    """Save the current set of preferences to the device.

    Only actually saves if some preferences have been modified or force_commit
    is set to True.

    Args:
      force_commit: Commit even if no changes have been made to the SharedPrefs
        instance.
    """
    if not (self.changed or force_commit):
      return
    self._device.RunShellCommand(
        ['mkdir', '-p', posixpath.dirname(self.path)],
        as_root=True,
        check_return=True)
    self._device.WriteFile(self.path, str(self), as_root=True)
    # Creating the directory/file can cause issues with SELinux if they did
    # not already exist. As a workaround, apply the package's security context
    # to the shared_prefs directory, which mimics the behavior of a file
    # created by the app itself
    if self._device.build_version_sdk >= version_codes.MARSHMALLOW:
      security_context = self._device.GetSecurityContextForPackage(
          self.package, encrypted=self._encrypted)
      if security_context is None:
        raise device_errors.CommandFailedError(
            'Failed to get security context for %s' % self.package)
      paths = [posixpath.dirname(self.path), self.path]
      self._device.ChangeSecurityContext(security_context, paths)

    # Ensure that there isn't both an encrypted and unencrypted version of the
    # file on the device at the same time.
    if self._device.build_version_sdk >= version_codes.NOUGAT:
      remove_path = (self._unencrypted_path
                     if self._encrypted else self._encrypted_path)
      if self._device.PathExists(remove_path, as_root=True):
        logging.warning('Found an equivalent shared prefs file at %s, removing',
                        remove_path)
        self._device.RemovePath(remove_path, as_root=True)

    self._device.KillAll(self.package, exact=True, as_root=True, quiet=True)
    self._changed = False

  def __len__(self):
    """Get the number of preferences in this collection."""
    return len(self.xml)

  def PropertyType(self, key):
    """Get the type (i.e. tag name) of a property in the collection."""
    return self._GetChild(key).tag

  def HasProperty(self, key):
    try:
      self._GetChild(key)
      return True
    except KeyError:
      return False

  def GetBoolean(self, key):
    """Get a boolean property."""
    return BooleanPref(self._GetChild(key)).get()

  def SetBoolean(self, key, value):
    """Set a boolean property."""
    self._SetPrefValue(key, value, BooleanPref)

  def GetFloat(self, key):
    """Get a float property."""
    return FloatPref(self._GetChild(key)).get()

  def SetFloat(self, key, value):
    """Set a float property."""
    self._SetPrefValue(key, value, FloatPref)

  def GetInt(self, key):
    """Get an int property."""
    return IntPref(self._GetChild(key)).get()

  def SetInt(self, key, value):
    """Set an int property."""
    self._SetPrefValue(key, value, IntPref)

  def GetLong(self, key):
    """Get a long property."""
    return LongPref(self._GetChild(key)).get()

  def SetLong(self, key, value):
    """Set a long property."""
    self._SetPrefValue(key, value, LongPref)

  def GetString(self, key):
    """Get a string property."""
    return StringPref(self._GetChild(key)).get()

  def SetString(self, key, value):
    """Set a string property."""
    self._SetPrefValue(key, value, StringPref)

  def GetStringSet(self, key):
    """Get a string set property."""
    return StringSetPref(self._GetChild(key)).get()

  def SetStringSet(self, key, value):
    """Set a string set property."""
    self._SetPrefValue(key, value, StringSetPref)

  def Remove(self, key):
    """Remove a preference from the collection."""
    self.xml.remove(self._GetChild(key))

  def AsDict(self):
    """Return the properties and their values as a dictionary."""
    d = {}
    for child in self.xml:
      pref = _PREF_TYPES[child.tag](child)
      d[child.get('name')] = pref.get()
    return d

  def __enter__(self):
    """Load preferences file from the device when entering a context."""
    self.Load()
    return self

  def __exit__(self, exc_type, _exc_value, _traceback):
    """Save preferences file to the device when leaving a context."""
    if not exc_type:
      self.Commit()

  def _GetChild(self, key):
    """Get the underlying xml node that holds the property of a given key.

    Raises:
      KeyError when the key is not found in the collection.
    """
    for child in self.xml:
      if child.get('name') == key:
        return child
    raise KeyError(key)

  def _SetPrefValue(self, key, value, pref_cls):
    """Set the value of a property.

    Args:
      key: The key of the property to set.
      value: The new value of the property.
      pref_cls: A subclass of BasePref used to access the property.

    Raises:
      TypeError when the key already exists but with a different type.
    """
    try:
      pref = pref_cls(self._GetChild(key))
      old_value = pref.get()
    except KeyError:
      pref = pref_cls(
          ElementTree.SubElement(self.xml, pref_cls.tag_name, {'name': key}))
      old_value = None
    if old_value != value:
      pref.set(value)
      self._changed = True
      logger.info('Setting property: %s', pref)


text:
import json
import time
import re
from collections import OrderedDict

from doajtest.helpers import DoajTestCase
from portality import models
from portality.upgrade import do_upgrade
from portality.lib.paths import rel2abs

def operation(journal):
    j = models.Journal.pull(journal.id)
    bj = j.bibjson()
    bj.title = "Updated Title"
    j.save()
    return j

class TestUpgrade(DoajTestCase):

    def test_upgrade(self):
        # populate the index with some journals with title
        saved_journals = {}
        for i in range(5):
            j = models.Journal()
            j.set_in_doaj(True)
            bj = j.bibjson()
            bj.title = "Test Journal"
            bj.add_identifier(bj.P_ISSN, "{x}000-0000".format(x=i))
            bj.publisher = "Test Publisher {x}".format(x=i)
            bj.add_url("http://homepage.com/{x}".format(x=i), "homepage")
            j.save()
            saved_journals[j.id] = j.last_updated

        # and with some journals without title
        for i in range(5):
            j = models.Journal()
            j.set_in_doaj(True)
            bj = j.bibjson()
            bj.add_identifier(bj.P_ISSN, "{x}000-0001".format(x=i))
            bj.title = "Journal to Change"
            bj.publisher = "Test Publisher {x}".format(x=i)
            bj.add_url("http://homepage.com/{x}".format(x=i), "homepage")
            j.save()
            saved_journals[j.id] = j.last_updated

        # make sure the last updated dates will be suitably different after migration
        time.sleep(1.5)

        path =rel2abs(__file__, ".", "resources", "migrate.json")
        with open(path) as f:
            instructions = json.loads(f.read(), object_pairs_hook=OrderedDict)
        do_upgrade(instructions,None)

        p = re.compile('[0-4]000-0001')

        for id in saved_journals:
            j = models.Journal.pull(id)
            bj = j.bibjson()
            pissn = bj.get_one_identifier(bj.P_ISSN)

            if not p.match(pissn):
                assert bj.title == "Test Journal"
                assert j.last_updated == saved_journals[j.id]

            else:
                assert bj.title == "Updated Title"
                assert not j.last_updated == saved_journals[j.id]


text:
import {JsonSchemesRegistry} from "../../../../src/common/jsonschema/registries/JsonSchemesRegistry";
import {PropertyRegistry} from "../../../../src/common/jsonschema/registries/PropertyRegistry";
import {decoratorSchemaFactory} from "../../../../src/common/jsonschema/utils/decoratorSchemaFactory";
import {Sinon} from "../../../tools";

class Test {

}

describe("decoratorSchemaFactory()", () => {
    describe("when there is a property", () => {
        before(() => {
            this.getStub = Sinon.stub(PropertyRegistry, "get");
            this.getStub.returns({schema: "schema"});

            this.decoratorStub = Sinon.stub();
            this.cbStub = Sinon.stub().returns(this.decoratorStub);
            decoratorSchemaFactory(this.cbStub)(Test, "test");
        });

        after(() => {
            this.getStub.restore();
        });

        it("should call PropertyRegistry.get()", () => {
            this.getStub.should.be.calledWithExactly(Test, "test");
        });

        it("should call the fn callback with the correct parameters", () => {
            this.cbStub.should.be.calledWithExactly("schema", [Test, "test"]);
        });

        it("should cal the decorators returned by the fn callback", () => {
            this.decoratorStub.should.be.calledWithExactly(Test, "test");
        });
    });

    describe("when there is a class", () => {
        before(() => {
            this.getStub = Sinon.stub(JsonSchemesRegistry, "get");
            this.getStub.returns("schema");

            this.decoratorStub = Sinon.stub();
            this.cbStub = Sinon.stub().returns(this.decoratorStub);
            decoratorSchemaFactory(this.cbStub)(Test);
        });

        after(() => {
            this.getStub.restore();
        });

        it("should call PropertyRegistry.get()", () => {
            this.getStub.should.be.calledWithExactly(Test);
        });

        it("should call the fn callback with the correct parameters", () => {
            this.cbStub.should.be.calledWithExactly("schema", [Test]);
        });

        it("should cal the decorators returned by the fn callback", () => {
            this.decoratorStub.should.be.calledWithExactly(Test);
        });
    });
});


text:
{%if env.isOwner %}
Tutorial Narratives
{% else %}
Browse Accessible Tutorial Narratives
{% endif %}


text:
package org.dependencytrack.integrations;

import org.dependencytrack.model.Finding;
import org.dependencytrack.model.Project;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;

import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

public class ProjectFindingUploaderTest {

    @Test
    @SuppressWarnings("unchecked")
    public final void projectFindingMethodsTest() throws IOException {
        Project project = new Project();
        List<Finding> findings = Collections.EMPTY_LIST;
        ProjectFindingUploader uploader = mock(ProjectFindingUploader.class);
        when(uploader.process(project, findings)).thenReturn(new InputStream() {
            @Override
            public int read() throws IOException {
                return 1;
            }
            @Override
            public int available() throws IOException {
                return 1;
            }
        });
        InputStream in = uploader.process(project, findings);
        Assert.assertTrue(in != null && in.available() == 1);
        uploader.upload(project, in);
        when(uploader.isProjectConfigured(project)).thenReturn(true);
        Assert.assertTrue(uploader.isProjectConfigured(project));
        when(uploader.isProjectConfigured(project)).thenReturn(false);
        Assert.assertFalse(uploader.isProjectConfigured(project));
    }
}


text:
<?php

require(__DIR__ . '/../vendor/yiisoft/yii2/BaseYii.php');

class Yii extends \yii\BaseYii{}
class Bee extends Yii{
    /**
     * Function t
     * Translates a message to the specified language.
     *
     * This is a shortcut method of [[\yii\i18n\I18N::translate()]].
     *
     * ```php
     * $username = 'Alexander';
     * echo \Bee::t('component_name', 'app', 'Hello, {username}!', ['username' => $username]);
     * ```
     *
     * @static
     * @param string $component name in "Component folder"
     * @param string $category the message category.
     * @param string $message the message to be translated.
     * @param array $params the parameters that will be used to replace the corresponding placeholders in the message.
     * @param string $language the language code (e.g. `en-US`, `en`). If this is null, the current
     * [[\yii\base\Application::language|application language]] will be used.
     * @return string the translated message.
     */
    public static function t($component = '', $category, $message, $params = [], $language = null) {
        if($component === 'bee') {
            $category = 'libs/bee/messages/' . $category;
        } elseif($component !== '') {
            $category = 'components/' . $component . '/language/' . $category;
        }
        return parent::t($category, $message, $params, $language);
    }

    /**
     * Returns an HTML hyperlink that can be displayed on your Web page showing "Powered by Yii Framework" information.
     * @return string an HTML hyperlink that can be displayed on your Web page showing "Powered by Yii Framework" information
     */
    public static function powered() {
        return \Bee::t('bee', 'bee', 'POWERED_BY {bee}', [
            'bee' => '' .
                         \Bee::t('bee', 'bee', 'BEE_CMS') .
                     ''
        ]);
    }
}

spl_autoload_register(['Bee', 'autoload'], true, true);
Bee::$classMap = require('/../vendor/yiisoft/yii2/classes.php');
Bee::$container = new yii\di\Container();


text:
namespace SourceIndex {

	using namespace LightSpeed;

	class FileInfo: public ComparableLess<FileInfo>
	{
	public:

		FilePath fullPath;
		TimeStamp modified;

		FileInfo(FilePath fullPath, TimeStamp modified);

		static FileInfo getFileInfo(ConstStrW fname);

		bool lessThan(const FileInfo &nfo) const;

	};

}