text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><link rel="stylesheet" type="text/css" media="all">

    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <meta charset="utf-8">
    <title>Beauty Product Testing (Results)</title>
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta name="description" content="Feeling Beautiful Skin Care application">

    <link href="/include/css/bootstrap-combined.min.css" rel="stylesheet">
    <link href="/include/css/bootstrap-responsive.min.css" rel="stylesheet">
    <link href="/include/css/dev.min.css" rel="stylesheet">

                        Warning!

                        Currently there are only 9 Deals remaining in
                        America:
                        Apply now to be accepted.

                    Terms & Conditions

Please read these Terms and Conditions ("Terms", "Terms and Conditions") carefully before using the http://youthfulbeautyproducts.org/ website (the "Service") operated by Ladies Beauty Care ("us", "we", or "our").

Your access to and use of the Service is conditioned on your acceptance of and compliance with these Terms. These Terms apply to all visitors, users and others who access or use the Service.

By accessing or using the Service you agree to be bound by these Terms. If you disagree with any part of the terms then you may not access the Service.

Links To Other Web Sites

Our Service may contain links to third-party web sites or services that are not owned or controlled by Ladies Beauty Care.

Ladies Beauty Care has no control over, and assumes no responsibility for, the content, privacy policies, or practices of any third party web sites or services. You further acknowledge and agree that Ladies Beauty Care shall not be responsible or liable, directly or indirectly, for any damage or loss caused or alleged to be caused by or in connection with use of or reliance on any such content, goods or services available on or through any such web sites or services.

We strongly advise you to read the terms and conditions and privacy policies of any third-party web sites or services that you visit.

Governing Law

These Terms shall be governed and construed in accordance with the laws of United States, without regard to its conflict of law provisions.

Our failure to enforce any right or provision of these Terms will not be considered a waiver of those rights. If any provision of these Terms is held to be invalid or unenforceable by a court, the remaining provisions of these Terms will remain in effect. These Terms constitute the entire agreement between us regarding our Service, and supersede and replace any prior agreements we might have between us regarding the Service.

Changes

We reserve the right, at our sole discretion, to modify or replace these Terms at any time. If a revision is material we will try to provide at least 30 days notice prior to any new terms taking effect. What constitutes a material change will be determined at our sole discretion.

By continuing to access or use our Service after those revisions become effective, you agree to be bound by the revised terms. If you do not agree to the new terms, please stop using the Service.

Contact Us

If you have any questions about these Terms, please contact us.

        <?php include('../footer.php'); ?>


text:
package fr.inra.maiage.bibliome.alvisnlp.bibliomefactory.modules.fasttext;

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;

import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Corpus;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.Element;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.EvaluationContext;
import fr.inra.maiage.bibliome.alvisnlp.core.corpus.expressions.Evaluator;
import fr.inra.maiage.bibliome.alvisnlp.core.module.ProcessingContext;
import fr.inra.maiage.bibliome.alvisnlp.core.module.lib.ExternalHandler;
import fr.inra.maiage.bibliome.util.Iterators;
import fr.inra.maiage.bibliome.util.Strings;
import fr.inra.maiage.bibliome.util.streams.FileTargetStream;
import fr.inra.maiage.bibliome.util.streams.TargetStream;

public abstract class FasttextClassifierBaseExternalHandler<R extends FasttextClassifierBaseResolvedObjects, M extends FasttextClassifierBase<R>> extends ExternalHandler<Corpus,M> {
	protected static final String FASTTEXT_CLASS_FEATURE_PREFIX = "__label__";

	protected FasttextClassifierBaseExternalHandler(ProcessingContext<Corpus> processingContext, M module, Corpus annotable) {
		super(processingContext, module, annotable);
	}

	protected void writeDocumentLines(EvaluationContext evalCtx, File file, Evaluator documents, FasttextAttribute.Resolved[] attributes, boolean includeClass, boolean classWeights) throws IOException {
		TargetStream target = new FileTargetStream("UTF-8", file.getAbsolutePath());
		try (PrintStream ps = target.getPrintStream()) {
			int nDocs = 0;
			for (Element doc : Iterators.loop(documents.evaluateElements(evalCtx, getAnnotable()))) {
				Collection<String> line = getDocumentLine(evalCtx, doc, attributes, includeClass);
				int w = classWeights ? getDocWeight(doc) : 1;
				for (int i = 0; i < w; ++i) {
					Strings.join(ps, line, '\t');
					ps.println();
				}
				nDocs++;
			}
			getLogger().info("# of documents: " + nDocs);
		}
	}

	protected abstract int getDocWeight(Element doc);

	@Override
	protected String getPrepareTask() {
		return "alvisnlp-to-fasttext";
	}

	protected Collection<String> getDocumentLine(EvaluationContext evalCtx, Element doc, FasttextAttribute.Resolved[] attributes, boolean includeClass) {
		FasttextClassifierBase<R> owner = getModule();
		Collection<String> result = new ArrayList<String>();
		for (FasttextAttribute.Resolved attr : attributes) {
			String value = getAttributeValue(attr, evalCtx, doc);
			result.add(value);
		}
		if (includeClass) {
			String theClass = doc.getLastFeature(owner.getClassFeature());
			result.add(FASTTEXT_CLASS_FEATURE_PREFIX + theClass);
		}
		return result;
	}

	private String getAttributeValue(FasttextAttribute.Resolved attr, EvaluationContext evalCtx, Element doc) {
		Collection<String> tokens = new ArrayList<String>();
		for (Element token : Iterators.loop(attr.getTokens().evaluateElements(evalCtx, doc))) {
			String form = attr.getForm().evaluateString(evalCtx, token);
			form = Strings.normalizeSpace(form);
			form = form.replace(' ', '_');
			tokens.add(form);
		}
		return Strings.join(tokens, ' ');
	}

	@Override
	protected void updateEnvironment(Map<String,String> env) {
	}

	@Override
	protected File getWorkingDirectory() {
		return null;
	}

	@Override
	protected String getInputFileame() {
		return null;
	}
}

redpajama_set_name: RedPajamaGithub


text:
/* TEMPLATE GENERATED TESTCASE FILE
Filename: CWE121_Stack_Based_Buffer_Overflow__CWE805_char_declare_loop_12.c
Label Definition File: CWE121_Stack_Based_Buffer_Overflow__CWE805.string.label.xml
Template File: sources-sink-12.tmpl.c
*/
/*
 * @description
 * CWE: 121 Stack Based Buffer Overflow
 * BadSource:  Set data pointer to the bad buffer
 * GoodSource: Set data pointer to the good buffer
 * Sink: loop
 *    BadSink : Copy string to data using a loop
 * Flow Variant: 12 Control flow: if(globalReturnsTrueOrFalse())
 *
 * */

#include "std_testcase.h"

#include <wchar.h>

#ifndef OMITBAD

void CWE121_Stack_Based_Buffer_Overflow__CWE805_char_declare_loop_12_bad()
{
    char * data;
    char dataBadBuffer[50];
    char dataGoodBuffer[100];
    if(globalReturnsTrueOrFalse())
    {
        /* FLAW: Set a pointer to a "small" buffer. This buffer will be used in the sinks as a destination
         * buffer in various memory copying functions using a "large" source buffer. */
        data = dataBadBuffer;
        data[0] = '\0'; /* null terminate */
    }
    else
    {
        /* FIX: Set a pointer to a "large" buffer, thus avoiding buffer overflows in the sinks. */
        data = dataGoodBuffer;
        data[0] = '\0'; /* null terminate */
    }
    {
        size_t i;
        char source[100];
        memset(source, 'C', 100-1); /* fill with 'C's */
        source[100-1] = '\0'; /* null terminate */
        /* POTENTIAL FLAW: Possible buffer overflow if the size of data is less than the length of source */
        for (i = 0; i < 100; i++)
        {
            data[i] = source[i];
        }
        data[100-1] = '\0'; /* Ensure the destination buffer is null terminated */
        printLine(data);
    }
}

#endif /* OMITBAD */

#ifndef OMITGOOD

/* goodG2B() - use goodsource and badsink by changing the "if" so that
 * both branches use the GoodSource */
static void goodG2B()
{
    char * data;
    char dataBadBuffer[50];
    char dataGoodBuffer[100];
    if(globalReturnsTrueOrFalse())
    {
        /* FIX: Set a pointer to a "large" buffer, thus avoiding buffer overflows in the sinks. */
        data = dataGoodBuffer;
        data[0] = '\0'; /* null terminate */
    }
    else
    {
        /* FIX: Set a pointer to a "large" buffer, thus avoiding buffer overflows in the sinks. */
        data = dataGoodBuffer;
        data[0] = '\0'; /* null terminate */
    }
    {
        size_t i;
        char source[100];
        memset(source, 'C', 100-1); /* fill with 'C's */
        source[100-1] = '\0'; /* null terminate */
        /* POTENTIAL FLAW: Possible buffer overflow if the size of data is less than the length of source */
        for (i = 0; i < 100; i++)
        {
            data[i] = source[i];
        }
        data[100-1] = '\0'; /* Ensure the destination buffer is null terminated */
        printLine(data);
    }
}

void CWE121_Stack_Based_Buffer_Overflow__CWE805_char_declare_loop_12_good()
{
    goodG2B();
}

#endif /* OMITGOOD */

/* Below is the main(). It is only used when building this testcase on
 * its own for testing or for building a binary to use in testing binary
 * analysis tools. It is not used when compiling all the testcases as one
 * application, which is how source code analysis tools are tested.
 */

#ifdef INCLUDEMAIN

int main(int argc, char * argv[])
{
    /* seed randomness */
    srand( (unsigned)time(NULL) );
#ifndef OMITGOOD
    printLine("Calling good()...");
    CWE121_Stack_Based_Buffer_Overflow__CWE805_char_declare_loop_12_good();
    printLine("Finished good()");
#endif /* OMITGOOD */
#ifndef OMITBAD
    printLine("Calling bad()...");
    CWE121_Stack_Based_Buffer_Overflow__CWE805_char_declare_loop_12_bad();
    printLine("Finished bad()");
#endif /* OMITBAD */
    return 0;
}

#endif


text:
const express = require('express');
const app = express();
const connect = require('connect');
const path    = require("path");
const request = require('request');
const EventEmitter = require('events');

const credentials = require('../../credentials.js');
const user = require('./models/user.js');    // DB 끌어오기

app.set('port', process.env.PORT || 62424);

// app.set('env', 'production');

app.use(function (req, res, next) {
    const domain = require('domain').create();
    domain.on('error', function (err) {
        console.error('DOMAIN ERROR CAUGHT\n', err.stack);
        try {
            // 5초 후 shutdown
            setTimeout(function () {
                console.error('Failsafe shutdown.');
                process.exit(1);
            }, 5000);

            // 클러스터 연결 해제
            const worker = require('cluster').worker;
            if (worker) worker.disconnect();

            // req를 그만 받음
            server.close();

            try {
                next(err);
            } catch (error) {
                console.error('Express error mechanism failed.\n', error.stack);
                res.statusCode = 500;
                res.setHeader('content-type', 'text/plain');
                res.end('Server error.');
            }
        } catch (error) {
            console.error('Unable to send 500 response.\n', error.stack);
        }
    });

    domain.add(req);
    domain.add(res);

    domain.run(next);
});
// static - public 폴더
app.use(express.static(__dirname + '/public'));

/* csurf 공격 방어
app.use(require('csurf')());
app.use(function (req, res, next) {
    res.locals._csrfToken = req.csrfToken();
    next();
});
*/

// logging 기능 추가
switch (app.get('env')) {
    case 'development':
        app.use(require('morgan')('dev'));
        break;
    case 'production':
        app.use(require('express-logger')({
            path: __dirname + '/log/requests.log'
        }));
}

// json, urlencoded 제공하는 미들웨어
app.use(require('body-parser').urlencoded({ extended : true }));

// 서명된 쿠키 시크릿 사용
app.use(require('cookie-parser')(credentials.cookieSecret));

// 세션 사용 - 저장소 : MongoDB
const session = require('express-session');
const MongoStore = require('connect-mongo')(session);
const sessionStore = new MongoStore({
    url:credentials.mongo[app.get('env')].sessionString,
    ttl:7 * 24 * 60 * 60
});

app.use(session({
    resave : false,
    saveUninitialized: false,
    secret : credentials.cookieSecret,
    store : sessionStore,
    cookie : {
        httpOnly :true, //서버에서만 쿠키 수정 가능
        secure : false, //https 연결에서만 보냄
        maxAge : 1000 * 60 * 60 * 24 * 7,
        signed : true
    }
}));

// mocha QA 테스트 코드 - 쿼리스트링 test=1 감지
app.use(function (req, res, next) {
    res.locals.showTests = app.get('env') !== 'production' && req.query.test === '1';
    next();
});

// DB 연동 - mongoose 모듈 사용(mongoDB)
const mongoose = require('mongoose');
const options = {
    server: {
       socketOptions: { keepAlive: 1 }
    }
};
switch(app.get('env')){
    case 'development':
        mongoose.connect(credentials.mongo.development.userString, options);
        break;
    case 'production':
        mongoose.connect(credentials.mongo.production.userString, options);
        break;
    default:
        throw new Error('Unknown execution environment: ' + app.get('env'));
}

// 인증
app.set('BASE_URL', process.env.BASE_URL || 'localhost:62424');

const auth = require('./lib/auth.js')(app, {
    baseUrl: process.env.BASE_URL,
    providers: credentials.authProviders,
    successRedirect: '/account',
    failureRedirect: '/unauthorized'
});
auth.init();
auth.registerRoutes();

// 고객 사용 가능 페이지
// >> 라우팅 시 app.get('/account',customerOnly,function ... ) 식으로 사용
const customerOnly = (req, res, next) => {
    if (req.user && req.user.role === 'customer') return next();
    res.redirect(303, '/unauthorized');
}

// 직원 사용 가능 페이지
const employeeOnly = (req, res, next) => {
    if (req.user && req.user.role === 'employee') return next();
    next('route');
}

// 여러 역할 사용 가능 페이지
const allow = (roles) => {
    return function (req, res, next) {
        if (req.user && roles.split(',').indexOf(req.user.role) !== -1) return next();
        res.redirect(303, '/unauthorized');
    };
}

// unauthorized 라우팅
app.get('/unauthorized', function (req, res) {
    res.status(403).render('unauthorized');
});

// 로그인 확인
const confirm_login = (req,res,user) => {
    if(req.session && req.session.passport && req.session.passport.user){
        user.findOne({_id:req.session.passport.user},function(err,data){
            if(err){
                console.log('login Error occured');
            }else{
                // client에 데이터 담아서 쏴주기
                console.log('로그인 굿');
                return true;
            }
        });
    }
    else
        console.log('ㅇㅅㅇ');
        return false;
}

// 페이지 라우팅
app.get('/', function (req, res) {
    confirm_login(req,res,user);
    res.type('text/plain');
    res.send('home');
});
app.get('/about', function (req, res) {
    res.type('text/plain');
    res.send('about');
});
app.get('/account', function (req, res) {
    res.type('text/plain');
    res.send('account');
});
app.get('/unauthorized', function (req, res) {
    res.type('text/plain');
    res.send('unauthorized');
});
app.get('/logout', function (req, res) {
    req.session.passport.user = null;
    delete req.session.passport.user;
    res.redirect('/');
});

// 와우자 연동 테스트
app.get('/requesting',function(req,res){

    request.get({
        url:'http://localhost:8087/v2/servers/_defaultServer_/vhosts/_defaultVHost_/applications',
        headers:{
            "Content-Type": "application/json",
            "Accept":'application/json',
            "charset":'utf-8'
        }
    },function (error, response, body) {
        if (!error && response.statusCode == 200) {
            const info = JSON.parse(body);
            console.log(info);
        }
        else
            console.log('error on wowza');
    }).auth(credentials.wowza.userid,credentials.wowza.userpw, false);
    res.redirect('/');
});

// 와우자 앱 추가테스트
app.get('/requesting2',function(req,res){
    const myEmitter = new EventEmitter();

    let name;
    let clientIP;
    let authID;
    if(req.session && req.session.passport && req.session.passport.user){
        user.findOne({_id:req.session.passport.user},function(err,data) {
            name = data.name;
            clientIP = data.ip;
            authID = data.authId;

            myEmitter.emit('login_success');
            if(err){
                console.log('login Error occured');
            }
        });
    }else{
        console.log("you're not logined");
        res.redirect('/');
    }

    myEmitter.on('login_success', () =>{ // 로그인 완료 후 앱 생성
        request.post({
            url : 'http://localhost:8087/v2/servers/_defaultServer_/vhosts/_defaultVHost_/applications/' + authID + "_vod",
            headers:{
                "Content-Type": "application/json",
                "Accept" : "application/json",
                "charset" : "utf-8"
            },
            json : {
                name: authID + "_vod",
                appType: "Live",
                clientStreamReadAccess: "*",
                clientStreamWriteAccess: "*",
                description: "Testing our Rest Service",
                securityConfig : {
                    publishIPWhiteList : clientIP,
                    publishBlockDuplicateStreamNames : true
                },
                streamConfig: {
                    streamType: "live"
                }
            }
        },function (error, response, body) {
            if (!error && response.statusCode == 200) {
                const info = JSON.parse(body);
                console.log(info);
            }
            else
                console.log(response.statusCode);
        }
        ).auth(credentials.wowza.userid,credentials.wowza.userpw, false);
        res.redirect('/');
    });
});
// 404 오류처리
app.use(function (req, res, next) {
    res.type('text/plain');
    res.status(404);
    res.send('404 Not Found');
});

// 500 오류처리
app.use(function (err, req, res, next) {
    console.error(err.stack);
    res.type('text/plain');
    res.status(500);
    res.send('500 Server Error');
});

function startServer() {
    app.listen(app.get('port'), function () {
        console.log('Express started in ' + app.get('env') +
        ' mode on http://localhost:' +
        app.get('port') + '; press Ctrl-C to terminate.');
    });
}
if (require.main === module) {
    startServer();
} else {
    module.exports = startServer;
}


text:
<?php

namespace app\assets;

use yii\web\AssetBundle;

/**
 * Clase donde añadir archivos css, javascript, dependencias, etc.
 *
 */
class AppAsset extends AssetBundle
{
    public $basePath = '@webroot';
    public $baseUrl = '@web';
    public $css = [
        'css/site.css',
        'css/login.css',
        'css/index.css',
        'css/chatpublico.css',
    ];
    public $js = [
        'js/login.js',
        'js/index.js',
        'js/jquery.galeria.js',
        'js/jquery.reproductor.js',
        'js/validacion.js',
    ];
    public $depends = [
        'yii\web\YiiAsset',
        'yii\bootstrap\BootstrapAsset',
    ];
}


text:
import { OvaleDebugClass } from "./Debug";
import { L } from "./Localization";
import { sort, insert, concat } from "@wowts/table";
import { pairs, ipairs, wipe, tostring, lualength, LuaObj, LuaArray } from "@wowts/lua";

let tsort = sort;
let tinsert = insert;
let tconcat = concat;

interface Trait {
    name?: string;
    spellID: string;
    currentRank?: number;
}

export class OvaleArtifactClass {
    self_traits: LuaObj<Trait> = {}

    debugOptions = {
        artifacttraits: {
            name: L["Artifact traits"],
            type: "group",
            args: {
                artifacttraits: {
                    name: L["Artifact traits"],
                    type: "input",
                    multiline: 25,
                    width: "full",
                    get: () => {
                        return this.DebugTraits();
                    }
                }
            }
        }
    }

    constructor(ovaleDebug: OvaleDebugClass) {
        for (const [k, v] of pairs(this.debugOptions)) {
            ovaleDebug.defaultOptions.args[k] = v;
        }
    }

    OnInitialize() {
    }
    OnDisable() {
    }
    UpdateTraits() {
        return;
    }
    HasTrait(spellId: number) {
        return false;
    }
    TraitRank(spellId: number) {
        return 0;
    }
    output: LuaArray<string> = {}
    DebugTraits() {
        wipe(this.output);
        let array: LuaArray<string> = {}
        for (const [k, v] of pairs(this.self_traits)) {
            tinsert(array, `${tostring(v.name)}: ${tostring(k)}`);
        }
        tsort(array);
        for (const [, v] of ipairs(array)) {
            this.output[lualength(this.output) + 1] = v;
        }
        return tconcat(this.output, "\n");
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE tiles-definitions PUBLIC
	"-//Apache Software Foundation//DTD Tiles Configuration 2.0//EN"
	"http://tiles.apache.org/dtds/tiles-config_2_0.dtd">

<tiles-definitions>

	<definition name="intro" extends="standardLayout">
		<put-attribute name="body" value="/WEB-INF/intro.jsp" />
	</definition>

	<definition name="login" extends="standardLayout">
		<put-attribute name="body" value="/WEB-INF/login.jsp" />
	</definition>

	<definition name="logoutSuccess" extends="standardLayout">
		<put-attribute name="body" value="/WEB-INF/logoutSuccess.jsp" />
	</definition>

</tiles-definitions>


text:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>exist-book</groupId>
        <artifactId>fluent-embedded-parent</artifactId>
        <version>1.0</version>
        <relativePath>../fluent-embedded-parent</relativePath>
    </parent>

    <artifactId>fluent-embedded-example</artifactId>
    <packaging>jar</packaging>

    <name>Fluent Embedded Client Example</name>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                        <configuration>
                            <finalName>${project.artifactId}-${project.version}-example</finalName>
                            <appendAssemblyId>false</appendAssemblyId>
                            <descriptorRefs>
                                <descriptorRef>jar-with-dependencies</descriptorRef>
                            </descriptorRefs>
                            <archive>
                                <manifest>
                                    <mainClass>exist.book.example.fluent.embedded.ExampleApp</mainClass>
                                </manifest>
                            </archive>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

    <dependencies>
        <dependency>
            <groupId>org.exist-db</groupId>
            <artifactId>existdb-fluent</artifactId>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <scope>runtime</scope>
        </dependency>
    </dependencies>

</project>


text:
#region File Description

#endregion

#region Using Statements
using Seshat.Structure;
using System;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using BinaryAnalysis.UnidecodeSharp;
using System.Xml;
using System.Windows;
using System.Security.Cryptography;
using System.Collections.Generic;
using Seshat.Helpers;
using System.Threading;
using System.Diagnostics;
#endregion

namespace Seshat.Tab
{
    public sealed class EstimatedTimer : IDisposable
    {
        #region Fields

        private object xmlLock = null;

        private CancellationTokenSource cancelToken;
        private List<AudioFile> mp3Files = null;
        private Book book = null;
        private Chapter lastChapter = null;
        private AudioBookStructure structure = AudioBookStructure.UNKNOWN;
        private Stopwatch stopwatch = null;

        private long completedSentences = 0;
        private long totalCompletedSentences = 0;
        private long totalAmountOfSentences = 0;

        private double charsPerSecond = 0;
        private double audioLength = 0;

        #endregion

        #region Events

        /// <summary>
        /// Event fired when the gui should update the progress.
        /// </summary>
        public event EventHandler<double> UpdateProgress;

        /// <summary>
        /// Event fired when the gui should update to show that we're saving.
        /// </summary>
        public event EventHandler Saving;

        /// <summary>
        /// Event fired when the engine has completed the search.
        /// </summary>
        public event EventHandler<Book> Completed;

        #endregion

        #region Constructors

        /// <summary>
        /// Creates a new estimated timing instance.
        /// </summary>
        /// <param name="xmlLock">The xml lock.</param>
        public EstimatedTimer(object xmlLock)
        {
            this.xmlLock = xmlLock;
            this.mp3Files = new List<AudioFile>();
            this.cancelToken = new CancellationTokenSource();
            this.stopwatch = new Stopwatch();
        }

        #endregion

        #region Public Methods

        /// <summary>
        /// Creates a estimated timing file for the book.
        /// </summary>
        /// <param name="book">The book to estimate the timing file for.</param>
        public void Estimate(Book book)
        {
            Task.Run(
            () =>
            {
                try
                {
                    this.book = book;
                    this.stopwatch.Start();
                    this.mp3Files = new List<AudioFile>();
                    foreach (AudioFile audioFile in book.AudioFiles)
                    {
                        if (cancelToken.Token.IsCancellationRequested)
                            cancelToken.Token.ThrowIfCancellationRequested();
                        this.mp3Files.Add(new AudioFile(audioFile.FileName));
                    }

                    bool chapterStructure = false;
                    int i = 0;
                    int count = 0;
                    foreach (Chapter chapter in book.Chapters)
                    {
                        string[] titleParts = chapter.Title.Split(':');
                        string chapterTitle = titleParts[titleParts.Length - 1].Unidecode().RemoveChars('´', '`', '"', ''', ''', '"', '"', ';', ',', '،', '、', '″', '~', '*');

                        if (!chapterStructure)
                            i = 0;

                        for (; i < this.mp3Files.Count; i++)
                        {
                            string targetFile = Path.GetFileNameWithoutExtension(this.mp3Files[i].FileName).Unidecode();
                            chapterStructure = false;

                            if (Regex.IsMatch(targetFile, chapterTitle, RegexOptions.IgnoreCase))
                            {
                                chapterStructure = true;
                                count++;

                                foreach (Sentence sentence in chapter.Sentences)
                                    sentence.FirstAudioPosition.AudioFileIndex = i;

                                break;
                            }
                        }
                    }

                    if (chapterStructure)
                    {
                        this.structure = AudioBookStructure.CHAPTER_STRUCTURE;
                        EstimateChapterStructure();
                    }
                    else
                    {
                        this.structure = AudioBookStructure.MULTI_FILES_STRUCTURE;
                        EstimateNonChapterStructure();
                    }
                }
                catch (Exception)
                {
                    for (int i = 0; i < this.mp3Files.Count; i++)
                        this.mp3Files[i].Dispose(true);
                }
            }, cancelToken.Token);
        }

        /// <summary>
        /// Disposes the estimated timing instance.
        /// </summary>
        public void Dispose()
        {
            this.cancelToken.Cancel();
            this.stopwatch.Stop();
        }

        #endregion

        #region Private Helpers

        /// <summary>
        /// Estimating all the sentences from the specified book with chapter structure.
        /// </summary>
        private async void EstimateChapterStructure()
        {
            try
            {
                string timingsFile = MainWindow.TimingsFolder + @"\" + this.book.Checksum + ".xml";

                double totalAudioLength = 0;
                for (int i = 0; i < this.mp3Files.Count; i++)
                    totalAudioLength += this.mp3Files[i].TotalTime.TotalSeconds;

                long totalCharCount = 0;
                foreach (Chapter chapter in this.book.Chapters)
                    if (chapter.Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter)
                        totalCharCount += chapter.CharCount;

                double totalCharsPerSecond = (double)totalCharCount / totalAudioLength;

                foreach (Chapter chapter in book.Chapters)
                {
                    if (cancelToken.Token.IsCancellationRequested)
                        cancelToken.Token.ThrowIfCancellationRequested();

                    if (chapter.Sentences.Count < SpeechTimer.MiniumSentencesPerChapter || chapter.Sentences[0].FirstAudioPosition.AudioFileIndex == -1)
                        continue;

                    this.totalAmountOfSentences += chapter.Sentences.Count;
                    this.audioLength = this.mp3Files[chapter.Sentences[0].FirstAudioPosition.AudioFileIndex].TotalTime.TotalSeconds;
                    long charCount = chapter.CharCount;

                    this.charsPerSecond = (double)charCount / this.audioLength;

                    // Estimate position
                    EstimateChapter(chapter);

                    this.completedSentences = 0;
                    this.totalCompletedSentences += chapter.Sentences.Count;

                    if (this.UpdateProgress != null)
                        this.UpdateProgress(this, (100 / book.GetAmountOfSentencesWithAudioFiles()) * (completedSentences + totalCompletedSentences));

                    this.lastChapter = chapter;
                }

                if (this.Saving != null)
                {
                    this.Saving(this, EventArgs.Empty);
                    this.Saving(this, EventArgs.Empty);
                }

                List<string> audioHashes = new List<string>();
                await Task.Run(() =>
                {
                    try
                    {
                        using (MD5 md5 = MD5.Create())
                        {
                            foreach (AudioFile audioFile in this.mp3Files)
                            {
                                if (cancelToken.Token.IsCancellationRequested)
                                    cancelToken.Token.ThrowIfCancellationRequested();

                                audioHashes.Add(BitConverter.ToString(md5.ComputeHash(audioFile)));
                            }
                        }
                    }
                    catch (Exception) { }
                }, cancelToken.Token);

                if (cancelToken.Token.IsCancellationRequested)
                    cancelToken.Token.ThrowIfCancellationRequested();

                this.stopwatch.Stop();

                lock (xmlLock)
                {
                    XmlDocument doc = new XmlDocument();
                    XmlDeclaration dec = doc.CreateXmlDeclaration("1.0", "UTF-8", null);
                    doc.AppendChild(dec);

                    XmlElement root = doc.CreateElement("Books");
                    doc.AppendChild(root);

                    XmlElement bookElement = doc.CreateElement("Book");
                    bookElement.SetAttribute("CompletedTime", Convert.ToString(stopwatch.Elapsed));
                    bookElement.SetAttribute("Successrate", Convert.ToString(book.Successrate));
                    bookElement.SetAttribute("CharactersPerSecond", Convert.ToString(totalCharsPerSecond));
                    bookElement.SetAttribute("Volume", "0");

                    bool indexSet = false;
                    for (int i = 0; i < this.book.Chapters.Count; i++)
                    {
                        if (this.book.Chapters[i].Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter && this.book.Chapters[i].Sentences[0].FirstAudioPosition.AudioFileIndex >= 0)
                        {
                            bookElement.SetAttribute("CurrentAudioFileIndex", Convert.ToString(this.book.Chapters[i].Sentences[0].FirstAudioPosition.AudioFileIndex));
                            indexSet = true;
                            break;
                        }
                    }

                    if (!indexSet)
                        bookElement.SetAttribute("CurrentAudioFileIndex", "0");

                    bookElement.SetAttribute("CurrentAudioPosition", "00:00:00.0000000");
                    bookElement.SetAttribute("ScrollValue", "0");
                    root.AppendChild(bookElement);

                    XmlElement audioFilesElement = doc.CreateElement("AudioFiles");
                    for (int i = 0; i < this.mp3Files.Count; i++)
                    {
                        XmlElement audioFileElement = doc.CreateElement("AudioFile");
                        audioFileElement.SetAttribute("Path", this.mp3Files[i].FileName);
                        audioFileElement.SetAttribute("Checksum", audioHashes[i]);

                        audioFilesElement.AppendChild(audioFileElement);
                    }
                    bookElement.AppendChild(audioFilesElement);

                    XmlElement chaptersElement = doc.CreateElement("Chapters");
                    foreach (Chapter chapter in book.Chapters)
                    {
                        if (chapter.Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter)
                        {
                            XmlElement chapterElement = doc.CreateElement("Chapter");
                            chapterElement.SetAttribute("Successrate", Convert.ToString(chapter.Successrate));

                            foreach (Sentence sentence in chapter.Sentences)
                            {
                                XmlElement sentenceElement = doc.CreateElement("Sentence");

                                sentenceElement.SetAttribute("AudioFileIndex", Convert.ToString(sentence.FirstAudioPosition.AudioFileIndex));
                                sentenceElement.SetAttribute("AudioPosition", Convert.ToString(sentence.FirstAudioPosition.Position));
                                sentenceElement.SetAttribute("Duration", Convert.ToString(sentence.FirstAudioPosition.Duration));

                                chapterElement.AppendChild(sentenceElement);
                            }

                            chaptersElement.AppendChild(chapterElement);
                        }
                    }

                    bookElement.AppendChild(chaptersElement);

                    doc.Save(timingsFile);
                }

                if (this.Completed != null)
                    this.Completed(this, book);
            }
            catch (Exception)
            {
                for (int i = 0; i < this.mp3Files.Count; i++)
                    this.mp3Files[i].Dispose(true);
            }
        }

        /// <summary>
        /// Estimating all the sentences from the specified book with a non chapter structure.
        /// </summary>
        private async void EstimateNonChapterStructure()
        {
            try
            {
                string timingsFile = MainWindow.TimingsFolder + @"\" + this.book.Checksum + ".xml";

                for (int i = 0; i < this.mp3Files.Count; i++)
                    this.audioLength += this.mp3Files[i].TotalTime.TotalSeconds;

                long charCount = 0;
                foreach (Chapter chapter in this.book.Chapters)
                    if (chapter.Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter)
                        charCount += chapter.CharCount;

                this.charsPerSecond = (double)charCount / this.audioLength;

                foreach (Chapter chapter in book.Chapters)
                {
                    if (cancelToken.Token.IsCancellationRequested)
                        cancelToken.Token.ThrowIfCancellationRequested();

                    if (chapter.Sentences.Count < SpeechTimer.MiniumSentencesPerChapter)
                        continue;

                    this.totalAmountOfSentences += chapter.Sentences.Count;

                    // Estimate position
                    EstimateChapter(chapter);

                    this.completedSentences = 0;
                    this.totalCompletedSentences += chapter.Sentences.Count;

                    if (this.UpdateProgress != null)
                        this.UpdateProgress(this, (100 / book.GetAmountOfSentencesWithAudioFiles()) * (completedSentences + totalCompletedSentences));

                    this.lastChapter = chapter;
                }

                if (this.Saving != null)
                {
                    this.Saving(this, EventArgs.Empty);
                    this.Saving(this, EventArgs.Empty);
                }

                List<string> audioHashes = new List<string>();
                await Task.Run(() =>
                {
                    try
                    {
                        using (MD5 md5 = MD5.Create())
                        {
                            foreach (AudioFile audioFile in this.mp3Files)
                            {
                                if (cancelToken.Token.IsCancellationRequested)
                                    cancelToken.Token.ThrowIfCancellationRequested();

                                audioHashes.Add(BitConverter.ToString(md5.ComputeHash(audioFile)));
                            }
                        }
                    }
                    catch (Exception) { }
                }, cancelToken.Token);

                if (cancelToken.Token.IsCancellationRequested)
                    cancelToken.Token.ThrowIfCancellationRequested();

                this.stopwatch.Stop();

                lock (xmlLock)
                {
                    XmlDocument doc = new XmlDocument();
                    XmlDeclaration dec = doc.CreateXmlDeclaration("1.0", "UTF-8", null);
                    doc.AppendChild(dec);

                    XmlElement root = doc.CreateElement("Books");
                    doc.AppendChild(root);

                    XmlElement bookElement = doc.CreateElement("Book");
                    bookElement.SetAttribute("CompletedTime", Convert.ToString(stopwatch.Elapsed));
                    bookElement.SetAttribute("Successrate", Convert.ToString(book.Successrate));
                    bookElement.SetAttribute("CharactersPerSecond", Convert.ToString(this.charsPerSecond));
                    bookElement.SetAttribute("Volume", "0");

                    bool indexSet = false;
                    for (int i = 0; i < this.book.Chapters.Count; i++)
                    {
                        if (this.book.Chapters[i].Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter && this.book.Chapters[i].Sentences[0].FirstAudioPosition.AudioFileIndex >= 0)
                        {
                            bookElement.SetAttribute("CurrentAudioFileIndex", Convert.ToString(this.book.Chapters[i].Sentences[0].FirstAudioPosition.AudioFileIndex));
                            indexSet = true;
                            break;
                        }
                    }

                    if (!indexSet)
                        bookElement.SetAttribute("CurrentAudioFileIndex", "0");

                    bookElement.SetAttribute("CurrentAudioPosition", "00:00:00.0000000");
                    bookElement.SetAttribute("ScrollValue", "0");
                    root.AppendChild(bookElement);

                    XmlElement audioFilesElement = doc.CreateElement("AudioFiles");
                    for (int i = 0; i < this.mp3Files.Count; i++)
                    {
                        XmlElement audioFileElement = doc.CreateElement("AudioFile");
                        audioFileElement.SetAttribute("Path", this.mp3Files[i].FileName);
                        audioFileElement.SetAttribute("Checksum", audioHashes[i]);

                        audioFilesElement.AppendChild(audioFileElement);
                    }

                    bookElement.AppendChild(audioFilesElement);

                    XmlElement chaptersElement = doc.CreateElement("Chapters");
                    foreach (Chapter chapter in book.Chapters)
                    {
                        if (chapter.Sentences.Count >= SpeechTimer.MiniumSentencesPerChapter)
                        {
                            XmlElement chapterElement = doc.CreateElement("Chapter");
                            chapterElement.SetAttribute("Successrate", Convert.ToString(chapter.Successrate));
                            foreach (Sentence sentence in chapter.Sentences)
                            {
                                XmlElement sentenceElement = doc.CreateElement("Sentence");

                                sentenceElement.SetAttribute("AudioFileIndex", Convert.ToString(sentence.FirstAudioPosition.AudioFileIndex));
                                sentenceElement.SetAttribute("AudioPosition", Convert.ToString(sentence.FirstAudioPosition.Position));
                                sentenceElement.SetAttribute("Duration", Convert.ToString(sentence.FirstAudioPosition.Duration));

                                chapterElement.AppendChild(sentenceElement);
                            }

                            chaptersElement.AppendChild(chapterElement);
                        }
                    }

                    bookElement.AppendChild(chaptersElement);

                    doc.Save(timingsFile);
                }

                if (this.Saving != null)
                    this.Saving(this, EventArgs.Empty);

                if (this.Completed != null)
                    this.Completed(this, book);
            }
            catch (Exception)
            {
                for (int i = 0; i < this.mp3Files.Count; i++)
                    this.mp3Files[i].Dispose();
            }
        }

        /// <summary>
        /// Estimates a chapter.
        /// </summary>
        /// <param name="chapter">The chapter to estimate</param>
        private void EstimateChapter(Chapter chapter)
        {
            int i = 0;
            foreach (Sentence sentence in chapter.Sentences)
            {
                if (cancelToken.Token.IsCancellationRequested)
                    break;

                sentence.FirstAudioPosition.Position = TimeSpan.FromSeconds(EstimatePositionInSeconds(i, chapter));
                sentence.FirstAudioPosition.Duration = TimeSpan.FromSeconds(sentence.CharCount / this.charsPerSecond);

                if (this.structure != AudioBookStructure.CHAPTER_STRUCTURE)
                {
                    TimeSpan time = sentence.FirstAudioPosition.Position;

                    int audioFileIndex = 0;
                    for (audioFileIndex = 0; audioFileIndex < this.mp3Files.Count; audioFileIndex++)
                    {
                        time -= this.mp3Files[audioFileIndex].TotalTime;

                        if (time < TimeSpan.Zero)
                            break;
                    }

                    if (this.mp3Files.Count > audioFileIndex)
                        sentence.FirstAudioPosition.AudioFileIndex = audioFileIndex;
                    else
                        sentence.FirstAudioPosition.AudioFileIndex = this.mp3Files.Count - 1;
                }
                i++;
            }
        }

        /// <summary>
        /// Estimates the position in seconds.
        /// </summary>
        /// <param name="sentenceIndex">The sentence index in the chapter.</param>
        /// <param name="chapter">The chapter.</param>
        /// <param name="offset">The offset to add.</param>
        /// <returns>Returns the position in seconds.</returns>
        private double EstimatePositionInSeconds(int sentenceIndex, Chapter chapter)
        {
            double charCount = 0;
            for (int i = 0; i < sentenceIndex; i++)
                charCount += chapter.Sentences[i].CharCount;

            if (lastChapter == null)
                return (charCount / this.charsPerSecond);
            else
            {
                if (this.structure == AudioBookStructure.CHAPTER_STRUCTURE)
                {
                    double totalSeconds = 0;
                    for (int i = 0; i < chapter.Sentences[0].FirstAudioPosition.AudioFileIndex; i++)
                        totalSeconds += this.mp3Files[i].TotalTime.TotalSeconds;

                    return (charCount / this.charsPerSecond) + totalSeconds;
                }
                else
                {
                    Sentence lastSentence = lastChapter.GetLastSentenceWithValues();
                    if (lastSentence == null)
                        return (charCount / this.charsPerSecond);
                    else
                        return (charCount / this.charsPerSecond) + lastSentence.FirstAudioPosition.TotalSeconds();
                }
            }
        }

        /// <summary>
        /// Helper for writing debug messages.
        /// </summary>
        private void DebugWriteLine(Chapter chapter, string header)
        {
            System.Diagnostics.Debug.WriteLine("###################" + header + "#####################");

            System.Diagnostics.Debug.WriteLine("ID # Position ### Duration ### Text");

            for (int i = 0; i < chapter.Sentences.Count; i++)
            {
                System.Diagnostics.Debug.Write(i);
                System.Diagnostics.Debug.Write(" - ");
                System.Diagnostics.Debug.Write(chapter.Sentences[i].FirstAudioPosition.Position);
                System.Diagnostics.Debug.Write(" - ");
                System.Diagnostics.Debug.Write(chapter.Sentences[i].FirstAudioPosition.Duration);
                System.Diagnostics.Debug.Write(" - ");
                System.Diagnostics.Debug.WriteLine(chapter.Sentences[i].Text);
            }

            System.Diagnostics.Debug.WriteLine("########################################");
        }

        #endregion
    }
}


text:
require 'elasticsearch'
require 'faraday/adapter/libuv'

class Elastic
    class Query
        def initialize(params = nil)
            query = params ? params.permit(:q, :limit, :offset) : {}

            @filters = nil
            @search = "#{query[:q]}*"
            @fields = ['_all'.freeze]

            @limit = query[:limit] || 20
            @limit = @limit.to_i
            @limit = 10000 if @limit > 10000

            @offset = query[:offset] || 0
            @offset = offset.to_i
            @offset = 10000 if offset > 10000
        end

        attr_accessor :offset, :limit, :sort, :fields, :query_settings

        def raw_filter(filter)
            @raw_filter ||= []
            @raw_filter << filter
            self
        end

        def search_field(field)
            @fields.unshift(field)
        end

        # filters is in the form {fieldname1: ['var1','var2',...], fieldname2: ['var1','var2'...]}
        # NOTE:: may overwrite an existing filter in merge
        def filter(filters)
            @filters ||= {}
            @filters.merge!(filters)
            self
        end

        # Like filter however all keys are OR's instead of AND's
        def or_filter(filters)
            @orFilter ||= {}
            @orFilter.merge!(filters)
            self
        end

        def and_filter(filters)
            @andFilter ||= {}
            @andFilter.merge!(filters)
            self
        end

        # Applys the query to child objects
        def has_child(name)
            @hasChild = name
        end

        def has_parent(name)
            @hasParent = name
        end

        def range(filter)
            @rangeFilter ||= []
            @rangeFilter << filter
            self
        end

        # Call to add fields that should be missing
        # Effectively adds a filter that ensures a field is missing
        def missing(*fields)
            @missing ||= Set.new
            @missing.merge(fields)
            self
        end

        # The opposite of filter
        def not(filters)
            @nots ||= {}
            @nots.merge!(filters)
            self
        end

        def exists(*fields)
            @exists ||= Set.new
            @exists.merge(fields)
            self
        end

        def build
            if @filters
                fieldfilters = []

                @filters.each do |key, value|
                    fieldfilter = { :or => [] }
                    build_filter(fieldfilter[:or], key, value)

                    # TODO:: Discuss this - might be a security issue
                    unless fieldfilter[:or].empty?
                        fieldfilters.push(fieldfilter)
                    end
                end
            end

            if @orFilter
                fieldfilters ||= []
                fieldfilter = { :or => [] }
                orArray = fieldfilter[:or]

                @orFilter.each do |key, value|
                    build_filter(orArray, key, value)
                end

                unless orArray.empty?
                    fieldfilters.push(fieldfilter)
                end
            end

            if @andFilter
                fieldfilters ||= []
                fieldfilter = { :and => [] }
                andArray = fieldfilter[:and]

                @andFilter.each do |key, value|
                    build_filter(andArray, key, value)
                end

                unless andArray.empty?
                    fieldfilters.push(fieldfilter)
                end
            end

            if @rangeFilter
                fieldfilters ||= []

                @rangeFilter.each do |value|
                    fieldfilters.push({range: value})
                end
            end

            if @nots
                fieldfilters ||= []

                @nots.each do |key, value|
                    fieldfilter = { not: { filter: { or: [] } } }
                    build_filter(fieldfilter[:not][:filter][:or], key, value)
                    unless fieldfilter[:not][:filter][:or].empty?
                        fieldfilters.push(fieldfilter)
                    end
                end
            end

            if @missing
                fieldfilters ||= []

                @missing.each do |field|
                    fieldfilters.push({
                        missing: { field: field }
                    })
                end
            end

            if @exists
                fieldfilters ||= []

                @exists.each do |field|
                    fieldfilters.push({
                        exists: { field: field }
                    })
                end
            end

            if @raw_filter
                fieldfilters ||= []
                fieldfilters += @raw_filter
            end

            if @search.length > 1
                # Break the terms up purely on whitespace
                query_obj = nil

                if @hasChild || @hasParent
                    should = [{
                            simple_query_string: {
                                query: @search,
                                fields: @fields
                            }
                        }]

                    if @query_settings
                        should[0][:simple_query_string].merge! @query_settings
                    end

                    if @hasChild
                        should << {
                                has_child: {
                                    type: @hasChild,
                                    query: {
                                        simple_query_string: {
                                            query: @search,
                                            fields: @fields
                                        }
                                    }
                                }
                            }
                    end

                    if @hasParent
                        should << {
                                has_parent: {
                                    parent_type: @hasParent,
                                    query: {
                                        simple_query_string: {
                                            query: @search,
                                            fields: @fields
                                        }
                                    }
                                }
                            }
                    end

                    query_obj = {
                        query: {
                            bool: {
                                should: should
                            }
                        },
                        filters: fieldfilters,
                        offset: @offset,
                        limit: @limit
                    }
                else
                    query_obj = {
                        query: {
                            simple_query_string: {
                                query: @search,
                                fields: @fields
                            }
                        },
                        filters: fieldfilters,
                        offset: @offset,
                        limit: @limit
                    }
                end

                query_obj
            else
                {
                    sort: @sort || DEFAULT_SORT,
                    filters: fieldfilters,
                    query: {
                        match_all: {}
                    },
                    offset: @offset,
                    limit: @limit
                }
            end
        end

        DEFAULT_SORT = [{
            'doc.created_at' => {
                order: :desc
            }
        }]

        #protected

        def build_filter(filters, key, values)
            values.each { |var|
                if var.nil?
                    filters.push({
                        missing: { field: key }
                    })
                else
                    filters.push({
                        :term => {
                            key => var
                        }
                    })
                end
            }
        end
    end

    HOST = if ENV['ELASTIC']
        ENV['ELASTIC'].split(' ').map {|item| "#{item}:9200"}
    else
        ['localhost:9200']
    end

    @@client ||= Elasticsearch::Client.new hosts: HOST, reload_connections: true, adapter: :libuv
    def self.search *args
        @@client.search *args
    end

    def self.count *args
        @@client.count *args
    end

    def self.client
        @@client
    end

    def self.new_client
      @@client = Elasticsearch::Client.new hosts: HOST, reload_connections: true, adapter: :libuv
    end

    COUNT = 'count'.freeze
    HITS = 'hits'.freeze
    TOTAL = 'total'.freeze
    ID = '_id'.freeze
    SCORE = ['_score'.freeze]
    INDEX = (ENV['ELASTIC_INDEX'] || 'default').freeze

    def initialize(klass = nil, **opts)
        if klass
            @klass = klass
            @filter = klass.design_document
        end
        @index = opts[:index] || INDEX
        @use_couch_type = opts[:use_couch_type] || false
    end

    # Safely build the query
    def query(params = nil, filters = nil)
        builder = ::Elastic::Query.new(params)
        builder.filter(filters) if filters
        builder
    end

    def search(builder, &block)
        query = generate_body(builder)

        # if a formatter block is supplied, each loaded record is passed to it
        # allowing annotation/conversion of records using data from the model
        # and current request (e.g groups are annotated with 'admin' if the
        # currently logged in user is an admin of the group). nils are removed
        # from the list.
        result = Elastic.search(query)
        records = if @klass
            Array(@klass.find_by_id(result[HITS][HITS].map {|entry| entry[ID]}))
        else
            Array(result[HITS][HITS].map { |entry| ::CouchbaseOrm.try_load(entry[ID]) }).compact
        end
        results = block_given? ? (records.map {|record| yield record}).compact : records

        # Ensure the total is accurate
        total = result[HITS][TOTAL] || 0
        total = total - (records.length - results.length) # adjust for compaction

        # We check records against limit (pre-compaction) and total against actual result length
        # Worst case senario is there is one additional request for results at an offset that returns no results.
        # The total results number will be accurate on the final page of results from the clients perspective.
        total = results.length + builder.offset if records.length < builder.limit && total > (results.length + builder.offset)
        {
            total: total,
            results: results
        }
    end

    def count(builder)
        query = generate_body(builder)

        # Simplify the query
        query[:body].delete(:from)
        query[:body].delete(:size)
        query[:body].delete(:sort)

        Elastic.count(query)[COUNT]
    end

    def generate_body(builder)
        opt = builder.build

        sort = (opt[:sort] || []) + SCORE

        queries = opt[:queries] || []
        queries.unshift(opt[:query])

        filters = opt[:filters] || []

        if @filter
            if @use_couch_type
                filters.unshift({term: {'doc.type' => @filter}})
            else
                filters.unshift({type: {value: @filter}})
            end
        end

        {
            index: @index,
            body: {
                sort: sort,
                query: {
                    bool: {
                        must: {
                            query: {
                                bool: {
                                    must: queries
                                }
                            }
                        },
                        filter: {
                            bool: {
                                must: filters
                            }
                        }
                    }
                },
                from: opt[:offset],
                size: opt[:limit]
            }
        }
    end
end

Thread.new do
  loop do
    sleep 600
    begin
      Elastic.new_client
    rescue => e
      puts "failed to create new elastic client: #{e.message}"
    end
  end
end