text:
import os
import time
import shutil
import unittest2 as unittest
import logging
import logging.config
logging.config.fileConfig("pyspider/logging.conf")

from pyspider.scheduler.task_queue import TaskQueue

class TestTaskQueue(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        self.task_queue = TaskQueue()
        self.task_queue.rate = 100000
        self.task_queue.burst = 100000
        self.task_queue.processing_timeout = 0.5

    def test_10_put(self):
        self.task_queue.put('a3', 0, time.time() + 0.5)
        self.task_queue.put('a4', 3, time.time() + 0.2)
        self.task_queue.put('a2', 0)
        self.task_queue.put('a1', 1)
        self.assertEqual(self.task_queue.size(), 4)

    def test_20_update(self):
        self.task_queue.put('a2', 4)
        self.assertEqual(self.task_queue.size(), 4)
        self.task_queue.put('a3', 2, 0)
        self.assertEqual(self.task_queue.size(), 4)

    def test_30_get_from_priority_queue(self):
        self.assertEqual(self.task_queue.get(), 'a2')
        self.assertEqual(self.task_queue.size(), 4)

    def test_40_time_queue_1(self):
        self.task_queue.check_update()
        self.assertEqual(self.task_queue.get(), 'a3')
        self.assertEqual(self.task_queue.size(), 4)

    def test_50_time_queue_2(self):
        time.sleep(0.3)
        self.task_queue.check_update()
        self.assertEqual(self.task_queue.get(), 'a4')
        self.assertEqual(self.task_queue.get(), 'a1')
        self.assertEqual(self.task_queue.size(), 4)

    def test_60_processing_queue(self):
        time.sleep(0.5)
        self.task_queue.check_update()
        self.assertEqual(self.task_queue.get(), 'a2')
        self.assertEqual(len(self.task_queue), 4)
        self.assertEqual(self.task_queue.get(), 'a4')
        self.assertEqual(self.task_queue.get(), 'a3')
        self.assertEqual(self.task_queue.get(), 'a1')
        self.assertEqual(len(self.task_queue), 4)

    def test_70_done(self):
        self.assertTrue(self.task_queue.done('a2'))
        self.assertTrue(self.task_queue.done('a1'))
        self.assertEqual(len(self.task_queue), 2)
        self.assertTrue(self.task_queue.done('a4'))
        self.assertTrue(self.task_queue.done('a3'))
        self.assertEqual(len(self.task_queue), 0)

from pyspider.scheduler.token_bucket import Bucket

class TestBucket(unittest.TestCase):

    def test_bucket(self):
        bucket = Bucket(100, 1000)
        self.assertEqual(bucket.get(), 1000)
        time.sleep(0.1)
        self.assertEqual(bucket.get(), 1000)
        bucket.desc(100)
        self.assertEqual(bucket.get(), 900)
        time.sleep(0.1)
        self.assertAlmostEqual(bucket.get(), 910, delta=2)
        time.sleep(0.1)
        self.assertAlmostEqual(bucket.get(), 920, delta=2)

try:
    from six.moves import xmlrpc_client
except ImportError:
    import xmlrpclib as xmlrpc_client
from multiprocessing import Queue
from pyspider.scheduler.scheduler import Scheduler
from pyspider.database.sqlite import taskdb, projectdb, resultdb
from pyspider.libs.utils import run_in_thread

class TestScheduler(unittest.TestCase):
    taskdb_path = './data/tests/task.db'
    projectdb_path = './data/tests/project.db'
    resultdb_path = './data/tests/result.db'
    check_project_time = 1
    scheduler_xmlrpc_port = 23333

    @classmethod
    def setUpClass(self):
        shutil.rmtree('./data/tests', ignore_errors=True)
        os.makedirs('./data/tests')

        def get_taskdb():
            return taskdb.TaskDB(self.taskdb_path)
        self.taskdb = get_taskdb()

        def get_projectdb():
            return projectdb.ProjectDB(self.projectdb_path)
        self.projectdb = get_projectdb()

        def get_resultdb():
            return resultdb.ResultDB(self.resultdb_path)
        self.resultdb = get_resultdb()

        self.newtask_queue = Queue(10)
        self.status_queue = Queue(10)
        self.scheduler2fetcher = Queue(10)
        self.rpc = xmlrpc_client.ServerProxy('http://localhost:%d' % self.scheduler_xmlrpc_port)

        def run_scheduler():
            scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(),
                                  newtask_queue=self.newtask_queue, status_queue=self.status_queue,
                                  out_queue=self.scheduler2fetcher, data_path="./data/tests/",
                                  resultdb=get_resultdb())
            scheduler.UPDATE_PROJECT_INTERVAL = 0.1
            scheduler.LOOP_INTERVAL = 0.1
            scheduler.INQUEUE_LIMIT = 10
            Scheduler.DELETE_TIME = 0
            scheduler._last_tick = int(time.time())  # not dispatch cronjob
            run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
            scheduler.run()

        self.process = run_in_thread(run_scheduler)
        time.sleep(1)

    @classmethod
    def tearDownClass(self):
        if self.process.is_alive():
            self.rpc._quit()
            self.process.join(5)
        assert not self.process.is_alive()
        shutil.rmtree('./data/tests', ignore_errors=True)
        time.sleep(1)

    def test_10_new_task_ignore(self):
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url'
        })
        self.assertEqual(self.rpc.size(), 0)
        self.assertEqual(len(self.rpc.get_active_tasks()), 0)

    def test_20_new_project(self):
        self.projectdb.insert('test_project', {
            'name': 'test_project',
            'group': 'group',
            'status': 'TODO',
            'script': 'import time\nprint(time.time())',
            'comments': 'test project',
            'rate': 1.0,
            'burst': 10,
        })

    def test_30_update_project(self):
        from six.moves import queue as Queue
        with self.assertRaises(Queue.Empty):
            task = self.scheduler2fetcher.get(timeout=1)
        self.projectdb.update('test_project', status="DEBUG")
        time.sleep(0.1)
        self.rpc.update_project()

        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)
        self.assertEqual(task['url'], 'data:,_on_get_info')

    def test_35_new_task(self):
        time.sleep(0.2)
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'age': 0,
            },
        })

        time.sleep(0.5)
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertGreater(len(self.rpc.get_active_tasks()), 0)
        self.assertIsNotNone(task)
        self.assertEqual(task['project'], 'test_project')
        self.assertIn('schedule', task)
        self.assertIn('fetch', task)
        self.assertIn('process', task)
        self.assertIn('track', task)
        self.assertEqual(task['fetch']['data'], 'abc')

    def test_37_force_update_processing_task(self):
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url_force_update',
            'schedule': {
                'age': 0,
                'force_update': True,
            },
        })
        time.sleep(0.2)
        # it should not block next

    def test_40_taskdone_error_no_project(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'no_project',
            'url': 'url'
        })
        time.sleep(0.1)
        self.assertEqual(self.rpc.size(), 1)

    def test_50_taskdone_error_no_track(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url'
        })
        time.sleep(0.1)
        self.assertEqual(self.rpc.size(), 1)
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'track': {}
        })
        time.sleep(0.1)
        self.assertEqual(self.rpc.size(), 1)

    def test_60_taskdone_failed_retry(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'track': {
                'fetch': {
                    'ok': True
                },
                'process': {
                    'ok': False
                },
            }
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

    def test_70_taskdone_ok(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'track': {
                'fetch': {
                    'ok': True
                },
                'process': {
                    'ok': True
                },
            }
        })
        time.sleep(0.2)
        self.assertEqual(self.rpc.size(), 0)

    def test_80_newtask_age_ignore(self):
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'age': 30,
            },
        })
        time.sleep(0.1)
        self.assertEqual(self.rpc.size(), 0)

    def test_82_newtask_via_rpc(self):
        self.rpc.newtask({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'age': 30,
            },
        })
        time.sleep(0.1)
        self.assertEqual(self.rpc.size(), 0)

    def test_90_newtask_with_itag(self):
        time.sleep(0.1)
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'itag': "abc",
                'retries': 1
            },
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

        self.test_70_taskdone_ok()

    def test_a10_newtask_restart_by_age(self):
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'age': 0,
                'retries': 1
            },
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

    def test_a20_failed_retry(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'track': {
                'fetch': {
                    'ok': True
                },
                'process': {
                    'ok': False
                },
            }
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'track': {
                'fetch': {
                    'ok': False
                },
                'process': {
                    'ok': False
                },
            }
        })

        from six.moves import queue as Queue
        with self.assertRaises(Queue.Empty):
            self.scheduler2fetcher.get(timeout=5)

    def test_a30_task_verify(self):
        self.assertFalse(self.rpc.newtask({
            #'taskid': 'taskid#',
            'project': 'test_project',
            'url': 'url',
        }))
        self.assertFalse(self.rpc.newtask({
            'taskid': 'taskid#',
            #'project': 'test_project',
            'url': 'url',
        }))
        self.assertFalse(self.rpc.newtask({
            'taskid': 'taskid#',
            'project': 'test_project',
            #'url': 'url',
        }))
        self.assertFalse(self.rpc.newtask({
            'taskid': 'taskid#',
            'project': 'not_exist_project',
            'url': 'url',
        }))
        self.assertTrue(self.rpc.newtask({
            'taskid': 'taskid#',
            'project': 'test_project',
            'url': 'url',
        }))

    def test_a40_success_recrawl(self):
        self.newtask_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'fetch': {
                'data': 'abc',
            },
            'process': {
                'data': 'abc',
            },
            'schedule': {
                'age': 0,
                'retries': 1,
                'auto_recrawl': True,
            },
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'schedule': {
                'age': 0,
                'retries': 1,
                'auto_recrawl': True,
            },
            'track': {
                'fetch': {
                    'ok': True
                },
                'process': {
                    'ok': True
                },
            }
        })
        task = self.scheduler2fetcher.get(timeout=10)
        self.assertIsNotNone(task)

    def test_a50_failed_recrawl(self):
        for i in range(3):
            self.status_queue.put({
                'taskid': 'taskid',
                'project': 'test_project',
                'url': 'url',
                'schedule': {
                    'age': 0,
                    'retries': 1,
                    'auto_recrawl': True,
                },
                'track': {
                    'fetch': {
                        'ok': True
                    },
                    'process': {
                        'ok': False
                    },
                }
            })
            task = self.scheduler2fetcher.get(timeout=10)
            self.assertIsNotNone(task)

    def test_a60_disable_recrawl(self):
        self.status_queue.put({
            'taskid': 'taskid',
            'project': 'test_project',
            'url': 'url',
            'schedule': {
                'age': 0,
                'retries': 1,
            },
            'track': {
                'fetch': {
                    'ok': True
                },
                'process': {
                    'ok': True
                },
            }
        })

        from six.moves import queue as Queue
        with self.assertRaises(Queue.Empty):
            self.scheduler2fetcher.get(timeout=5)

    def test_x10_inqueue_limit(self):
        self.projectdb.insert('test_inqueue_project', {
            'name': 'test_inqueue_project',
            'group': 'group',
            'status': 'DEBUG',
            'script': 'import time\nprint(time.time())',
            'comments': 'test project',
            'rate': 0,
            'burst': 0,
        })
        time.sleep(0.1)
        pre_size = self.rpc.size()
        for i in range(20):
            self.newtask_queue.put({
                'taskid': 'taskid%d' % i,
                'project': 'test_inqueue_project',
                'url': 'url',
                'schedule': {
                    'age': 3000,
                    'force_update': True,
                },
            })
        time.sleep(1)
        self.assertEqual(self.rpc.size() - pre_size, 10)

    def test_x20_delete_project(self):
        self.assertIsNotNone(self.projectdb.get('test_inqueue_project'))
        #self.assertIsNotNone(self.taskdb.get_task('test_inqueue_project', 'taskid1'))
        self.projectdb.update('test_inqueue_project', status="STOP", group="lock,delete")
        time.sleep(1)
        self.assertIsNone(self.projectdb.get('test_inqueue_project'))
        self.taskdb._list_project()
        self.assertIsNone(self.taskdb.get_task('test_inqueue_project', 'taskid1'))

    def test_z10_startup(self):
        self.assertTrue(self.process.is_alive())

    def test_z20_quit(self):
        self.rpc._quit()
        time.sleep(0.2)
        self.assertFalse(self.process.is_alive())
        self.assertEqual(
            self.taskdb.get_task('test_project', 'taskid')['status'],
            self.taskdb.SUCCESS
        )

if __name__ == '__main__':
    unittest.main()


text:
![Breakthrough Men's Community](https://raw.github.com/breakthroughmenscommunity/bmc/267df30366c7ca48cb5b7a76bced98a7e30e5aeb/app/assets/images/bmc_banner.png)

**Website:** http://breakthroughmenscommunity.org

[![Build Status](https://semaphoreapp.com/api/v1/projects/06b2f2ed-9fdd-4a8d-9bd0-8d8851eeac82/233062/shields_badge.png)](https://semaphoreapp.com/shillcock/bmc) [![Coverage Status](https://coveralls.io/repos/breakthroughmenscommunity/bmc/badge.png)](https://coveralls.io/r/breakthroughmenscommunity/bmc) [![Code Climate](https://codeclimate.com/github/breakthroughmenscommunity/bmc.png)](https://codeclimate.com/github/breakthroughmenscommunity/bmc)

## README

This README would normally document whatever steps are necessary to get the
application up and running.

Things you may want to cover:

* Ruby version

* System dependencies

* Configuration

* Database creation

* Database initialization

* How to run the test suite

* Services (job queues, cache servers, search engines, etc.)

* Deployment instructions

redpajama_set_name: RedPajamaGithub


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using DependencyInjector.Constants;
using NUnit.Framework;

namespace DependencyInjector.Tests.Constants
{
    [TestFixture]
    public class BuildInfoTests
    {
        // SetUp & Mocks
        [SetUp]
        public void SetUp()
        {

        }

        [TearDown]
        public void TearDown()
        {

        }

        // Tests
        [Test]
        public void Version_ReturnsAssemblyVersionOfCoreAssembly()
        {
            string expected = Assembly.GetAssembly(typeof(Contract)).GetName().Version.ToString();

            string result = BuildInfo.Version;

            Assert.AreEqual(expected, result);
        }

#if DEBUG
        [Test]
        public void IsDebug_ReturnsTrue()
        {
            Assert.IsTrue(BuildInfo.IsDebug);
        }
#else
        [Test]
        public void IsDebug_ReturnsFalse()
        {
            Assert.IsFalse(BuildInfo.IsDebug);
        }
#endif

    }
}


text:
NSString * const kgmtTimeZome = @"gmtTimeZome";

@implementation NSDate (display)

static NSDateFormatter *dateDisplayFormatter;

- (NSString *) formattedDisplayDate {

    if (dateDisplayFormatter == nil) {
        dateDisplayFormatter = [[NSDateFormatter alloc] init];
        dateDisplayFormatter.dateFormat = @"yyyy-MM-dd HH:mm zzz";
    }
    [self setTimeZoneForDateFormatter:dateDisplayFormatter];

    return [dateDisplayFormatter stringFromDate:self];
}

- (NSString *) formattedDisplayDateWithDateStyle: (NSDateFormatterStyle) dateStyle andTimeStyle: (NSDateFormatterStyle) timeStyle {
    NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init];
    dateFormatter.dateStyle = dateStyle;
    dateFormatter.timeStyle = timeStyle;
    [self setTimeZoneForDateFormatter:dateFormatter];
    return [dateFormatter stringFromDate:self];
}

- (void) setTimeZoneForDateFormatter: (NSDateFormatter *) dateFormatter {
    BOOL gmtTimeZone = [NSDate isDisplayGMT];
    if (!gmtTimeZone) {
        [dateFormatter setTimeZone:[NSTimeZone systemTimeZone]];
    } else {
        [dateFormatter setTimeZone:[NSTimeZone timeZoneForSecondsFromGMT:0]];
    }
}

+ (BOOL) isDisplayGMT {
    NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];
    return [defaults boolForKey:kgmtTimeZome];
}

+ (void) setDisplayGMT: (BOOL) gmt {
    NSUserDefaults *defaults = [NSUserDefaults standardUserDefaults];
    [defaults setBool:gmt forKey:kgmtTimeZome];
    [defaults synchronize];
}

@end


text:
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="@drawable/bg"
    android:paddingBottom="@dimen/activity_vertical_margin"
    android:paddingLeft="@dimen/activity_horizontal_margin"
    android:paddingRight="@dimen/activity_horizontal_margin"
    android:paddingTop="@dimen/activity_vertical_margin"
    tools:context="com.asim.demodoughnutchartwithtext.MainActivity" >

    <TextView
        android:id="@+id/txtDoughnut"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_centerHorizontal="true"
        android:text="Doughnut Chart"
        android:textColor="#ffffff"
        android:textSize="30dp" />

    <CheckBox
        android:id="@+id/chkplayDoughnut"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_below="@+id/txtDoughnut"
        android:layout_centerHorizontal="true"
        android:text="Play Doughnut"
        android:textColor="#ffffff" />

    <RelativeLayout
        android:id="@+id/doughnutchartmain"
        android:layout_width="250dp"
        android:layout_height="250dp"

        android:layout_below="@+id/chkplayDoughnut" >

        <LinearLayout
            android:id="@+id/doughnutchart"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:orientation="vertical" >
        </LinearLayout>

        <TextView
            android:id="@+id/tvdonutcenterlayout"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_centerHorizontal="true"
            android:layout_centerVertical="true"
            android:text="0%"
            android:textColor="#ffffff"
            android:textSize="20sp" />
    </RelativeLayout>

    <SeekBar
        android:id="@+id/control_bar"
        android:layout_width="fill_parent"
        android:layout_height="wrap_content"
        android:layout_below="@+id/doughnutchartmain"
        android:layout_margin="10dp"
        android:max="100"
        android:progress="0" />

</RelativeLayout>


text:
package stroom.query.common.v2;

import stroom.dashboard.expression.v1.ChildData;
import stroom.dashboard.expression.v1.Expression;
import stroom.dashboard.expression.v1.FieldIndex;
import stroom.dashboard.expression.v1.Generator;
import stroom.dashboard.expression.v1.Val;
import stroom.dashboard.expression.v1.ValLong;
import stroom.dashboard.expression.v1.ValNull;
import stroom.dashboard.expression.v1.ValSerialiser;
import stroom.dashboard.expression.v1.ValString;
import stroom.lmdb.LmdbEnv;
import stroom.lmdb.LmdbEnv.BatchingWriteTxn;
import stroom.lmdb.LmdbEnvFactory;
import stroom.query.api.v2.QueryKey;
import stroom.query.api.v2.TableSettings;
import stroom.query.common.v2.SearchProgressLog.SearchPhase;
import stroom.util.concurrent.CompleteException;
import stroom.util.logging.LambdaLogger;
import stroom.util.logging.LambdaLoggerFactory;
import stroom.util.logging.Metrics;

import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.unsafe.UnsafeByteBufferInput;
import com.esotericsoftware.kryo.unsafe.UnsafeByteBufferOutput;
import org.lmdbjava.CursorIterable;
import org.lmdbjava.CursorIterable.KeyVal;
import org.lmdbjava.Dbi;
import org.lmdbjava.EnvFlags;
import org.lmdbjava.KeyRange;
import org.lmdbjava.PutFlags;
import org.lmdbjava.Txn;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Supplier;
import javax.inject.Provider;
import javax.validation.constraints.NotNull;

public class LmdbDataStore implements DataStore {

    private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(LmdbDataStore.class);

    private static final long COMMIT_FREQUENCY_MS = 1000;
    private final Key rootKey;

    private final LmdbEnv lmdbEnv;
    private final ResultStoreConfig resultStoreConfig;
    private final Dbi<ByteBuffer> dbi;

    private final CompiledField[] compiledFields;
    private final CompiledSorter<Item>[] compiledSorters;
    private final CompiledDepths compiledDepths;
    private final Sizes maxResults;
    private final AtomicLong totalResultCount = new AtomicLong();
    private final boolean limitResultCount;
    private final AtomicLong resultCount = new AtomicLong();

    private final AtomicBoolean hasEnoughData = new AtomicBoolean();
    private final AtomicBoolean shutdown = new AtomicBoolean();

    private final LmdbKVQueue queue;
    private final CountDownLatch complete = new CountDownLatch(1);
    private final CompletionState completionState = new CompletionStateImpl(this, complete);
    private final AtomicLong uniqueKey = new AtomicLong();
    private final QueryKey queryKey;
    private final String componentId;
    private final boolean producePayloads;
    private final ErrorConsumer errorConsumer;
    private final LmdbPayloadCreator payloadCreator;

    private final LmdbKey rootParentRowKey;
    private final TransferState transferState = new TransferState();

    private final Serialisers serialisers;

    LmdbDataStore(final Serialisers serialisers,
                  final LmdbEnvFactory lmdbEnvFactory,
                  final ResultStoreConfig resultStoreConfig,
                  final QueryKey queryKey,
                  final String componentId,
                  final TableSettings tableSettings,
                  final FieldIndex fieldIndex,
                  final Map<String, String> paramMap,
                  final Sizes maxResults,
                  final boolean producePayloads,
                  final Provider<Executor> executorProvider,
                  final ErrorConsumer errorConsumer) {
        this.serialisers = serialisers;
        this.resultStoreConfig = resultStoreConfig;
        this.maxResults = maxResults;
        this.queryKey = queryKey;
        this.componentId = componentId;
        this.producePayloads = producePayloads;
        this.errorConsumer = errorConsumer;

        queue = new LmdbKVQueue(resultStoreConfig.getValueQueueSize());
        compiledFields = CompiledFields.create(tableSettings.getFields(), fieldIndex, paramMap);
        compiledDepths = new CompiledDepths(compiledFields, tableSettings.showDetail());
        compiledSorters = CompiledSorter.create(compiledDepths.getMaxDepth(), compiledFields);

        payloadCreator = new LmdbPayloadCreator(serialisers, queryKey, this, compiledFields, resultStoreConfig);

        rootKey = Key.createRoot(serialisers);
        rootParentRowKey = new LmdbKey.Builder()
                .keyBytes(rootKey.getBytes())
                .build();

        final String uuid = (queryKey + "_" + componentId + "_" + UUID.randomUUID());
        // Make safe for the file system.
        final String dirName = uuid.replaceAll("[^A-Za-z0-9]", "_");
        this.lmdbEnv = lmdbEnvFactory.builder(resultStoreConfig.getLmdbConfig())
                .withMaxDbCount(1)
                .withSubDirectory(dirName)
                .addEnvFlag(EnvFlags.MDB_NOTLS)
                .build();
        this.dbi = lmdbEnv.openDbi(uuid);

        // Find out if we have any sorting.
        boolean hasSort = false;
        for (final CompiledSorter<Item> sorter : compiledSorters) {
            if (sorter != null) {
                hasSort = true;
                break;
            }
        }

        // Determine if we are going to limit the result count.
        limitResultCount = maxResults != null && !hasSort && !compiledDepths.hasGroup();

        // Start transfer loop.
        executorProvider.get().execute(this::transfer);
    }

    /**
     * Add some values to the data store.
     *
     * @param values The values to add to the store.
     */
    @Override
    public void add(final Val[] values) {
        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_ADD);
        LOGGER.trace(() -> "add() called for " + values.length + " values");
        final int[] groupSizeByDepth = compiledDepths.getGroupSizeByDepth();
        final boolean[][] groupIndicesByDepth = compiledDepths.getGroupIndicesByDepth();
        final boolean[][] valueIndicesByDepth = compiledDepths.getValueIndicesByDepth();

        Key key = rootKey;
        LmdbKey parentRowKey = rootParentRowKey;

        for (int depth = 0; depth < groupIndicesByDepth.length; depth++) {
            final LmdbKey.Builder rowKeyBuilder = new LmdbKey.Builder();
            final Generator[] generators = new Generator[compiledFields.length];

            final int groupSize = groupSizeByDepth[depth];
            final boolean[] groupIndices = groupIndicesByDepth[depth];
            final boolean[] valueIndices = valueIndicesByDepth[depth];

            Val[] groupValues = ValSerialiser.EMPTY_VALUES;
            if (groupSize > 0) {
                groupValues = new Val[groupSize];
            }

            int groupIndex = 0;
            for (int fieldIndex = 0; fieldIndex < compiledFields.length; fieldIndex++) {
                final CompiledField compiledField = compiledFields[fieldIndex];

                final Expression expression = compiledField.getExpression();
                if (expression != null) {
                    Generator generator = null;
                    Val value = null;

                    // If this is the first level then check if we should filter out this data.
                    if (depth == 0) {
                        final CompiledFilter compiledFilter = compiledField.getCompiledFilter();
                        if (compiledFilter != null) {
                            generator = expression.createGenerator();
                            generator.set(values);

                            // If we are filtering then we need to evaluate this field
                            // now so that we can filter the resultant value.
                            value = generator.eval(null);

                            if (!compiledFilter.match(value.toString())) {
                                // We want to exclude this item so get out of this method ASAP.
                                return;
                            }
                        }
                    }

                    // If we are grouping at this level then evaluate the expression and add to the group values.
                    if (groupIndices[fieldIndex]) {
                        // If we haven't already created the generator then do so now.
                        if (value == null) {
                            generator = expression.createGenerator();
                            generator.set(values);
                            value = generator.eval(null);
                        }
                        groupValues[groupIndex++] = value;
                    }

                    // If we need a value at this level then evaluate the expression and add the value.
                    if (valueIndices[fieldIndex]) {
                        // If we haven't already created the generator then do so now.
                        if (generator == null) {
                            generator = expression.createGenerator();
                            generator.set(values);
                        }
                        generators[fieldIndex] = generator;
                    }
                }
            }

            final boolean grouped = depth <= compiledDepths.getMaxGroupDepth();
            final byte[] keyBytes;
            if (grouped) {
                // This is a grouped item.
                key = key.resolve(groupValues);
                keyBytes = key.getBytes();

                final LmdbKey rowKey = rowKeyBuilder
                        .depth(depth)
                        .parentRowKey(parentRowKey)
                        .keyBytes(keyBytes)
                        .group(true)
                        .build();
                final LmdbValue rowValue = new LmdbValue(
                        serialisers,
                        keyBytes,
                        new Generators(serialisers, compiledFields, generators));
                parentRowKey = rowKey;
                put(new LmdbKV(rowKey, rowValue));

            } else {
                // This item will not be grouped.
                final long uniqueId = getUniqueId();
                key = key.resolve(uniqueId);
                keyBytes = key.getBytes();

                final LmdbKey rowKey = rowKeyBuilder
                        .depth(depth)
                        .parentRowKey(parentRowKey)
                        .uniqueId(uniqueId)
                        .group(false)
                        .build();
                final LmdbValue rowValue = new LmdbValue(
                        serialisers,
                        keyBytes,
                        new Generators(serialisers, compiledFields, generators));
                put(new LmdbKV(rowKey, rowValue));
            }
        }
    }

    long getUniqueId() {
        return uniqueKey.incrementAndGet();
    }

    void put(final LmdbKV queueItem) {
        LOGGER.trace(() -> "put");
        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_PUT);

        // Some searches can be terminated early if the user is not sorting or grouping.
        boolean allow = true;
        if (limitResultCount) {
            // No sorting or grouping, so we can stop the search as soon as we have the number of results requested by
            // the client
            allow = !hasEnoughData.get();
            if (allow) {
                final long currentResultCount = totalResultCount.getAndIncrement();
                if (currentResultCount >= maxResults.size(0)) {
                    allow = false;

                    // If we have enough data then we can stop transferring data and complete.
                    if (hasEnoughData.compareAndSet(false, true)) {
                        completionState.signalComplete();
                    }
                }
            }
        }

        if (allow) {
            doPut(queueItem);
        }
    }

    private void doPut(final LmdbKV queueItem) {
        try {
            queue.put(queueItem);
        } catch (final InterruptedException e) {
            LOGGER.trace(e::getMessage, e);
            // Keep interrupting this thread.
            Thread.currentThread().interrupt();
        }
    }

    private void transfer() {
        Metrics.measure("Transfer", () -> {
            transferState.setThread(Thread.currentThread());

            final int maxPutsBeforeCommit = resultStoreConfig.getMaxPutsBeforeCommit();
            try (final BatchingWriteTxn batchingWriteTxn = lmdbEnv.openBatchingWriteTxn(maxPutsBeforeCommit)) {
                long lastCommitMs = System.currentTimeMillis();
                long uncommittedCount = 0;

                try {
                    while (!transferState.isTerminated()) {
                        LOGGER.trace(() -> "Transferring");
                        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_QUEUE_POLL);
                        final LmdbKV lmdbKV = queue.poll(1, TimeUnit.SECONDS);

                        if (lmdbKV != null) {
                            insert(batchingWriteTxn, dbi, lmdbKV);
                            uncommittedCount++;
                        }

                        if (producePayloads && payloadCreator.isEmpty()) {
                            // Commit
                            LOGGER.debug(() -> "Committing for new payload");
                            batchingWriteTxn.commit();
                            lastCommitMs = System.currentTimeMillis();
                            uncommittedCount = 0;

                            // Create payload and clear the DB.
                            payloadCreator.addPayload(batchingWriteTxn, dbi, false);

                        } else if (uncommittedCount > 0) {
                            if (uncommittedCount >= maxPutsBeforeCommit ||
                                    lastCommitMs < System.currentTimeMillis() - COMMIT_FREQUENCY_MS) {

                                // Commit
                                LOGGER.debug(() -> "Committing for elapsed time");
                                batchingWriteTxn.commit();
                                lastCommitMs = System.currentTimeMillis();
                                uncommittedCount = 0;
                            }
                        }
                    }
                } catch (final InterruptedException e) {
                    LOGGER.trace(e::getMessage, e);
                    // Keep interrupting this thread.
                    Thread.currentThread().interrupt();
                } catch (final CompleteException e) {
                    LOGGER.debug(() -> "Complete");
                    LOGGER.trace(e::getMessage, e);
                } catch (final RuntimeException e) {
                    LOGGER.error(e::getMessage, e);
                    errorConsumer.add(e);
                }

                if (!transferState.isTerminated() && uncommittedCount > 0) {
                    LOGGER.debug(() -> "Final commit");
                    batchingWriteTxn.commit();
                }

                // Create final payloads and ensure they are all delivered before we complete.
                if (!transferState.isTerminated() && producePayloads) {
                    LOGGER.debug(() -> "Producing final payloads");
                    // Create payload and clear the DB.
                    boolean finalPayload = false;
                    while (!finalPayload) {
                        finalPayload = payloadCreator.addPayload(batchingWriteTxn, dbi, true);
                    }
                    // Make sure we end with an empty payload to indicate completion.
                    // Adding a final empty payload to the queue ensures that a consuming node will have to request the
                    // payload from the queue before we complete.
                    LOGGER.debug(() -> "Final payload");
                    payloadCreator.finalPayload();
                }

            } catch (final RuntimeException e) {
                LOGGER.error(e::getMessage, e);
                errorConsumer.add(e);
            } finally {
                // Ensure we complete.
                complete.countDown();
                LOGGER.debug(() -> "Finished transfer while loop");
                transferState.setThread(null);
            }
        });
    }

    private void insert(final BatchingWriteTxn batchingWriteTxn,
                        final Dbi<ByteBuffer> dbi,
                        final LmdbKV queueItem) {
        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_INSERT);
        Metrics.measure("Insert", () -> {
            try {
                LOGGER.trace(() -> "insert");

                final LmdbKey rowKey = queueItem.getRowKey();
                final LmdbValue rowValue = queueItem.getRowValue();

                // Just try to put first.
                final boolean success = put(
                        batchingWriteTxn,
                        dbi,
                        rowKey.getByteBuffer(),
                        rowValue.getByteBuffer(),
                        PutFlags.MDB_NOOVERWRITE);
                if (success) {
                    resultCount.incrementAndGet();

                } else if (rowKey.isGroup()) {
                    // Get the existing entry for this key.
                    final ByteBuffer existingValueBuffer = dbi.get(batchingWriteTxn.getTxn(), rowKey.getByteBuffer());

                    final int minValSize = Math.max(1024, existingValueBuffer.remaining());
                    try (final UnsafeByteBufferOutput output =
                            serialisers.getOutputFactory().createByteBufferOutput(minValSize)) {
                        boolean merged = false;

                        try (final UnsafeByteBufferInput input = serialisers.getInputFactory().createByteBufferInput(
                                existingValueBuffer)) {
                            while (!input.end()) {
                                final LmdbValue existingRowValue = LmdbValue.read(serialisers, compiledFields, input);

                                // If this is the same value then update it and reinsert.
                                if (existingRowValue.getKey().equals(rowValue.getKey())) {
                                    final Generator[] generators = existingRowValue.getGenerators().getGenerators();
                                    final Generator[] newValue = rowValue.getGenerators().getGenerators();
                                    final Generator[] combined = combine(generators, newValue);

                                    LOGGER.trace(() -> "Merging combined value to output");
                                    final LmdbValue combinedValue = new LmdbValue(serialisers,
                                            existingRowValue.getKey().getBytes(),
                                            new Generators(serialisers, compiledFields, combined));
                                    combinedValue.write(output);

                                    // Copy any remaining values.
                                    if (!input.end()) {
                                        final byte[] remainingBytes = input.readAllBytes();
                                        output.writeBytes(remainingBytes, 0, remainingBytes.length);
                                    }

                                    merged = true;

                                } else {
                                    LOGGER.debug(() -> "Copying value to output");
                                    existingRowValue.write(output);
                                }
                            }
                        }

                        // Append if we didn't merge.
                        if (!merged) {
                            LOGGER.debug(() -> "Appending value to output");
                            rowValue.write(output);
                            resultCount.incrementAndGet();
                        }

                        final ByteBuffer newValue = output.getByteBuffer().flip();
                        final boolean ok = put(batchingWriteTxn, dbi, rowKey.getByteBuffer(), newValue);
                        if (!ok) {
                            LOGGER.debug(() -> "Unable to update");
                            throw new RuntimeException("Unable to update");
                        }
                    }

                } else {
                    // We do not expect a key collision here.
                    LOGGER.debug(() -> "Unexpected collision");
                    throw new RuntimeException("Unexpected collision");
                }

            } catch (final RuntimeException | IOException e) {
                if (LOGGER.isTraceEnabled()) {
                    // Only evaluate queue item value in trace as it can be expensive.
                    LOGGER.trace(() -> "Error putting " + queueItem + " (" + e.getMessage() + ")", e);
                } else {
                    LOGGER.debug(() -> "Error putting queueItem (" + e.getMessage() + ")", e);
                }

                final RuntimeException exception =
                        new RuntimeException("Error putting queueItem (" + e.getMessage() + ")", e);
                errorConsumer.add(exception);

                // Treat all errors as fatal so complete.
                completionState.signalComplete();

                throw exception;
            }
        });
    }

    private boolean put(final BatchingWriteTxn batchingWriteTxn,
                        final Dbi<ByteBuffer> dbi,
                        final ByteBuffer key,
                        final ByteBuffer val,
                        final PutFlags... flags) {
        try {
            SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_DBI_PUT);
            final boolean didPutSucceed = dbi.put(batchingWriteTxn.getTxn(), key, val, flags);
            if (didPutSucceed) {
                batchingWriteTxn.commitIfRequired();
            }
            return didPutSucceed;
        } catch (final Exception e) {
            LOGGER.debug(e::getMessage, e);
            errorConsumer.add(e);
            throw new RuntimeException(e.getMessage(), e);
        }
    }

    private Generator[] combine(final Generator[] existing, final Generator[] value) {
        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_COMBINE);
        return Metrics.measure("Combine", () -> {
            // Combine the new item into the original item.
            for (int i = 0; i < existing.length; i++) {
                Generator existingGenerator = existing[i];
                Generator newGenerator = value[i];
                if (newGenerator != null) {
                    if (existingGenerator == null) {
                        existing[i] = newGenerator;
                    } else {
                        existingGenerator.merge(newGenerator);
                    }
                }
            }

            return existing;
        });
    }

    /**
     * Get data from the store
     * Synchronised with clear to prevent a shutdown happening while reads are going on.
     *
     * @param consumer Consumer for the data.
     */
    @Override
    public synchronized void getData(final Consumer<Data> consumer) {
        SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_GET);
        LOGGER.trace(() -> "getData()");

        if (lmdbEnv.isClosed()) {
            // If we query LMDB after the env has been closed then we are likely to crash the JVM
            // see https://github.com/lmdbjava/lmdbjava/issues/185
            LOGGER.debug(() -> "getData() called (queryKey =" +
                    queryKey +
                    ", componentId=" +
                    componentId +
                    ") after store has been shut down");
        } else {
            lmdbEnv.doWithReadTxn(readTxn ->
                    Metrics.measure("getData", () ->
                            consumer.accept(new LmdbData(
                                    serialisers,
                                    rootKey,
                                    dbi,
                                    readTxn,
                                    compiledFields,
                                    compiledSorters,
                                    maxResults,
                                    queryKey))));
        }
    }

    /**
     * Clear the data store.
     * Synchronised with get() to prevent a shutdown happening while reads are going on.
     */
    @Override
    public synchronized void clear() {
        LOGGER.debug(() -> "clear called");
        LOGGER.trace(() -> "clear()", new RuntimeException("clear"));
        if (shutdown.compareAndSet(false, true)) {
            SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_CLEAR);

            // Let the transfer loop know it should stop ASAP.
            transferState.terminate();

            // Clear the queue.
            queue.clear();

            // If the transfer loop is waiting on new queue items ensure it loops once more.
            completionState.signalComplete();

            // Wait for transferring to stop.
            try {
                LOGGER.debug(() -> "Waiting for transfer to stop");
                completionState.awaitCompletion();
            } catch (final InterruptedException e) {
                LOGGER.trace(e::getMessage, e);
                // Keep interrupting this thread.
                Thread.currentThread().interrupt();
            }

            try {
                try {
                    dbi.close();
                } catch (final RuntimeException e) {
                    LOGGER.error(e::getMessage, e);
                    errorConsumer.add(e);
                }

                try {
                    lmdbEnv.close();
                } catch (final RuntimeException e) {
                    LOGGER.error(e::getMessage, e);
                    errorConsumer.add(e);
                }

                try {
                    lmdbEnv.delete();
                } catch (final RuntimeException e) {
                    LOGGER.error(e::getMessage, e);
                    errorConsumer.add(e);
                }
            } finally {
                resultCount.set(0);
                totalResultCount.set(0);
            }
        }
    }

    /**
     * Get the completion state associated with receiving all search results and having added them to the store
     * successfully.
     *
     * @return The search completion state for the data store.
     */
    @Override
    public CompletionState getCompletionState() {
        return completionState;
    }

    /**
     * Read items from the supplied input and transfer them to the data store.
     *
     * @param input The input to read.
     */
    @Override
    public void readPayload(final Input input) {
        // Return false if we aren't happy to accept any more data.
        payloadCreator.readPayload(input);
    }

    /**
     * Write data from the data store to an output removing them from the datastore as we go as they will be transferred
     * to another store.
     *
     * @param output The output to write to.
     */
    @Override
    public void writePayload(final Output output) {
        if (!producePayloads) {
            throw new RuntimeException("Not producing payloads");
        }
        payloadCreator.writePayload(output);
    }

    private static class LmdbData implements Data {

        private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(LmdbData.class);

        private final Serialisers serialisers;
        private final Key rootKey;
        private final Dbi<ByteBuffer> dbi;
        private final Txn<ByteBuffer> readTxn;
        private final CompiledField[] compiledFields;
        private final CompiledSorter<Item>[] compiledSorters;
        private final Sizes maxResults;
        private final QueryKey queryKey;

        public LmdbData(final Serialisers serialisers,
                        final Key rootKey,
                        final Dbi<ByteBuffer> dbi,
                        final Txn<ByteBuffer> readTxn,
                        final CompiledField[] compiledFields,
                        final CompiledSorter<Item>[] compiledSorters,
                        final Sizes maxResults,
                        final QueryKey queryKey) {
            this.serialisers = serialisers;
            this.rootKey = rootKey;
            this.dbi = dbi;
            this.readTxn = readTxn;
            this.compiledFields = compiledFields;
            this.compiledSorters = compiledSorters;
            this.maxResults = maxResults;
            this.queryKey = queryKey;
        }

        /**
         * Get root items from the data store.
         *
         * @return Root items.
         */
        @Override
        public Items get() {
            LOGGER.trace(() -> "get() called");
            return get(rootKey);
        }

        /**
         * Get child items from the data store for the provided parent key.
         * Synchronised with clear to prevent a shutdown happening while reads are going on.
         *
         * @param parentKey The parent key to get child items for.
         * @return The child items for the parent key.
         */
        @Override
        public Items get(final Key parentKey) {
            SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_GET);
            LOGGER.trace(() -> "get() called for parentKey: " + parentKey);

            return Metrics.measure("get", () -> {
                final int depth = parentKey.size();
                final int trimmedSize = maxResults.size(depth);
                return getChildren(parentKey, depth, trimmedSize, false);
            });
        }

        private ItemsImpl getChildren(final Key parentKey,
                                      final int depth,
                                      final int trimmedSize,
                                      final boolean trimTop) {
            SearchProgressLog.increment(queryKey, SearchPhase.LMDB_DATA_STORE_GET_CHILDREN);
            // If we don't have any children at the requested depth then return an empty list.
            if (compiledSorters.length <= depth) {
                return ItemsImpl.EMPTY;
            }

            final ItemsImpl list = new ItemsImpl(10);

            final ByteBuffer start = LmdbKey.createKeyStem(depth, parentKey);
            final KeyRange<ByteBuffer> keyRange = KeyRange.atLeast(start);

            final int maxSize;
            if (trimmedSize < Integer.MAX_VALUE / 2) {
                maxSize = Math.max(1000, trimmedSize * 2);
            } else {
                maxSize = Integer.MAX_VALUE;
            }
            final CompiledSorter<Item> sorter = compiledSorters[depth];

            boolean trimmed = true;
            boolean inRange = true;

            try (final CursorIterable<ByteBuffer> cursorIterable = dbi.iterate(readTxn, keyRange)) {
                final Iterator<KeyVal<ByteBuffer>> iterator = cursorIterable.iterator();

                while (iterator.hasNext()
                        && inRange
                        && !Thread.currentThread().isInterrupted()) {

                    final KeyVal<ByteBuffer> keyVal = iterator.next();

                    // Make sure the first part of the row key matches the start key we are looking for.
                    boolean match = true;
                    for (int i = 0; i < start.remaining() && match; i++) {
                        if (start.get(i) != keyVal.key().get(i)) {
                            match = false;
                        }
                    }

                    if (match) {
                        final ByteBuffer valueBuffer = keyVal.val();
                        try (final UnsafeByteBufferInput input =
                                serialisers.getInputFactory().createByteBufferInput(valueBuffer)) {
                            while (!input.end() && inRange) {
                                final LmdbValue rowValue = LmdbValue.read(serialisers, compiledFields, input);
                                final Key key = rowValue.getKey();
                                if (key.getParent().equals(parentKey)) {
                                    final Generator[] generators = rowValue.getGenerators().getGenerators();
                                    list.add(new ItemImpl(this, key, generators));
                                    if (list.size >= trimmedSize && sorter == null) {
                                        // Stop without sorting etc.
                                        inRange = false;

                                    } else {
                                        trimmed = false;
                                        if (list.size() > maxSize) {
                                            list.sortAndTrim(sorter, trimmedSize, trimTop);
                                            trimmed = true;
                                        }
                                    }
                                }
                            }
                        }
                    } else {
                        inRange = false;
                    }
                }
            }

            if (!trimmed) {
                list.sortAndTrim(sorter, trimmedSize, trimTop);
            }

            return list;
        }

        private long countChildren(final Key parentKey,
                                   final int depth) {
            long count = 0;

            // If we don't have any children at the requested depth then return 0.
            if (compiledSorters.length <= depth) {
                return 0;
            }

            final ByteBuffer start = LmdbKey.createKeyStem(depth, parentKey);
            final KeyRange<ByteBuffer> keyRange = KeyRange.atLeast(start);

            boolean inRange = true;
            try (final CursorIterable<ByteBuffer> cursorIterable = dbi.iterate(readTxn, keyRange)) {
                final Iterator<KeyVal<ByteBuffer>> iterator = cursorIterable.iterator();

                while (iterator.hasNext()
                        && inRange
                        && !Thread.currentThread().isInterrupted()) {

                    final KeyVal<ByteBuffer> keyVal = iterator.next();

                    // Make sure the first part of the row key matches the start key we are looking for.
                    boolean match = true;
                    for (int i = 0; i < start.remaining() && match; i++) {
                        if (start.get(i) != keyVal.key().get(i)) {
                            match = false;
                        }
                    }

                    if (match) {
                        count++;
                    } else {
                        inRange = false;
                    }
                }
            }

            return count;
        }
    }

    private static class ItemsImpl implements Items {

        private static final ItemsImpl EMPTY = new ItemsImpl(0);

        private final int minArraySize;
        private ItemImpl[] array;
        private int size;

        public ItemsImpl(final int minArraySize) {
            this.minArraySize = minArraySize;
            array = new ItemImpl[minArraySize];
        }

        void sortAndTrim(final CompiledSorter<Item> sorter,
                         final int trimmedSize,
                         final boolean trimTop) {
            if (sorter != null && size > 0) {
                Arrays.sort(array, 0, size, sorter);
            }
            if (size > trimmedSize) {
                final int len = Math.max(minArraySize, trimmedSize);
                final ItemImpl[] newArray = new ItemImpl[len];
                if (trimTop) {
                    System.arraycopy(array, size - trimmedSize, newArray, 0, trimmedSize);
                } else {
                    System.arraycopy(array, 0, newArray, 0, trimmedSize);
                }
                array = newArray;
                size = trimmedSize;
            }
        }

        void add(final ItemImpl item) {
            if (array.length <= size) {
                final ItemImpl[] newArray = new ItemImpl[size * 2];
                System.arraycopy(array, 0, newArray, 0, array.length);
                array = newArray;
            }
            array[size++] = item;
        }

        @Override
        public Item get(final int index) {
            return array[index];
        }

        @Override
        public int size() {
            return size;
        }

        @Override
        @NotNull
        public Iterator<Item> iterator() {
            return new Iterator<>() {
                private int pos = 0;

                @Override
                public boolean hasNext() {
                    return size > pos;
                }

                @Override
                public Item next() {
                    return array[pos++];
                }
            };
        }

    }

    public static class ItemImpl implements Item {

        private final LmdbData data;
        private final Key key;
        private final Generator[] generators;

        public ItemImpl(final LmdbData data,
                        final Key key,
                        final Generator[] generators) {
            this.data = data;
            this.key = key;
            this.generators = generators;
        }

        @Override
        public Key getKey() {
            return key;
        }

        @Override
        public Val getValue(final int index, final boolean evaluateChildren) {
            Val val = null;

            final Generator generator = generators[index];
            if (generator != null) {
                if (evaluateChildren) {
                    final Supplier<ChildData> childDataSupplier = () -> {
                        // If we don't have any children at the requested depth then return null.
                        if (data.compiledSorters.length <= key.size()) {
                            return null;
                        }

                        return new ChildData() {
                            @Override
                            public Val first() {
                                return singleValue(1, false);
                            }

                            @Override
                            public Val last() {
                                return singleValue(1, true);
                            }

                            @Override
                            public Val nth(final int pos) {
                                return singleValue(pos + 1, false);
                            }

                            @Override
                            public Val top(final String delimiter, final int limit) {
                                return join(delimiter, limit, false);
                            }

                            @Override
                            public Val bottom(final String delimiter, final int limit) {
                                return join(delimiter, limit, true);
                            }

                            @Override
                            public Val count() {
                                final long count = data.countChildren(
                                        key,
                                        key.size());
                                if (count <= 0) {
                                    return ValNull.INSTANCE;
                                }
                                return ValLong.create(count);
                            }

                            private Val singleValue(final int trimmedSize, final boolean trimTop) {
                                final Items items = data.getChildren(
                                        key,
                                        key.size(),
                                        trimmedSize,
                                        trimTop);
                                if (items != null && items.size() == trimmedSize) {
                                    return items.get(trimmedSize - 1).getValue(index, false);
                                }
                                return ValNull.INSTANCE;
                            }

                            private Val join(final String delimiter, final int limit, final boolean trimTop) {
                                final Items items = data.getChildren(
                                        key,
                                        key.size(),
                                        limit,
                                        trimTop);
                                if (items != null && items.size() > 0) {
                                    final StringBuilder sb = new StringBuilder();
                                    for (final Item item : items) {
                                        final Val val = item.getValue(index, false);
                                        if (val.type().isValue()) {
                                            if (sb.length() > 0) {
                                                sb.append(delimiter);
                                            }
                                            sb.append(val);
                                        }
                                    }
                                    return ValString.create(sb.toString());
                                }
                                return ValNull.INSTANCE;
                            }
                        };
                    };
                    val = generator.eval(childDataSupplier);

                } else {
                    val = generator.eval(null);
                }
            }

            return val;
        }
    }

    private static class CompletionStateImpl implements CompletionState {

        private final LmdbDataStore lmdbDataStore;
        private final CountDownLatch complete;

        public CompletionStateImpl(final LmdbDataStore lmdbDataStore,
                                   final CountDownLatch complete) {
            this.lmdbDataStore = lmdbDataStore;
            this.complete = complete;
        }

        @Override
        public void signalComplete() {
            if (!isComplete()) {
                // Add an empty item to the transfer queue.
                lmdbDataStore.queue.complete();
            }
        }

        @Override
        public boolean isComplete() {
            boolean complete = true;

            try {
                complete = this.complete.await(0, TimeUnit.MILLISECONDS);
            } catch (final InterruptedException e) {
                LOGGER.trace(e::getMessage, e);
                // Keep interrupting this thread.
                Thread.currentThread().interrupt();
            }
            return complete;
        }

        @Override
        public void awaitCompletion() throws InterruptedException {
            complete.await();
        }

        @Override
        public boolean awaitCompletion(final long timeout, final TimeUnit unit) throws InterruptedException {
            return complete.await(timeout, unit);
        }
    }

    private static class TransferState {

        private final AtomicBoolean terminated = new AtomicBoolean();
        private volatile Thread thread;

        public boolean isTerminated() {
            return terminated.get();
        }

        public synchronized void terminate() {
            terminated.set(true);
            if (thread != null) {
                thread.interrupt();
            }
        }

        public synchronized void setThread(final Thread thread) {
            this.thread = thread;
            if (terminated.get()) {
                if (thread != null) {
                    thread.interrupt();
                } else if (Thread.interrupted()) {
                    LOGGER.debug(() -> "Cleared interrupt state");
                }
            }
        }
    }
}


text:
/**
 * <!-- Package description. -->
 * Contains APIs for security, authentication, and authorization.
 */
package org.apache.ignite.plugin.security;


text:
BeforeAll {

    Set-StrictMode -Version latest

    # Make sure MetaFixers.psm1 is loaded - it contains Get-TextFilesList
    Import-Module -Name (Join-Path -Path $PSScriptRoot -ChildPath 'MetaFixers.psm1') -Verbose:$false -Force

    $projectRoot = $ENV:BHProjectPath
    if (-not $projectRoot) {
        $projectRoot = $PSScriptRoot
    }

    $allTextFiles      = Get-TextFilesList $projectRoot
    $unicodeFilesCount = 0
    $totalTabsCount    = 0
    foreach ($textFile in $allTextFiles) {
        if (Test-FileUnicode $textFile) {
            $unicodeFilesCount++
            Write-Warning (
                "File $($textFile.FullName) contains 0x00 bytes." +
                " It probably uses Unicode/UTF-16 and needs to be converted to UTF-8." +
                " Use Fixer 'Get-UnicodeFilesList `$pwd | ConvertTo-UTF8'."
            )
        }
        $unicodeFilesCount | Should -Be 0

        $fileName = $textFile.FullName
        (Get-Content $fileName -Raw) | Select-String "`t" | Foreach-Object {
            Write-Warning (
                "There are tabs in $fileName." +
                " Use Fixer 'Get-TextFilesList `$pwd | ConvertTo-SpaceIndentation'."
            )
            $totalTabsCount++
        }
    }
}

Describe 'Text files formatting' {
    Context 'File encoding' {
        It "No text file uses Unicode/UTF-16 encoding" {
            $unicodeFilesCount | Should -Be 0
        }
    }

    Context 'Indentations' {
        It "No text file use tabs for indentations" {
            $totalTabsCount | Should -Be 0
        }
    }
}


text:
CKEDITOR.plugins.setLang( 'indent', 'gl', {
	indent: 'Aumentar a sangría',
	outdent: 'Reducir a sangría'
} );


text:
require 'spec_helper'

describe StagesHelper do
  include StagesHelper, GoUtil

  before do
    @stage = StageMother.scheduledStage("cruise", 10, "dev", 5, "unit")
    @stage_summary = StageSummaryModel.new(@stage, Stages.new(), JobDurationStrategy::ConstantJobDuration.new(10), @stage.getIdentifier())
    @new_stage = Stage.new()
    @new_stage_summary = StageSummaryModel.new(@new_stage, Stages.new(), JobDurationStrategy::ConstantJobDuration.new(10), StageIdentifier.new())
  end

  it "should generate pipeline url when stage identifier is given" do
    expect(stage_detail_path_for_identifier(StageIdentifier.new("pipeline", 10, "stage", "5"))).to eq("/pipelines/pipeline/10/stage/5")
  end

  it "should generate a stage tabl url for a given tab" do
    expect(tab_aware_path_for_stage(StageIdentifier.new("pipeline", 10, "stage", "5"), "jobs")).to eq("/pipelines/pipeline/10/stage/5/jobs")
  end

  it "should generate pipeline url when stage identifier is given" do
    expect(stage_detail_pipeline_tab_for_identifier(StageIdentifier.new("pipeline", 10, "stage", "5"))).to eq("/pipelines/pipeline/10/stage/5/pipeline")
  end

  it "should understand if stage is dummy" do
    expect(placeholder_stage?(@stage_summary)).to be_false
    expect(placeholder_stage?(@new_stage_summary)).to be_true
  end

  it "should understand if stage is the current stage" do
    params[:pipeline_name] = "pipeline_name"
    params[:pipeline_counter] = "1"
    params[:stage_name] = "stage_name"
    params[:stage_counter] = "2"
    expect(is_current_stage?(StageIdentifier.new('pipeline_name', 1, "stage_name", "2"))).to be_true
    expect(is_current_stage?(StageIdentifier.new('pipeline_name_x', 1, "stage_name", "2"))).to be_false
    expect(is_current_stage?(StageIdentifier.new('pipeline_name', 10, "stage_name", "2"))).to be_false
    expect(is_current_stage?(StageIdentifier.new('pipeline_name', 1, "stage_name_x", "2"))).to be_false
    expect(is_current_stage?(StageIdentifier.new('pipeline_name', 1, "stage_name", "20"))).to be_false
  end

  it "should generate link with current tab css if this is the current tab" do
    in_params(:pipeline_name => "foo_bar", :stage_name => "stage-name", :pipeline_counter => "1", :stage_counter => "1", :action => "jobs", :controller => "stages")
    link = link_with_current_tab "Jobs", "jobs"
    expect(link).to have_selector("li.current a[href='/pipelines/foo_bar/1/stage-name/1/jobs']", :text => "Jobs")
  end

  it "should not generate link with current tab css if this is not the current tab" do
    in_params(:pipeline_name => "foo_bar", :stage_name => "stage-name", :pipeline_counter => "1", :stage_counter => "1", :action => "jobs", :controller => "stages")
    link = link_with_current_tab "Tests", "tests"
    expect(link).to_not have_selector("li.current")
    expect(link).to have_selector("li a[href='/pipelines/foo_bar/1/stage-name/1/tests']", :text => "Tests")
  end

  it "should return true when config version mismatches" do
    expect(is_config_used_to_run_this_stage_out_of_sync_with_current?('foo', 'bar')).to eq(true)
  end

  it "should return false when config version matches" do
    expect(is_config_used_to_run_this_stage_out_of_sync_with_current?('foo', 'foo')).to eq(false)
  end
end


text:
require 'rubygems'
require 'json'
require 'net/http'
require 'uri'

module ESS
  module Pusher
    ##
    # Sets the default aggregator services. Accepts a list of links in strings.
    #
    def self.aggregators= aggs
      raise ArgumentError, "this method requires a list of links" if aggs.class != Array
      @@aggregators = aggs
    end

    ##
    # Returns the aggregator services currently set as default.
    #
    def self.aggregators
      @@aggregators ||= ["http://www.robby.ai/api/v1/ess/aggregator.json"]
    end

    ##
    # Pushes the feed to aggregators.
    #
    # === Options
    #
    # ==== :data
    #
    # A string, with an XML document representing the feed that needs to be
    # pushed.
    #
    # ==== :feed
    #
    # This is an alternative to the :data option. This method accepts the feed
    # link instead of the whole document. The aggregator service will pull the
    # feed using this link.
    #
    # ==== :request
    #
    # This should be the request object that generated this feed. It can be
    # useful for the aggregator service to receive some of th information
    # from this object for crawling purposes.
    #
    # ==== :aggregators
    #
    # Intead of using the default aggregator services, this options can be used
    # to specify what aggregators should the document be pushed to.
    #
    # ==== :ignore_errors
    #
    # The method will ignore any response from the aggregator services if this
    # option is true. Default is false.
    #
    def self.push_to_aggregators options={}
      options = { :aggregators => Pusher::aggregators,
                  :feed => nil,
                  :data => nil,
                  :request => nil,
                  :ignore_errors => false }.merge options
      options[:aggregators].each do |aggregator|
        url = URI.parse(aggregator)
        request = Net::HTTP::Post.new(url.path)

        form_data = { "output" => "json",
                      "LIBRARY_VERSION" => VERSION }
        form_data["SERVER_ADMIN"] = ENV["SERVER_ADMIN"] if ENV["SERVER_ADMIN"]
        if ENV["SERVER_PROTOCOL"]
          if ENV["SERVER_PROTOCOL"].include? "HTTPS"
            form_data["SERVER_PROTOCOL"] = "https://"
          else
            form_data["SERVER_PROTOCOL"] = "http://"
          end
        end
        form_data["HTTP_HOST"] = ENV["HTTP_HOST"] if ENV["HTTP_HOST"]
        if options[:request]
          form_data["REMOTE_ADDR"] = options[:request].remote_ip
          form_data["REQUEST_URI"] = options[:request].fullpath
        end
        form_data["GEOIP_LATITUDE"] = ENV["GEOIP_LATITUDE"] if ENV["GEOIP_LATITUDE"]
        form_data["GEOIP_LONGITUDE"] = ENV["GEOIP_LONGITUDE"] if ENV["GEOIP_LONGITUDE"]

        if options[:feed].nil?
          form_data["feed_file"] = options[:data]
        else
          form_data["feed"] = options[:feed]
        end
        request.form_data = form_data
        response = Net::HTTP.start(url.host, url.port) { |http| http.request(request) }
        response = JSON.parse response.body
        unless options[:ignore_errors]
          errors = response["result"]["error"]
          unless errors.nil?
            raise RuntimeError, "errors while submitting feed: #{errors}"
          end
        end
      end
    end
  end
end


text:
package com.tle.mycontent.workflow.operations;

import com.tle.beans.item.ItemStatus;
import com.tle.core.item.standard.operations.workflow.StatusOperation;

public class MyContentStatusOperation extends StatusOperation {
  @Override
  public ItemStatus getItemStatus() {
    return ItemStatus.PERSONAL;
  }
}


text:
class MessageBusProxyAbstract;

namespace {
	using Cursor = long unsigned int;
}

class MessageProxy
{
	public:
		template <typename T>
		static MessageProxy create();

	private:
		MessageProxy()=default;
		void* m_busAddress;
};

class MessageBusQueue
{
	public:
		static MessageBusQueue* instance();
		template<typename T>
		void push();

		void dispatch(std::vector<MessageBusProxyAbstract*> proxys);

	private:
		using MessageQueue = std::deque<MessageProxy>;

		MessageBusQueue()=default;

		static std::unique_ptr<MessageBusQueue> m_instance;

		MessageQueue m_messages;
		std::map<MessageBusProxyAbstract*, typename MessageQueue::iterator> m_proxys;
};

#include "messagebus.hpp"

template <typename MessageType>
MessageProxy MessageProxy::create()
{
	MessageProxy proxy;
	proxy.m_busAddress = static_cast<void*>(MessageBus<MessageType>::getBus());
	return proxy;
}

template <typename MessageType>
void MessageBusQueue::push()
{
	m_messages.push_back(MessageProxy::create<MessageType>());
}


text:
package au.com.acegi.hashbench.hashers;

import java.nio.ByteBuffer;
import java.util.Map;
import java.util.zip.Adler32;

public class AdlerHasher implements Hasher {
  public static final String ADLER_32 = "adler32-jre";

  public static final void register(final Map<String, Hasher> hashers) {
    hashers.put(AdlerHasher.ADLER_32, new AdlerHasher());
  }

  private final Adler32 delegate = new Adler32();

  private AdlerHasher() {}

  @Override
  public long hash(final byte[] in, final int off, final int len) {
    this.delegate.reset();
    this.delegate.update(in, off, len);
    return this.delegate.getValue();
  }

  @Override
  public long hash(final ByteBuffer bb, final int off, final int len) {
    this.delegate.reset();

    if (bb.hasArray()) {
      this.delegate.update(bb.array(), off, len);
      return this.delegate.getValue();
    }

    final ByteBuffer view = bb.duplicate();
    view.position(off);
    view.limit(off + len);
    this.delegate.update(view);
    return this.delegate.getValue();
  }
}


text:
static std::recursive_timed_mutex m;

typedef std::chrono::system_clock Clock;
typedef Clock::time_point time_point;
typedef Clock::duration duration;
typedef std::chrono::milliseconds ms;
typedef std::chrono::nanoseconds ns;

static int f()
{
    time_point t0 = Clock::now();
    m.lock();
    time_point t1 = Clock::now();
    m.lock();
    m.unlock();
    m.unlock();
    ns d = t1 - t0 - ms(250);
    TC_ASSERT_EXPR(d < ms(50));  // within 50ms
    return 0;
}

int tc_libcxx_thread_thread_timedmutex_recursive_lock(void)
{
    m.lock();
    std::thread t(f);
    std::this_thread::sleep_for(ms(250));
    m.unlock();
    t.join();
    TC_SUCCESS_RESULT();
    return 0;
}


text:
namespace mozart {

/////////////////////////////////
// Unicode encoding conversion //
/////////////////////////////////

nativeint toUTF(char32_t character, char utf[4]) {
  if (character < 0x80) {
    utf[0] = (char) character;
    return 1;
  } else if (character < 0x800) {
    utf[0] = (char) (0xc0 | (character >> 6));
    utf[1] = (char) (0x80 | (character & 0x3f));
    return 2;
  } else if (character < 0x10000) {
    if (0xd800 <= character && character < 0xe000)
      return UnicodeErrorReason::surrogate;
    utf[0] = (char) (0xe0 | (character >> 12));
    utf[1] = (char) (0x80 | ((character >> 6) & 0x3f));
    utf[2] = (char) (0x80 | (character & 0x3f));
    return 3;
  } else if (character < 0x110000) {
    utf[0] = (char) (0xf0 | (character >> 18));
    utf[1] = (char) (0x80 | ((character >> 12) & 0x3f));
    utf[2] = (char) (0x80 | ((character >> 6) & 0x3f));
    utf[3] = (char) (0x80 | (character & 0x3f));
    return 4;
  } else {
    return UnicodeErrorReason::outOfRange;
  }
}

nativeint toUTF(char32_t character, char16_t utf[2]) {
  if (character < 0x10000) {
    if (0xd800 <= character && character < 0xe000)
      return UnicodeErrorReason::surrogate;
    utf[0] = (char16_t) character;
    return 1;
  } else if (character < 0x110000) {
    character -= 0x10000;
    utf[0] = (char16_t) (0xd800 | (character >> 10));
    utf[1] = (char16_t) (0xdc00 | (character & 0x3ff));
    return 2;
  } else {
    return UnicodeErrorReason::outOfRange;
  }
}

nativeint toUTF(char32_t character, char32_t utf[1]) {
  if (0xd800 <= character && character < 0xe000)
    return UnicodeErrorReason::surrogate;
  else if (character >= 0x110000)
    return UnicodeErrorReason::outOfRange;

  utf[0] = character;
  return 1;
}

inline std::pair<char32_t, nativeint> fromUTF8ContSeq(const char* utf,
                                                      char32_t val,
                                                      nativeint length,
                                                      char32_t lowerLimit) {
  for (nativeint i = 1; i < length; ++ i) {
    if ((utf[i] & 0xc0) != 0x80)
      return std::make_pair(0xfffd, UnicodeErrorReason::invalidUTF8);
    val = val << 6 | (0x3f & (unsigned char) utf[i]);
  }
  if (val < lowerLimit)
    length = UnicodeErrorReason::invalidUTF8;
  return std::make_pair(val, length);
}

std::pair<char32_t, nativeint> fromUTF(const char* utf, nativeint length) {
  unsigned char leadingByte = *utf;

  // 1-byte code.
  if (leadingByte < 0x80) {
    return std::make_pair(leadingByte, 1);
  }

  // invalid leading byte.
  else if (leadingByte < 0xc2) {
    return std::make_pair(0xfffd, UnicodeErrorReason::invalidUTF8);
  }

  else if (leadingByte > 0xf4) {
    return std::make_pair(0xfffd, UnicodeErrorReason::outOfRange);
  }

  // 2-byte code.
  else if (leadingByte < 0xe0) {
    if (length >= 2)
      return fromUTF8ContSeq(utf, leadingByte & 0x1f, 2, 0x80);
    else
      return std::make_pair(0xfffd, UnicodeErrorReason::truncated);
  }

  // 3-byte code.
  else if (leadingByte < 0xf0) {
    if (length < 3)
      return std::make_pair(0xfffd, UnicodeErrorReason::truncated);
    auto retval = fromUTF8ContSeq(utf, leadingByte & 0xf, 3, 0x800);
    if (0xd800 <= retval.first && retval.first < 0xe000)
      retval.second = UnicodeErrorReason::surrogate;
    return retval;
  }

  // 4-byte code.
  else {
    if (length < 4)
      return std::make_pair(0xfffd, UnicodeErrorReason::truncated);
    auto retval = fromUTF8ContSeq(utf, leadingByte & 7, 4, 0x10000);
    if (retval.first >= 0x110000)
      retval.second = UnicodeErrorReason::outOfRange;
    return retval;
  }
}

std::pair<char32_t, nativeint> fromUTF(const char16_t* utf, nativeint length) {
  char16_t lead = *utf;
  if (lead < 0xd800 || lead >= 0xe000) {
    return std::make_pair(lead, 1);
  } else if (lead < 0xdc00) {
    if (length < 2)
      return std::make_pair(0xfffd, UnicodeErrorReason::truncated);
    char16_t trail = utf[1];
    if (0xdc00 <= trail && trail < 0xe000) {
      char32_t codePoint = 0x10000 + ((lead & 0x3ff) << 10 | (trail & 0x3ff));
      return std::make_pair(codePoint, 2);
    }
  }
  return std::make_pair(0xfffd, UnicodeErrorReason::invalidUTF16);
}

std::pair<char32_t, nativeint> fromUTF(const char32_t* utf, nativeint) {
  char32_t c = *utf;
  nativeint length = 1;
  if (0xd800 <= c && c < 0xe000)
    length = UnicodeErrorReason::surrogate;
  else if (c >= 0x110000)
    length = UnicodeErrorReason::outOfRange;
  return std::make_pair(c, length);
}

namespace internal {
  template <size_t i>
  struct FromUTFWCharT {
    static std::pair<char32_t, nativeint> call(const wchar_t* utf,
                                               nativeint length) {
      static_assert(i != i, "Calling fromUTF(wchar_t) with an unknown wchar_t");
      return std::make_pair(0, 0);
    }
  };

  template <>
  struct FromUTFWCharT<sizeof(char16_t)> {
    static std::pair<char32_t, nativeint> call(const wchar_t* utf,
                                               nativeint length) {
      return fromUTF(reinterpret_cast<const char16_t*>(utf), length);
    }
  };

  template <>
  struct FromUTFWCharT<sizeof(char32_t)> {
    static std::pair<char32_t, nativeint> call(const wchar_t* utf,
                                               nativeint length) {
      return fromUTF(reinterpret_cast<const char32_t*>(utf), length);
    }
  };
}

std::pair<char32_t, nativeint> fromUTF(const wchar_t* utf, nativeint length) {
  return internal::FromUTFWCharT<sizeof(wchar_t)>::call(utf, length);
}

template <class C, class F, class G>
static void forEachCodePoint(const BaseLString<C>& string,
                             const F& onChar, const G& onError) {
  const C* cur = string.begin();
  const C* end = string.end();
  while (cur < end) {
    auto codePointSizePair = fromUTF(cur, end-cur);
    if (codePointSizePair.second < 0) {
      bool cont = onError(*cur, (UnicodeErrorReason) codePointSizePair.second);
      if (cont) {
        ++ cur;
        continue;
      } else {
        return;
      }
    }

    bool cont = onChar(codePointSizePair.first);
    if (cont) {
      cur += codePointSizePair.second;
    } else {
      break;
    }
  }
}

template <class To, class From>
struct UTFConvertor {
  static ContainedLString<std::vector<To>> call(const BaseLString<From>& input) {
    // propagate error if needed.
    if (input.isErrorOrEmpty())
      return input.error;

    // The slow branch for required conversion.
    std::vector<To> tempVector;
    tempVector.reserve(input.length);
    UnicodeErrorReason curError = UnicodeErrorReason::empty;

    forEachCodePoint(
      input,
      [&](char32_t codePoint) -> bool {
        To encoded[4];
        nativeint encodedLength = toUTF(codePoint, encoded);
        if (encodedLength < 0) {
          curError = (UnicodeErrorReason) encodedLength;
          return false;
        } else {
          tempVector.insert(tempVector.end(), encoded, encoded + encodedLength);
          return true;
        }
      },
      [&](From, UnicodeErrorReason reason) -> bool {
        curError = reason;
        return false;
      });

    if (curError != UnicodeErrorReason::empty)
      return curError;
    else
      return std::move(tempVector);
  }
};

template <class To>
struct UTFConvertor<To, To> {
  static ContainedLString<std::vector<To>> call(const BaseLString<To>& input) {
    // propagate error if needed.
    if (input.isErrorOrEmpty())
      return input.error;

    // Just validate if the whole string is correct.
    UnicodeErrorReason curError = UnicodeErrorReason::empty;
    forEachCodePoint(
      input,
      [](char32_t) { return true; },
      [&](To, UnicodeErrorReason error) -> bool {
        curError = error;
        return false;
      });

    if (curError != UnicodeErrorReason::empty)
      return curError;
    else
      return ContainedLString<std::vector<To>>(input.begin(), input.end());
  }
};

template <class To, class From>
ContainedLString<std::vector<To>> toUTF(const BaseLString<From>& input) {
  return UTFConvertor<To, From>::call(input);
}

template <class C>
int compareByCodePoint(const BaseLString<C>& a, const BaseLString<C>& b) {
  if (a.string == b.string) {
    return a.length < b.length ? -1 : a.length > b.length ? 1 : 0;
  }

  size_t minLength = std::min(a.length, b.length);

  // UTF-8 and UTF-32 can be sorted directly.
  if (!std::is_same<C, char16_t>::value) {
    int compareRes = std::char_traits<C>::compare(a.string, b.string,
                                                  minLength);
    if (compareRes != 0)
      return compareRes;
  }

  // UTF-16 need be compared manually, because a surrogate pair rank higher
  // than 0xe000, but the value in a string is lower.
  else {
    auto aa = a.string, bb = b.string;
    for (; minLength; --minLength, ++aa, ++bb) {
      if (*aa == *bb)
        continue;

      bool aIsSurrogate = 0xd800 <= *aa && *aa < 0xe000;
      bool bIsSurrogate = 0xd800 <= *bb && *bb < 0xe000;

      if (aIsSurrogate == bIsSurrogate) {
        return *aa < *bb ? -1 : 1;
      } else {
        return aIsSurrogate ? 1 : -1;
      }
    }
  }

  return a.length < b.length ? -1 : a.length > b.length ? 1 : 0;
}

nativeint getUTFStride(const char* utf) {
  unsigned char leadingByte = *utf;
  if (leadingByte < 0x80)
    return 1;
  else if (leadingByte < 0xc2)
    return UnicodeErrorReason::invalidUTF8;
  else if (leadingByte > 0xf4)
    return UnicodeErrorReason::outOfRange;
  else if (leadingByte < 0xe0)
    return 2;
  else if (leadingByte < 0xf0)
    return 3;
  else
    return 4;
}

nativeint getUTFStride(const char16_t* utf) {
  char16_t lead = *utf;
  if (lead < 0xd800 || lead >= 0xe000)
    return 1;
  else if (lead < 0xdc00)
    return 2;
  else
    return UnicodeErrorReason::invalidUTF16;
}

nativeint getUTFStride(const char32_t* utf) {
  return 1;
}

constexpr bool isLeadingCodeUnit(char c) {
  return ('\x00' <= c && c <= '\x7f') || ('\xc2' <= c && c <= '\xf4');
}

constexpr bool isLeadingCodeUnit(char16_t c) {
  return !(0xdc00 <= c && c < 0xe000);
}

constexpr bool isLeadingCodeUnit(char32_t c) {
  return true;
}

template <class C>
nativeint codePointCount(const BaseLString<C>& input) {
  if (std::is_same<C, char32_t>::value || input.isErrorOrEmpty())
    return input.length;
  return std::count_if(input.begin(), input.end(), [](C c) {
    return isLeadingCodeUnit(c);
  });
}

namespace internal {
  template <class C>
  inline
  bool skipCodePoints(const C*& iter, const C* end, nativeint count) {
    // Assume isLeadingCodeUnit(*iter) at that point - but do not assert it

    while (count > 0 && iter < end) {
      if (isLeadingCodeUnit(*++iter))
        --count;
    }

    return count == 0;
  }
}

template <class C>
LString<C> sliceByCodePointsFromTo(const LString<C>& input,
                                   nativeint from, nativeint to) {
  if (from < 0 || to < from)
    return UnicodeErrorReason::indexOutOfBounds;

  if (std::is_same<C, char32_t>::value) {
    if (to > input.length)
      return UnicodeErrorReason::indexOutOfBounds;
    else
      return input.slice(from, to);
  } else {
    const C* begin = input.begin();
    if (!internal::skipCodePoints(begin, input.end(), from))
      return UnicodeErrorReason::indexOutOfBounds;

    const C* end = begin;
    if (!internal::skipCodePoints(end, input.end(), to - from))
      return UnicodeErrorReason::indexOutOfBounds;

    return input.slice(begin - input.begin(), end - input.begin());
  }
}

template <class C>
LString<C> sliceByCodePointsFrom(const LString<C>& input,
                                 nativeint from) {
  if (from < 0)
    return UnicodeErrorReason::indexOutOfBounds;

  if (std::is_same<C, char32_t>::value) {
    if (from > input.length)
      return UnicodeErrorReason::indexOutOfBounds;
    else
      return input.slice(from);
  } else {
    const C* begin = input.begin();
    if (!internal::skipCodePoints(begin, input.end(), from))
      return UnicodeErrorReason::indexOutOfBounds;

    return input.slice(begin - input.begin());
  }
}

}

#endif // __UTF_H


text:
class CreateLoadBalancerListeners < ActiveRecord::Migration[5.0]
  def change
    create_table :load_balancer_listeners do |t|
      t.string  :ems_ref
      t.string  :name
      t.string  :description
      t.string  :load_balancer_protocol
      t.integer :load_balancer_port
      t.string  :instance_protocol
      t.integer :instance_port

      t.belongs_to :ems,                :type => :bigint
      t.belongs_to :cloud_tenant,       :type => :bigint
      t.belongs_to :load_balancer,      :type => :bigint
      t.belongs_to :load_balancer_pool, :type => :bigint
      t.belongs_to :network_port,       :type => :bigint

      t.timestamps
    end

    add_index :load_balancer_listeners, :ems_ref
  end
end


text:
<?php

declare(strict_types=1);

namespace App\Repository;

use App\Entity\CompanyMediaLocalize;
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\Persistence\ManagerRegistry;

class CompanyMediaLocalizeRepository extends ServiceEntityRepository
{
    public function __construct(ManagerRegistry $registry)
    {
        parent::__construct($registry, CompanyMediaLocalize::class);
    }

    /*
    public function findBySomething($value)
    {
        return $this->createQueryBuilder('c')
            ->where('c.something = :value')->setParameter('value', $value)
            ->orderBy('c.id', 'ASC')
            ->setMaxResults(10)
            ->getQuery()
            ->getResult()
        ;
    }
    */
}


text: for testing auto sync project


text:
'use strict';

// Configuring the Pets module
angular.module('pets').run(['Menus',
	function(Menus) {
		// Set top bar menu items
		Menus.addMenuItem('topbar', 'Pets', 'pets', 'dropdown', '/pets(/create)?');
		Menus.addSubMenuItem('topbar', 'pets', 'List Pets', 'pets');
		Menus.addSubMenuItem('topbar', 'pets', 'New Pet', 'pets/create');
	}
]);