text:
import elasticsearch

import datetime
import operator
import unittest
import logging as std_logging

from django.test import TestCase
from django.conf import settings
from django.apps import apps
from django.test.utils import override_settings

from elasticsearch.exceptions import ConnectionError

from haystack import connections, reset_search_queries
from haystack.utils.loading import UnifiedIndex
from haystack.utils import log as logging
from haystack.query import SQ, RelatedSearchQuerySet, SearchQuerySet
from haystack.inputs import AutoQuery
from haystack.models import SearchResult
# from haystack.utils.geo import Point

from .mocks import MockSearchResult
from tests.test_app.models import MockModel, AnotherMockModel, AFourthMockModel
from tests.test_app.search_indexes import (ElasticsearchMockSearchIndex,
                                           ElasticsearchMockSearchIndexWithSkipDocument,
                                           ElasticsearchMaintainTypeMockSearchIndex,
                                           ElasticsearchComplexFacetsMockSearchIndex,
                                           ElasticsearchMockSpellingIndex,
                                           ElasticsearchMockModelSearchIndex,
                                           ElasticsearchAnotherMockModelSearchIndex,
                                           ElasticsearchAutocompleteMockModelSearchIndex,
                                           ElasticsearchRoundTripSearchIndex,
                                           ElasticsearchFacetingMockSearchIndex,
                                           ElasticsearchBoostMockSearchIndex,
                                           )

from .utils import clear_elasticsearch_index

test_pickling = True

try:
    import cPickle as pickle
except ImportError:
    try:
        import pickle
    except ImportError:
        test_pickling = False

# TODO implement get_connection_params
# TODO spelling suggestion should be test with both status of the INCLUDE_SPELLING;
# current is always True
# TODO The [string] field is deprecated, please use [text] or [keyword] instead on [django_ct]
# TODO https://www.elastic.co/guide/en/elasticsearch/reference/5.0/fielddata.html#_fielddata_is_disabled_on_literal_text_literal_fields_by_default
# TODO check timezone awareness of returned datetime objects

class CaptureHandler(std_logging.Handler):
    logs_seen = []

    def emit(self, record):
        CaptureHandler.logs_seen.append(record)

class TestTheSettings(TestCase):

    def setUp(self):
        from haystack_elasticsearch5 import Elasticsearch5SearchBackend

        self.backend = Elasticsearch5SearchBackend('alias', **{
            'URL': 'http://11.22.33.44:3333/', # TODO check https
            'INDEX_NAME': 'whocare',
            'KWARGS': {'max_retries': 42} # TODO check another settings?
        })

    def test_kwargs_are_passed_on(self):

        self.assertEqual(self.backend.conn.transport.max_retries, 42)

    def test_url_are_passed_on(self):

        hosts = self.backend.conn.transport.hosts[0] # we passed parameters for only one connection
        expected = {'scheme': 'http', 'host': '11.22.33.44', 'port': 3333}

        self.assertEqual(hosts, expected)

    def test_alias_are_passed_on(self):

        self.assertEqual(self.backend.connection_alias, 'alias')

    def test_index_name_are_passed_on(self):

        self.assertEqual(self.backend.index_name, 'whocare')

class Elasticsearch5BackendTest(TestCase):

    maxDiff = None

    def setUp(self):

        super(Elasticsearch5BackendTest, self).setUp()

        self.bad_sb = connections['default'].backend('bad', URL='http://omg.wtf.bbq:1000/', INDEX_NAME='whatver', SILENTLY_FAIL=False, TIMEOUT=1)

        # Wipe it clean.
        self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        self.smmidni = ElasticsearchMockSearchIndexWithSkipDocument()
        self.smtmmi = ElasticsearchMaintainTypeMockSearchIndex()

        self.ui.build(indexes=[self.smmi])

        connections['default']._index = self.ui
        self.search_backend = connections['default'].get_backend()

        # Force the backend to rebuild the mapping each time.
        self.search_backend.existing_mapping = {}
        self.search_backend.setup()

        # TODO - factory
        self.sample_objs = []
        for i in range(1, 4):
            mock = MockModel()
            mock.id = i
            mock.author = 'daniel%s' % i
            mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i)
            self.sample_objs.append(mock)

    def tearDown(self):
        connections['default']._index = self.old_ui
        super(Elasticsearch5BackendTest, self).tearDown()
        self.search_backend.silently_fail = True

    def raw_search(self, query):
        try:
            return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])
        except elasticsearch.TransportError:
            return {}

    # TODO mb test logs
    def test_update_non_silent(self):

        with self.assertRaises(ConnectionError):
            self.bad_sb.update(self.smmi, self.sample_objs)

    # TODO mb test logs
    def test_remove_non_silent(self):

        with self.assertRaises(ConnectionError):
            self.bad_sb.remove('test_app.mockmodel.1')

    # TODO mb test logs
    def test_clear_non_silent(self):

        with self.assertRaises(ConnectionError):
            self.bad_sb.clear()

    # TODO mb test logs
    def test_search_non_silent(self):

        with self.assertRaises(ConnectionError):
            self.bad_sb.search('foo')

    def test_update_if_there_is_no_documents_and_silently_fail(self):
        url = settings.HAYSTACK_CONNECTIONS['default']['URL']
        index_name = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
        documents = []

        search_backend = connections['default'].backend('default', URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True)

        self.assertEqual(search_backend.update(self.smmi, documents), None)

    # TODO test logging
    def test_update_if_there_is_no_documents_and_not_silently_fail(self):
        url = settings.HAYSTACK_CONNECTIONS['default']['URL']
        index_name = settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']
        documents = []

        # with self.assertLogs('elasticsearch') as cm:
        # self.assertEqual(cm.output, ['GET http://127.0.0.1:9200/test_backend/_mapping [status:404 request:0.008s]'])

        search_backend = connections['default'].backend('default', URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=False)
        search_backend.update(self.smmi, documents)

        self.assertEqual(search_backend.update(self.smmi, documents), None)

    def test_update(self):

        self.search_backend.update(self.smmi, self.sample_objs)

        # Check what Elasticsearch thinks is there.
        self.assertEqual(self.raw_search('*:*')['hits']['total'], 3)

        raw_documents = sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], key=lambda x: x['id'])

        expected_documents = [
            {
                'django_id': '1',
                'django_ct': 'test_app.mockmodel',
                'name': 'daniel1',
                'name_exact': 'daniel1',
                'text': 'Indexed!\n1\n',
                'pub_date': '2009-02-24T00:00:00',
                'id': 'test_app.mockmodel.1'
            },
            {
                'django_id': '2',
                'django_ct': 'test_app.mockmodel',
                'name': 'daniel2',
                'name_exact': 'daniel2',
                'text': 'Indexed!\n2\n',
                'pub_date': '2009-02-23T00:00:00',
                'id': 'test_app.mockmodel.2'
            },
            {
                'django_id': '3',
                'django_ct': 'test_app.mockmodel',
                'name': 'daniel3',
                'name_exact': 'daniel3',
                'text': 'Indexed!\n3\n',
                'pub_date': '2009-02-22T00:00:00',
                'id': 'test_app.mockmodel.3'
            }
        ]

        self.assertEqual(raw_documents, expected_documents)

    def test_update_with_SkipDocument_raised(self):

        self.search_backend.update(self.smmidni, self.sample_objs)

        # Check what Elasticsearch thinks is there.
        res = self.raw_search('*:*')['hits']

        raw_documents = sorted([x['_source']['id'] for x in res['hits']])
        expected_documents = ['test_app.mockmodel.1', 'test_app.mockmodel.2']

        self.assertEqual(res['total'], 2)
        self.assertListEqual(raw_documents, expected_documents)

    def test_remove(self):

        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*')['hits']['total'], 3)

        self.search_backend.remove(self.sample_objs[0])
        self.assertEqual(self.raw_search('*:*')['hits']['total'], 2)

        raw_documents = sorted([res['_source'] for res in self.raw_search('*:*')['hits']['hits']], key=operator.itemgetter('django_id'))
        expected_documents = [
            {
                'django_id': '2',
                'django_ct': 'test_app.mockmodel',
                'name': 'daniel2',
                'name_exact': 'daniel2',
                'text': 'Indexed!\n2\n',
                'pub_date': '2009-02-23T00:00:00',
                'id': 'test_app.mockmodel.2'
            },
            {
                'django_id': '3',
                'django_ct': 'test_app.mockmodel',
                'name': 'daniel3',
                'name_exact': 'daniel3',
                'text': 'Indexed!\n3\n',
                'pub_date': '2009-02-22T00:00:00',
                'id': 'test_app.mockmodel.3'
            }
        ]

        self.assertEqual(raw_documents, expected_documents)

    def test_remove_succeeds_on_404(self):
        self.search_backend.silently_fail = False
        self.search_backend.remove('test_app.mockmodel.421')

    def test_clear_all_documents(self):

        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3)

        self.search_backend.clear()
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0)

    def test_clear_documents_of_model_that_does_not_have_them(self):

        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3)

        self.search_backend.clear([AnotherMockModel])
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3)

    # TODO
    def test_clear(self):

        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3)

        self.search_backend.clear([MockModel])
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0)

        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 3)

    def test_clean_should_work_if_some_model_documents_does_not_exist(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        self.search_backend.clear([AnotherMockModel, MockModel])
        self.assertEqual(self.raw_search('*:*').get('hits', {}).get('total', 0), 0)

    def test_search_an_empty_string(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search(''), {'hits': 0, 'results': []})

    def test_search_all_documents(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search('*:*')['hits'], 3)
        self.assertEqual(set([result.pk for result in self.search_backend.search('*:*')['results']]), set([u'2', u'1', u'3']))

    def test_search_an_empty_string_with_highlight(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search('', highlight=True), {'hits': 0, 'results': []})

    def test_search_highlight(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        self.assertEqual(
                sorted([result.highlighted[0] for result in self.search_backend.search('Index', highlight=True)['results']]),
                [u'Indexed!\n1\n', u'Indexed!\n2\n', u'Indexed!\n3\n']
         )

    def test_search_highlight_with_custom_tags(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        self.assertEqual(
                sorted([result.highlighted[0] for result in self.search_backend.search('Index', highlight={'pre_tags': ['<start>'],'post_tags': ['</end>']})['results']]),
                [u'<start>Indexed</end>!\n1\n', u'<start>Indexed</end>!\n2\n', u'<start>Indexed</end>!\n3\n']
         )

    def test_spelling_if_spelling_does_not_set(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search('Indx')['hits'], 0)

    def test_spelling1(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search('indaxed')['spelling_suggestion'], 'indexed')

    def test_spelling2(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        suggestion = self.search_backend.search('arf', spelling_query='indexyd')['spelling_suggestion']
        self.assertEqual(suggestion, 'indexed')

    def test_facet_search1(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.search_backend.search('', facets={'name': {}}), {'hits': 0, 'results': []})

    def test_facet_search2(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        response = self.search_backend.search('Index', facets={'name': {}})
        result = sorted(response['facets']['fields']['name'])
        expected = [('daniel1', 1), ('daniel2', 1), ('daniel3', 1)]

        self.assertEqual(result, expected)

    def test_facet_search3(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        results = self.search_backend.search('Index', facets={'name': {}})
        self.assertEqual(self.search_backend.search('', date_facets={'pub_date': {'start_date': datetime.date(2008, 1, 1), 'end_date': datetime.date(2009, 4, 1), 'gap_by': 'month', 'gap_amount': 1}}), {'hits': 0, 'results': []})

    def test_facet_search4(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        results = self.search_backend.search('Index', date_facets={'pub_date': {'start_date': datetime.date(2008, 1, 1), 'end_date': datetime.date(2009, 4, 1), 'gap_by': 'month', 'gap_amount': 1}})
        self.assertEqual(results['facets']['dates']['pub_date'], [(datetime.datetime(2009, 2, 1, 0, 0), 3)])

    def test_facet_search5(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        self.assertEqual(self.search_backend.search('', query_facets=[('name', '[* TO e]')]), {'hits': 0, 'results': []})

    def test_facet_search6(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        results = self.search_backend.search('Index', query_facets=[('name', '[* TO e]')])
        self.assertEqual(results['facets']['buckets'], {u'name': 3})

    def test_narrow1(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        results = self.search_backend.search('Index', narrow_queries=set(['name:daniel1']))
        self.assertEqual(results['hits'], 1)

    def test_narrow2(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        results = self.search_backend.search('Index', query_facets=[('name', '[* TO e]')])
        self.assertEqual(self.search_backend.search('', narrow_queries=set(['name:daniel1'])), {'hits': 0, 'results': []})

    def test_ensure_that_swapping_the_result_class_works(self):

        self.search_backend.update(self.smmi, self.sample_objs)
        swapped = self.search_backend.search(u'index', result_class=MockSearchResult)['results'][0]

        self.assertIsInstance(swapped, MockSearchResult)

    def test_check_the_use_limit_to_registered_models1(self):
        self.search_backend.update(self.smmi, self.sample_objs)

        self.assertEqual(self.search_backend.search('', limit_to_registered_models=False), {'hits': 0, 'results': []})

        result = sorted([result.pk for result in self.search_backend.search('*:*', limit_to_registered_models=False)['results']])
        expected = ['1', '2', '3']

        self.assertEqual(result, expected)

    @override_settings(HAYSTACK_LIMIT_TO_REGISTERED_MODELS=False)
    def test_check_the_use_limit_to_registered_models2(self):

        self.search_backend.update(self.smmi, self.sample_objs)

        self.assertEqual(self.search_backend.search(''), {'hits': 0, 'results': []})

        result = sorted([result.pk for result in self.search_backend.search('*:*')['results']])
        expected = ['1', '2', '3']

        self.assertEqual(result, expected)

    # TODO
    # def test_spatial_search_parameters(self):
    #     p1 = Point(1.23, 4.56)
    #     kwargs = self.search_backend.build_search_kwargs(
    #                                             '*:*',
    #                                             distance_point={'field': 'location', 'point': p1},
    #                                             sort_by=(('distance', 'desc'), )
    #                                          )

    #     self.assertIn('sort', kwargs)
    #     self.assertEqual(1, len(kwargs['sort']))
    #     geo_d = kwargs['sort'][0]['_geo_distance']

    #     # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be
    #     # in the same order as we used to create the Point():
    #     # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4

    #     self.assertDictEqual(geo_d, {'location': [1.23, 4.56], 'unit': 'km', 'order': 'desc'})

    def test_more_like_this(self):
        self.search_backend.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*')['hits']['total'], 3)

        self.assertEqual(self.search_backend.more_like_this(self.sample_objs[0])['hits'], 0)
        self.assertEqual([result.pk for result in self.search_backend.more_like_this(self.sample_objs[0])['results']], [])

    def test_build_schema_from_unified_index(self):

        unified_index = connections['default'].get_unified_index()

        content_field_name, current_mapping = self.search_backend.build_schema(unified_index.all_searchfields())

        expected_mapping = {
            'django_id': {'include_in_all': False, 'index': 'not_analyzed', 'type': 'text'},
            'django_ct': {'include_in_all': False, 'index': 'not_analyzed', 'type': 'text'},
            'text': {'analyzer': 'snowball', 'fielddata': True, 'type': 'text'},
            'pub_date': {'type': 'date'},
            'name': {'analyzer': 'snowball', 'fielddata': True, 'type': 'text'},
            'name_exact': {'fielddata': True, 'index': 'not_analyzed', 'type': 'text'},
         }

        self.assertEqual(content_field_name, 'text')
        self.assertEqual(current_mapping, expected_mapping)

    def test_build_schema_from_custom_unified_index(self):

        custom_ui = UnifiedIndex()
        custom_ui.build(indexes=[ElasticsearchComplexFacetsMockSearchIndex()])

        content_field_name, current_mapping = self.search_backend.build_schema(custom_ui.all_searchfields())

        self.assertEqual(content_field_name, 'text')
        expected_mapping = {
            'django_id': {'index': 'not_analyzed', 'type': 'text', 'include_in_all': False},
            'django_ct': {'index': 'not_analyzed', 'type': 'text', 'include_in_all': False},
            'name': {'type': 'text', 'analyzer': 'snowball', 'fielddata': True},
            'is_active_exact': {'type': 'boolean'},
            'created': {'type': 'date'},
            'post_count': {'type': 'long'},
            'created_exact': {'type': 'date'},
            'sites_exact': {'index': 'not_analyzed', 'type': 'text', 'fielddata': True},
            'is_active': {'type': 'boolean'},
            'sites': {'type': 'text', 'analyzer': 'snowball', 'fielddata': True},
            'post_count_i': {'type': 'long'},
            'average_rating': {'type': 'float'},
            'text': {'type': 'text', 'analyzer': 'snowball', 'fielddata': True},
            'pub_date_exact': {'type': 'date'},
            'name_exact': {'index': 'not_analyzed', 'type': 'text', 'fielddata': True},
            'pub_date': {'type': 'date'},
            'average_rating_exact': {'type': 'float'}
        }
        self.assertEqual(current_mapping, expected_mapping)

    # XXX what for?
    def test_verify_type(self):

        new_unified_index = UnifiedIndex()
        smtmmi = ElasticsearchMaintainTypeMockSearchIndex()
        new_unified_index.build(indexes=[smtmmi]) # populate index

        connections['default']._index = new_unified_index
        search_backend = connections['default'].get_backend()

        search_backend.update(smtmmi, self.sample_objs)

        results = [result['_source']['month'] for result in self.raw_search('*:*')['hits']['hits']]

        self.assertEqual(results, [u'02', u'02', u'02'])

class FailedElasticsearchSearchBackendTestCase(TestCase):

    def setUp(self):
        self.sample_objs = []

        for i in range(1, 4):
            mock = MockModel()
            mock.id = i
            mock.author = 'daniel%s' % i
            mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i)
            self.sample_objs.append(mock)

        # Stow.
        # Point the backend at a URL that doesn't exist so we can watch the sparks fly.
        self.old_es_url = settings.HAYSTACK_CONNECTIONS['default']['URL']
        settings.HAYSTACK_CONNECTIONS['default']['URL'] = "%s/foo/" % self.old_es_url
        self.cap = CaptureHandler()
        logging.getLogger('haystack').addHandler(self.cap)
        config = apps.get_app_config('haystack')
        logging.getLogger('haystack').removeHandler(config.stream)

        # Setup the rest of the bits.
        self.old_ui = connections['default'].get_unified_index()
        ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        ui.build(indexes=[self.smmi])
        connections['default']._index = ui
        self.sb = connections['default'].get_backend()

    def tearDown(self):
        # Restore.
        settings.HAYSTACK_CONNECTIONS['default']['URL'] = self.old_es_url
        connections['default']._index = self.old_ui
        config = apps.get_app_config('haystack')
        logging.getLogger('haystack').removeHandler(self.cap)
        logging.getLogger('haystack').addHandler(config.stream)

    @unittest.expectedFailure
    def test_all_cases(self):
        # Prior to the addition of the try/except bits, these would all fail miserably.
        self.assertEqual(len(CaptureHandler.logs_seen), 0)

        self.sb.update(self.smmi, self.sample_objs)
        self.assertEqual(len(CaptureHandler.logs_seen), 1)

        self.sb.remove(self.sample_objs[0])
        self.assertEqual(len(CaptureHandler.logs_seen), 2)

        self.sb.search('search')
        self.assertEqual(len(CaptureHandler.logs_seen), 3)

        self.sb.more_like_this(self.sample_objs[0])
        self.assertEqual(len(CaptureHandler.logs_seen), 4)

        self.sb.clear([MockModel])
        self.assertEqual(len(CaptureHandler.logs_seen), 5)

        self.sb.clear()
        self.assertEqual(len(CaptureHandler.logs_seen), 6)

class LiveElasticsearchSearchQueryTestCase(TestCase):
    fixtures = ['base_data.json']

    def setUp(self):
        super(LiveElasticsearchSearchQueryTestCase, self).setUp()

        # Wipe it clean.
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui
        self.sb = connections['default'].get_backend()
        self.sq = connections['default'].get_query()

        # Force indexing of the content.
        self.smmi.update(using='default')

    def tearDown(self):
        connections['default']._index = self.old_ui
        super(LiveElasticsearchSearchQueryTestCase, self).tearDown()

    def test_log_query(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)

        with self.settings(DEBUG=False):
            len(self.sq.get_results())
            self.assertEqual(len(connections['default'].queries), 0)

        with self.settings(DEBUG=True):
            # Redefine it to clear out the cached results.
            self.sq = connections['default'].query(using='default')
            self.sq.add_filter(SQ(name='bar'))
            len(self.sq.get_results())
            self.assertEqual(len(connections['default'].queries), 1)
            self.assertEqual(connections['default'].queries[0]['query_string'],
                             'name:(bar)')

            # And again, for good measure.
            self.sq = connections['default'].query('default')
            self.sq.add_filter(SQ(name='bar'))
            self.sq.add_filter(SQ(text='moof'))
            len(self.sq.get_results())
            self.assertEqual(len(connections['default'].queries), 2)
            self.assertEqual(connections['default'].queries[0]['query_string'],
                             'name:(bar)')
            self.assertEqual(connections['default'].queries[1]['query_string'],
                             u'(name:(bar) AND text:(moof))')

lssqstc_all_loaded = None

@override_settings(DEBUG=True)
class LiveElasticsearchSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['base_data.json', 'bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet('default')
        self.rsqs = RelatedSearchQuerySet('default')

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_elasticsearch_index()

            # Force indexing of the content.
            self.smmi.update(using='default')

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.order_by('pub_date').load_all()

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)

        expected = u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.'

        self.assertEqual(sqs[2].object.foo, expected)

    def test_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        results = sorted([int(result.pk) for result in list(sqs)])
        self.assertEqual(results, list(range(1, 24)))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all().order_by('pub_date')
        self.assertEqual([int(result.pk) for result in results[1:11]], [3, 2, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all().order_by('pub_date')
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 1)

    def test_values_slicing(self):
        reset_search_queries()

        self.assertEqual(len(connections['default'].queries), 0)

        # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends

        # The values will come back as strings because Hasytack doesn't assume PKs are integers.
        # We'll prepare this set once since we're going to query the same results in multiple ways:
        expected_pks = [str(item) for item in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]]

        results = self.sqs.all().order_by('pub_date').values('pk')
        self.assertListEqual([item['pk'] for item in results[1:11]], expected_pks)

        results = self.sqs.all().order_by('pub_date').values_list('pk')
        self.assertListEqual([item[0] for item in results[1:11]], expected_pks)

        results = self.sqs.all().order_by('pub_date').values_list('pk', flat=True)
        self.assertListEqual(results[1:11], expected_pks)

        self.assertEqual(len(connections['default'].queries), 3)

    def test_count(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(connections['default'].queries), 1)

    def test_highlight(self):
        reset_search_queries()

        sqs = self.sqs.filter(content='index').highlight()

        expected = \
            [['Indexed!\n1\n'],
             ['Indexed!\n10\n'],
             ['Indexed!\n11\n'],
             ['Indexed!\n12\n'],
             ['Indexed!\n13\n'],
             ['Indexed!\n14\n'],
             ['Indexed!\n15\n'],
             ['Indexed!\n16\n'],
             ['Indexed!\n17\n'],
             ['Indexed!\n18\n'],
             ['Indexed!\n19\n'],
             ['Indexed!\n2\n'],
             ['Indexed!\n20\n'],
             ['Indexed!\n21\n'],
             ['Indexed!\n22\n'],
             ['Indexed!\n23\n'],
             ['Indexed!\n3\n'],
             ['Indexed!\n4\n'],
             ['Indexed!\n5\n'],
             ['Indexed!\n6\n'],
             ['Indexed!\n7\n'],
             ['Indexed!\n8\n'],
             ['Indexed!\n9\n']]

        result = [item.highlighted for item in sqs]
        result.sort()

        self.assertEqual(result, expected)

    def test_highlight_options(self):
        reset_search_queries()

        sqs = self.sqs.filter(content='index')
        sqs = sqs.highlight(pre_tags=[''], post_tags=[''])

        expected = \
            [['Indexed!\n1\n'],
             ['Indexed!\n10\n'],
             ['Indexed!\n11\n'],
             ['Indexed!\n12\n'],
             ['Indexed!\n13\n'],
             ['Indexed!\n14\n'],
             ['Indexed!\n15\n'],
             ['Indexed!\n16\n'],
             ['Indexed!\n17\n'],
             ['Indexed!\n18\n'],
             ['Indexed!\n19\n'],
             ['Indexed!\n2\n'],
             ['Indexed!\n20\n'],
             ['Indexed!\n21\n'],
             ['Indexed!\n22\n'],
             ['Indexed!\n23\n'],
             ['Indexed!\n3\n'],
             ['Indexed!\n4\n'],
             ['Indexed!\n5\n'],
             ['Indexed!\n6\n'],
             ['Indexed!\n7\n'],
             ['Indexed!\n8\n'],
             ['Indexed!\n9\n']]

        result = [item.highlighted for item in sqs]
        result.sort()

        self.assertEqual(result, expected)

    def test_manual_iter(self):
        results = self.sqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = set([int(result.pk) for result in results._manual_iter()])
        self.assertEqual(results, set([2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = list(results)
        self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 4)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.

        sqs = self.sqs.auto_query('"pants:rule"')
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
        self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")')
        self.assertEqual(len(sqs), 0)

    def test_query__in(self):
        self.assertGreater(len(self.sqs), 0)

        sqs = self.sqs.filter(django_ct='test_app.mockmodel', django_id__in=[1, 2])
        self.assertEqual(len(sqs), 2)

    def test_query__in_empty_list(self):
        """Confirm that an empty list avoids a Elasticsearch exception"""
        self.assertGreater(len(self.sqs), 0)
        sqs = self.sqs.filter(id__in=[])
        self.assertEqual(len(sqs), 0)

    # Regressions
    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail("Result with id '%s' seen more than once in the results." % key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text:index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail("Result with id '%s' seen more than once in the results." % pk)

    # RelatedSearchQuerySet Tests
    def test_related_load_all(self):
        sqs = self.rsqs.order_by('pub_date').load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[2].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all().order_by('pub_date')
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24)))

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual(set([obj.object.id for obj in sqs]), set([12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20]))
        self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), set([21, 22, 23]))

    def test_related_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.rsqs.all()
        results = set([int(result.pk) for result in list(sqs)])
        self.assertEqual(results, set([2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all().order_by('pub_date')
        self.assertEqual([int(result.pk) for result in results[1:11]], [3, 2, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all().order_by('pub_date')
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all().order_by('pub_date')
        self.assertEqual(set([int(result.pk) for result in results[20:30]]), set([21, 22, 23]))
        self.assertEqual(len(connections['default'].queries), 1)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = sorted([int(result.pk) for result in results._manual_iter()])
        self.assertEqual(results, list(range(1, 24)))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_related_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_related_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = list(results)
        self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 4)

    def test_quotes_regression(self):
        sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
        # Should not have empty terms.
        self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)")
        # Should not cause Elasticsearch to 500.
        self.assertEqual(sqs.count(), 0)

        sqs = self.sqs.auto_query('blazing')
        self.assertEqual(sqs.query.build_query(), u'(blazing)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)')
        self.assertEqual(sqs.count(), 0)

    def test_query_generation(self):
        sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")))
        self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))")

    def test_result_class(self):
        # Assert that we're defaulting to ``SearchResult``.
        sqs = self.sqs.all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

        # Custom class.
        sqs = self.sqs.result_class(MockSearchResult).all()
        self.assertTrue(isinstance(sqs[0], MockSearchResult))

        # Reset to default.
        sqs = self.sqs.result_class(None).all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

@override_settings(DEBUG=True)
class LiveElasticsearchSpellingTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['base_data.json', 'bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchSpellingTestCase, self).setUp()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSpellingIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet('default')

        # Wipe it clean.
        clear_elasticsearch_index()

        # Reboot the schema.
        self.sb = connections['default'].get_backend()
        self.sb.setup()

        self.smmi.update(using='default')

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchSpellingTestCase, self).tearDown()

    def test_spelling(self):
        self.assertEqual(self.sqs.auto_query('structurd').spelling_suggestion(), 'structured')
        self.assertEqual(self.sqs.spelling_suggestion('structurd'), 'structured')
        self.assertEqual(self.sqs.auto_query('srchindex instanc').spelling_suggestion(), 'searchindex instance')
        self.assertEqual(self.sqs.spelling_suggestion('srchindex instanc'), 'searchindex instance')

        sqs = self.sqs.auto_query('something completely different').set_spelling_query('structurd')
        self.assertEqual(sqs.spelling_suggestion(), 'structured')

class LiveElasticsearchMoreLikeThisTestCase(TestCase):
    fixtures = ['base_data.json', 'bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchMoreLikeThisTestCase, self).setUp()

        # Wipe it clean.
        clear_elasticsearch_index()

        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockModelSearchIndex()
        self.sammi = ElasticsearchAnotherMockModelSearchIndex()
        self.ui.build(indexes=[self.smmi, self.sammi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet('default')

        self.smmi.update(using='default')
        self.sammi.update(using='default')

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchMoreLikeThisTestCase, self).tearDown()

    def test_simple_more_like_this(self):

        mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1))
        self.assertEqual(set([result.pk for result in mlt]), set([u'2', u'6', u'16', u'23']))

    def test_more_like_this_with_filtering(self):

        mlt = self.sqs.filter(name='daniel3').more_like_this(MockModel.objects.get(pk=2))
        self.assertEqual(set([result.pk for result in mlt]), set([u'2', u'6', u'16', u'23', u'1', u'11']))

    def test_more_like_this_within_set_of_a_model(self):

        mlt = self.sqs.models(MockModel).more_like_this(MockModel.objects.get(pk=1))
        self.assertEqual(set([result.pk for result in mlt]), set([u'2', u'6', u'16', u'23']))

    # TODO does this decorator realy need?
    @unittest.skipUnless(hasattr(MockModel.objects, 'defer'), "Models should support deferring")
    def test_more_like_this_with_deferred_models(self):

        mi = MockModel.objects.defer('foo').get(pk=1)

        deferred = self.sqs.models(MockModel).more_like_this(mi)

        self.assertEqual(deferred.count(), 0)
        self.assertEqual([result.pk for result in deferred], [])
        self.assertEqual(len([result.pk for result in deferred]), 0)

    def test_more_like_this_ensure_that_swapping_the_result_class_works(self):

        instance = self.sqs.result_class(MockSearchResult).more_like_this(MockModel.objects.get(pk=1))[0]
        self.assertIsInstance(instance, MockSearchResult)

class LiveElasticsearchAutocompleteTestCase(TestCase):
    fixtures = ['base_data.json', 'bulk_data.json']

    maxDiff = None

    def setUp(self):
        super(LiveElasticsearchAutocompleteTestCase, self).setUp()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchAutocompleteMockModelSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet('default')

        # Wipe it clean.
        clear_elasticsearch_index()

        # Reboot the schema.
        self.sb = connections['default'].get_backend()
        self.sb.setup()

        self.smmi.update(using='default')

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchAutocompleteTestCase, self).tearDown()

    def test_build_schema(self):
        self.sb = connections['default'].get_backend()

        content_name, mapping = self.sb.build_schema(self.ui.all_searchfields())

        expected_mapping = \
        {
            'django_id': {'index': 'not_analyzed', 'type': 'text', 'include_in_all': False},
            'django_ct': {'index': 'not_analyzed', 'type': 'text', 'include_in_all': False},
            'name_auto': {
                'type': 'text',
                'analyzer': 'edgengram_analyzer',
            },
            'text': {
                'type': 'text',
                'analyzer': 'snowball',
                'fielddata': True,
            },
            'pub_date': {
                'type': 'date'
            },
            'name': {
                'type': 'text',
                'analyzer': 'snowball',
                'fielddata': True,
            },
            'text_auto': {
                'type': 'text',
                'analyzer': 'edgengram_analyzer',
            }
        }

        self.assertEqual(mapping, expected_mapping)

    def test_autocomplete(self):
        autocomplete = self.sqs.autocomplete(text_auto='mod')
        self.assertEqual(autocomplete.count(), 16)

        result = set([result.pk for result in autocomplete])
        expected = set(['1', '12', '6', '14', '7', '4', '23', '17', '13', '18', '20', '22', '19', '15', '10', '2'])
        self.assertEqual(result, expected)

        self.assertTrue('mod' in autocomplete[0].text.lower())
        self.assertTrue('mod' in autocomplete[1].text.lower())
        self.assertTrue('mod' in autocomplete[2].text.lower())
        self.assertTrue('mod' in autocomplete[3].text.lower())
        self.assertTrue('mod' in autocomplete[4].text.lower())
        self.assertEqual(len([result.pk for result in autocomplete]), 16)

        # Test multiple words.
        autocomplete_2 = self.sqs.autocomplete(text_auto='your mod')
        self.assertEqual(autocomplete_2.count(), 13)

        result = set([result.pk for result in autocomplete_2])
        expected = set(['1', '6', '2', '14', '12', '13', '10', '19', '4', '20', '23', '22', '15'])
        self.assertEqual(result, expected)
        self.assertTrue('your' in autocomplete_2[0].text.lower())
        self.assertTrue('mod' in autocomplete_2[0].text.lower())
        self.assertTrue('your' in autocomplete_2[1].text.lower())
        self.assertTrue('mod' in autocomplete_2[1].text.lower())
        self.assertTrue('your' in autocomplete_2[2].text.lower())
        self.assertEqual(len([result.pk for result in autocomplete_2]), 13)

        # Test multiple fields.
        autocomplete_3 = self.sqs.autocomplete(text_auto='Django', name_auto='dan')
        self.assertEqual(autocomplete_3.count(), 4)
        self.assertEqual(set([result.pk for result in autocomplete_3]), set(['12', '1', '22', '14']))
        self.assertEqual(len([result.pk for result in autocomplete_3]), 4)

        # Test numbers in phrases
        autocomplete_4 = self.sqs.autocomplete(text_auto='Jen 867')
        self.assertEqual(autocomplete_4.count(), 1)
        self.assertEqual(set([result.pk for result in autocomplete_4]), set(['20']))

        # Test numbers alone
        autocomplete_4 = self.sqs.autocomplete(text_auto='867')
        self.assertEqual(autocomplete_4.count(), 1)
        self.assertEqual(set([result.pk for result in autocomplete_4]), set(['20']))

class LiveElasticsearchRoundTripTestCase(TestCase):
    def setUp(self):
        super(LiveElasticsearchRoundTripTestCase, self).setUp()

        # Wipe it clean.
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.srtsi = ElasticsearchRoundTripSearchIndex()
        self.ui.build(indexes=[self.srtsi])
        connections['default']._index = self.ui
        self.sb = connections['default'].get_backend()

        self.sqs = SearchQuerySet('default')

        # Fake indexing.
        mock = MockModel()
        mock.id = 1
        self.sb.update(self.srtsi, [mock])

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchRoundTripTestCase, self).tearDown()

    def test_round_trip(self):
        results = self.sqs.filter(id='test_app.mockmodel.1')

        # Sanity check.
        self.assertEqual(results.count(), 1)

        # Check the individual fields.
        result = results[0]
        self.assertEqual(result.id, 'test_app.mockmodel.1')
        self.assertEqual(result.text, 'This is some example text.')
        self.assertEqual(result.name, 'Mister Pants')
        self.assertEqual(result.is_active, True)
        self.assertEqual(result.post_count, 25)
        self.assertEqual(result.average_rating, 3.6)
        self.assertEqual(result.price, u'24.99')
        self.assertEqual(result.pub_date, datetime.date(2009, 11, 21))
        self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00))
        self.assertEqual(result.tags, ['staff', 'outdoor', 'activist', 'scientist'])
        self.assertEqual(result.sites, [3, 5, 1])

@unittest.skipUnless(test_pickling, 'Skipping pickling tests')
class LiveElasticsearchPickleTestCase(TestCase):
    fixtures = ['base_data.json', 'bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchPickleTestCase, self).setUp()

        # Wipe it clean.
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockModelSearchIndex()
        self.sammi = ElasticsearchAnotherMockModelSearchIndex()
        self.ui.build(indexes=[self.smmi, self.sammi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet('default')

        self.smmi.update(using='default')
        self.sammi.update(using='default')

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        super(LiveElasticsearchPickleTestCase, self).tearDown()

    def test_pickling(self):
        results = self.sqs.all()

        for res in results:
            # Make sure the cache is full.
            pass

        in_a_pickle = pickle.dumps(results)
        like_a_cuke = pickle.loads(in_a_pickle)
        self.assertEqual(len(like_a_cuke), len(results))
        self.assertEqual(like_a_cuke[0].id, results[0].id)

class ElasticsearchBoostBackendTestCase(TestCase):
    def setUp(self):
        super(ElasticsearchBoostBackendTestCase, self).setUp()

        # Wipe it clean.
        self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchBoostMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui
        self.sb = connections['default'].get_backend()

        self.sample_objs = []

        for i in range(1, 5):
            mock = AFourthMockModel()
            mock.id = i

            if i % 2:
                mock.author = 'daniel'
                mock.editor = 'david'
            else:
                mock.author = 'david'
                mock.editor = 'daniel'

            mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i)
            self.sample_objs.append(mock)

    def tearDown(self):
        connections['default']._index = self.old_ui
        super(ElasticsearchBoostBackendTestCase, self).tearDown()

    def raw_search(self, query):
        return self.raw_es.search(q='*:*', index=settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME'])

    def test_boost(self):
        self.sb.update(self.smmi, self.sample_objs)
        self.assertEqual(self.raw_search('*:*')['hits']['total'], 4)

        results = SearchQuerySet(using='default').filter(SQ(author='daniel') | SQ(editor='daniel'))

        self.assertEqual(set([result.id for result in results]), set([
            'test_app.afourthmockmodel.4',
            'test_app.afourthmockmodel.3',
            'test_app.afourthmockmodel.1',
            'test_app.afourthmockmodel.2'
        ]))

    def test__to_python(self):
        self.assertEqual(self.sb._to_python('abc'), 'abc')
        self.assertEqual(self.sb._to_python('1'), 1)
        self.assertEqual(self.sb._to_python('2653'), 2653)
        self.assertEqual(self.sb._to_python('25.5'), 25.5)
        self.assertEqual(self.sb._to_python('[1, 2, 3]'), [1, 2, 3])
        self.assertEqual(self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {'a': 1, 'c': 3, 'b': 2})
        self.assertEqual(self.sb._to_python('2009-05-09T16:14:00'), datetime.datetime(2009, 5, 9, 16, 14))
        self.assertEqual(self.sb._to_python('2009-05-09T00:00:00'), datetime.datetime(2009, 5, 9, 0, 0))
        self.assertEqual(self.sb._to_python(None), None)

class RecreateIndexTestCase(TestCase):

    def setUp(self):
        self.raw_es = elasticsearch.Elasticsearch(settings.HAYSTACK_CONNECTIONS['default']['URL'])

    def test_recreate_index(self):
        """
        django.core.exceptions.ImproperlyConfigured:
            Model '<class 'tests.test_app.models.MockModel'>' has more than one 'SearchIndex`` handling it.
            Please exclude either '<tests.test_app.search_indexes.ElasticsearchAutocompleteMockModelSearchIndex object at 0x10b7881c8>'
                or
            '<tests.test_app.search_indexes.ElasticsearchComplexFacetsMockSearchIndex object at 0x10b788228>'
            using the 'EXCLUDED_INDEXES' setting defined in 'settings.HAYSTACK_CONNECTIONS'.
        """

        clear_elasticsearch_index()

        search_backend = connections['default'].get_backend()
        search_backend.silently_fail = True
        search_backend.setup()

        original_mapping = self.raw_es.indices.get_mapping(index=search_backend.index_name)

        search_backend.clear()
        search_backend.setup()

        try:
            updated_mapping = self.raw_es.indices.get_mapping(search_backend.index_name)
        except elasticsearch.NotFoundError:
            self.fail("There is no mapping after recreating the index")

        self.assertEqual(original_mapping, updated_mapping,
                         "Mapping after recreating the index differs from the original one")

class ElasticsearchFacetingTestCase(TestCase):

    def setUp(self):
        super(ElasticsearchFacetingTestCase, self).setUp()

        # Wipe it clean.
        clear_elasticsearch_index()

        # Stow.
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchFacetingMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui
        self.sb = connections['default'].get_backend()

        # Force the backend to rebuild the mapping each time.
        self.sb.existing_mapping = {}
        self.sb.setup()

        self.sample_objs = []

        for i in range(1, 10):
            mock = AFourthMockModel()
            mock.id = i
            if i > 5:
                mock.editor = 'George Taylor'
            else:
                mock.editor = 'Perry White'

            if i % 2:
                mock.author = 'Daniel Lindsley'
            else:
                mock.author = 'Dan Watson'

            mock.pub_date = datetime.date(2013, 9, (i % 4) + 1)
            self.sample_objs.append(mock)

    def tearDown(self):
        connections['default']._index = self.old_ui
        super(ElasticsearchFacetingTestCase, self).tearDown()

    # TODO https://www.elastic.co/guide/en/elasticsearch/reference/5.0/fielddata.html#_fielddata_is_disabled_on_literal_text_literal_fields_by_default
    # TODO decision needed
    @unittest.expectedFailure
    def test_facet(self):
        self.sb.update(self.smmi, self.sample_objs)
        counts = SearchQuerySet('default').facet('author').facet('editor').facet_counts()

        self.assertEqual(counts['fields']['author'], [
            ('Daniel Lindsley', 5),
            ('Dan Watson', 4),
        ])
        self.assertEqual(counts['fields']['editor'], [
            ('Perry White', 5),
            ('George Taylor', 4),
        ])

        counts = SearchQuerySet('default').filter(content='white').facet('facet_field', order='reverse_count').facet_counts()

        self.assertEqual(counts['fields']['facet_field'], [
            ('Dan Watson', 2),
            ('Daniel Lindsley', 3),
        ])

    # TODO https://www.elastic.co/guide/en/elasticsearch/reference/5.0/fielddata.html#_fielddata_is_disabled_on_literal_text_literal_fields_by_default
    # TODO decision needed
    @unittest.expectedFailure
    def test_multiple_narrow(self):
        self.sb.update(self.smmi, self.sample_objs)
        counts = SearchQuerySet('default').narrow('editor_exact:"Perry White"').narrow('author_exact:"Daniel Lindsley"').facet('author').facet_counts()

        self.assertEqual(counts['fields']['author'], [ ('Daniel Lindsley', 3), ])

    # TODO https://www.elastic.co/guide/en/elasticsearch/reference/5.0/fielddata.html#_fielddata_is_disabled_on_literal_text_literal_fields_by_default
    # TODO decision needed
    @unittest.expectedFailure
    def test_narrow(self):
        self.sb.update(self.smmi, self.sample_objs)

        counts = SearchQuerySet('default').facet('author').facet('editor').narrow('editor_exact:"Perry White"').facet_counts()

        self.assertEqual(counts['fields']['author'], [
            ('Daniel Lindsley', 3),
            ('Dan Watson', 2),
        ])

        self.assertEqual(counts['fields']['editor'], [
            ('Perry White', 5),
        ])

    def test_date_facet(self):
        self.sb.update(self.smmi, self.sample_objs)

        start = datetime.date(2013, 9, 1)
        end = datetime.date(2013, 9, 30)

        # Facet by day
        counts = SearchQuerySet('default').date_facet('pub_date', start_date=start, end_date=end, gap_by='day').facet_counts()

        expected = \
            [
                (datetime.datetime(2013, 9, 1), 2),
                (datetime.datetime(2013, 9, 2), 3),
                (datetime.datetime(2013, 9, 3), 2),
                (datetime.datetime(2013, 9, 4), 2),
            ]

        self.assertEqual(sorted(counts['dates']['pub_date']), expected)
        # By month
        counts = SearchQuerySet('default').date_facet('pub_date', start_date=start, end_date=end, gap_by='month').facet_counts()

        expected = \
            [
                (datetime.datetime(2013, 9, 1), 9),
            ]

        self.assertEqual(counts['dates']['pub_date'], expected)


text:
<!doctype html>

        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1">

        <title>KTRA indexer - login</title>

        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.1/dist/css/bootstrap.min.css" rel="stylesheet"
              integrity="sha384-iYQeCzEYFbKjA/T2uDLTpkwGzCiq6soy8tYaI1GyVh/UjpbCx/TYkiZhlZB6+fzT" crossorigin="anonymous">
        <link href="css/login.css" rel="stylesheet">

                KTRA indexer

            <form method="post" action="login">

Please sign in

                    <input type="text" class="form-control" id="username" name="username" placeholder="Username" required autofocus>
                    Username

                <button class="btn btn-lg btn-primary btn-block" type="button" id="webAuthnLogin">WebAuthn login</button>
            </form>

redpajama_set_name: RedPajamaGithub


text:
This is a Visual Studio C# project that provides a GUI for filtering and reporting on data within XML files generated using the csexport.exe utility that is included with MIM/Azure AD Connect.  Reports can be made in HTML (quick and dirty HTML table), CSV, and Excel.

Although not required, it is capable of executing csexport.exe on your behalf if ran on the server where MIM Sync/Azure AD Connect is installed.  It uses Registry Keys to determine the install location in order to locate csexport.exe.

**NOTE:  csexport.exe is included with Azure AD Connect.**

## Recent Updates

**\~\~Import Error Support\~\~**

# How-Tos

## Create connector space file
The connector space file is created using the csexport.exe command-line tool included with the Sync Engine installation.  It is typically located in the 'Synchronization Service\Bin' directory under the installation folder.  Azure AD Connect uses a different path.

**Manually create connector space file**

Official documentation for the csexport tool can be found at <https://docs.microsoft.com/en-us/previous-versions/mim/jj590346(v=ws.10)>.  Simple usage examples can be seen by clicking on the 'csexport examples' button as shown below.

![](images/GetDataExamples.png)
![](images/Examples.png)

**Automatically generate connector space file**

**NOTE:**  This requires the CSReporter tool to be run on the server where the Sync Engine is installed.  This is detected by using Registry Keys to determine the install location in order to locate csexport.exe.

Select the 'Generate file' option and the name of the management agent to report on from the drop down menu.  In the Data Selection, pick the type of data to include in the export file.  Click 'Generate File' to pick the name\location of the connector space file and launch the csexport.exe tool.

![](images/GetData_2.png)
![](images/csexport.png)

## Filtering

**Filtering works from left (Hologram selection) to right (Attribute value matching).**  Once a file has been selected and parsed, the Filter screen will be shown.

![](images/Filter.png)

From here select the name of a Hologram and see the 'Object Types' box dynamically display the available types.  Also notice the 'Matching Count' in the bottom left of the screen is updated with the number of objects with the selected hologram.

Once a Hologram is selected, all available filtering options to the right (Object Types, Operations, Attributes) are dynamically updated.  In order for a filtering option to become available, all filtering options to the left must be selected.  For example, the Attributes drop-down isn't populated until Hologram, Object Types, Operations are selected.  Similarly, the Operations box is empty until both Hologram and Object Types are selected.

**NOTE:** The Operations box will not be populated when the Synchronized hologram is selected.  Only the Hologram and Object Types options are required for the Attributes drop down to be populated.

![](images/Filter_3.png)

## Reporting

Reports can be generated by clicking on the 'Create Report' button.  The Report Settings screen will appear with options for the type of report to generate.  Select the attributes to include in the report and the report type.  Click OK to select name\path of the report.

![](images/ReportSettings.png)

Report Types
- HTML - Quick and dirty table.  Browsers may hang on large reports

![](images/HTML.png)

- CSV - Standard comma delemited file made for importing into Excel where additional filtering can be completed

![](images/CSV.png)

- Excel - Native Excel file that allows for additional filtering
  - Horizontal - A CSV\table-like layout with attributes listed across the top from left to right.

![](images/ExcelReport.png)

  - Vertical - Similar to the HTML table, attributes are listed underneath each anchor from top to bottom

![](images/ExcelVertical.png)

## Advanced Features

**Contains Active Directory data**

This checkbox can be used when the data comes from an Active Directory Management Agent.  It will translate the below AD values into human readable data.  **This can be used for both filtering and reports**

Examples:

- accountExpires
  - 0x7fffffffffffffff to Never
- objectSid
  - AQUAAAAAAAUVAAAApDhY7yZmbpJEbWYeigQAAA== to S-1-5-21-4015536292-2456708646-510029124-1162
- pwdLastSet
  - 0x1d098fc86f51e23 to 5/28/2015 4:12 AM
- groupType
  - 0xffffffff80000002 to -2147483646
- userAccountControl
  - 0x200 to 512
- createTimeStamp
  - 20010928060000.0Z to 9/28/2001 6:00 AM

The below pictures show filtering on the userAccountControl attribute and how the 'Contains Active Directory data' checkbox can be used when filtering.

![](images/ADData.png)
![](images/ADData_2.png)

**Filter on non-changing attributes**

The purpose of this is to allow reporting of changes by using an Attribute filter based on Synchronized values.  For example, finding all users with pending import\export changes where a specific Department value is not chaning (i.e. exists only in Sync hologram).
**NOTE: As the warning states, any filters applied at the Attribute level will be applied ONLY to Attributes in the Sync hologram.**

![](images/Warning.png)

**Include system attributes**

This allows filtering on object meta-data present in the connector space file such as Connector State.

Current supported system attributes
- DN (when not changing)
- Connector State
- Disconnect Time
- Connector Operation
- Connect Time
- Connector
- Export Error Details

![](images/SystemAttributes.png)

**Note:** The connector space file must contain this meta-data.  If the csexport option /o is used, it will be excluded.  When generating files using CSReporter, the System Data box must be checked on the Get Data form.
**NOTE:** Generating a connector space file with this additional data will be much slower as much more data is written to the XML file.

![](images/GetData_3.png)

## Large Files
Connector space files generated with csexport will vary in size greating depending on the environment.  I have generated files over 5GB in size (and generated reports using CSReporter).  In the countless hours of development and usage of CSReporter, generally the amount of memory (RAM) required is 2.5-4x the size of the connector space file on disk.  This means that a 300MB file could require upwards of 1.2GB of available memory. Not all systems have this much unused for CSReporter to run.  Thus for files larger than **150MB**, CSReporter does not keep an in-memory representation of the connector space file.

**For files larger than 150MB, each time the filter is adjusted, the file is parsed line by line from disk.**

This allows for CSReporter to support large files, without the need for large amounts of memory.  It allows for CSReporter to be used in more environments, such as Production, without creating memory issues.


text:
<!doctype html>

    <title>socket.io client test</title>

Sample chat client

Connecting...

    <form id="form" onSubmit="send(); return false">
      <input type="text" autocomplete="off" id="text"><input type="submit" value="Send">
    </form>


text:
module Mimsy
  module ACL
    class Base
      include Cannot

      def initialize(request)
        @request = request
      end

      def can?(action_name, resource_name)
        return false unless resources.has_key?(resource_name.to_sym)

        resource = resources[resource_name.to_sym]
        context  = create_context(resource)

        resource.can?(context, action_name.to_sym)
      end

      def self.describe(resource_name, &block)
        resources[resource_name] = Resource.new(resource_name, &block)
      end

      private

        def create_context(resource)
          Context.new(self, resource, request: @request, params: @request.params, payload: @request.payload)
        end

        def resources
          self.class.resources
        end

        def self.resources
          @resources ||= {}
        end

    end
  end
end


text:
<?xml version="1.0" encoding="ISO-8859-1"?>
<!--
  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing,
  software distributed under the License is distributed on an
  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  KIND, either express or implied.  See the License for the
  specific language governing permissions and limitations
  under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>${groupId}</groupId>
    <artifactId>${rootArtifactId}</artifactId>
    <version>${version}</version>
    <packaging>pom</packaging>

    <properties>
        <!-- plugin versions to use in generated archetypes -->
        <scrplugin.version>1.21.0</scrplugin.version>
        <bundleplugin.version>3.0.0</bundleplugin.version>
        <compilerplugin.version>3.3</compilerplugin.version>
        <slingplugin.version>2.1.2</slingplugin.version>
        <launchpadplugin.version>2.3.2</launchpadplugin.version>
        <jarplugin.version>2.6</jarplugin.version>
        <warplugin.version>2.6</warplugin.version>
        <jettyplugin.version>6.1.23</jettyplugin.version>

        <!-- various properties which influence generated archetypes -->
        <archetype.java.version>7</archetype.java.version>

        <!-- dependency versions to use in generated archetypes -->
        <junit.version>4.12</junit.version>
        <scrannotations.version>1.9.12</scrannotations.version>
        <slingapi.version>2.9.0</slingapi.version>
        <slf4japi.version>1.7.6</slf4japi.version>

        <!-- define archetype tooling version in a central place -->
        <archetype.version>2.2</archetype.version>

        !-- apache Sling configurations -->
        <sling.host>localhost</sling.host>
        <sling.port>8080</sling.port>

    </properties>

    <modules>
        <module>core</module>
        <module>ui.apps</module>
    </modules>
</project>


text:
#ifndef IntPoint_h
#define IntPoint_h

#include "core/platform/graphics/IntSize.h"
#include "wtf/MathExtras.h"
#include "wtf/Platform.h"

#if OS(DARWIN)
typedef struct CGPoint CGPoint;
#ifdef NSGEOMETRY_TYPES_SAME_AS_CGGEOMETRY_TYPES
typedef struct CGPoint NSPoint;
#else
typedef struct _NSPoint NSPoint;
#endif
#endif

namespace WebCore {

class IntPoint {
public:
    IntPoint() : m_x(0), m_y(0) { }
    IntPoint(int x, int y) : m_x(x), m_y(y) { }
    explicit IntPoint(const IntSize& size) : m_x(size.width()), m_y(size.height()) { }

    static IntPoint zero() { return IntPoint(); }

    int x() const { return m_x; }
    int y() const { return m_y; }

    void setX(int x) { m_x = x; }
    void setY(int y) { m_y = y; }

    void move(const IntSize& s) { move(s.width(), s.height()); }
    void moveBy(const IntPoint& offset) { move(offset.x(), offset.y()); }
    void move(int dx, int dy) { m_x += dx; m_y += dy; }
    void scale(float sx, float sy)
    {
        m_x = lroundf(static_cast<float>(m_x * sx));
        m_y = lroundf(static_cast<float>(m_y * sy));
    }

    IntPoint expandedTo(const IntPoint& other) const
    {
        return IntPoint(m_x > other.m_x ? m_x : other.m_x,
            m_y > other.m_y ? m_y : other.m_y);
    }

    IntPoint shrunkTo(const IntPoint& other) const
    {
        return IntPoint(m_x < other.m_x ? m_x : other.m_x,
            m_y < other.m_y ? m_y : other.m_y);
    }

    int distanceSquaredToPoint(const IntPoint&) const;

    void clampNegativeToZero()
    {
        *this = expandedTo(zero());
    }

    IntPoint transposedPoint() const
    {
        return IntPoint(m_y, m_x);
    }

#if OS(DARWIN)
    explicit IntPoint(const CGPoint&); // don't do this implicitly since it's lossy
    operator CGPoint() const;

#if !defined(NSGEOMETRY_TYPES_SAME_AS_CGGEOMETRY_TYPES)
    explicit IntPoint(const NSPoint&); // don't do this implicitly since it's lossy
    operator NSPoint() const;
#endif
#endif

private:
    int m_x, m_y;
};

inline IntPoint& operator+=(IntPoint& a, const IntSize& b)
{
    a.move(b.width(), b.height());
    return a;
}

inline IntPoint& operator-=(IntPoint& a, const IntSize& b)
{
    a.move(-b.width(), -b.height());
    return a;
}

inline IntPoint operator+(const IntPoint& a, const IntSize& b)
{
    return IntPoint(a.x() + b.width(), a.y() + b.height());
}

inline IntPoint operator+(const IntPoint& a, const IntPoint& b)
{
    return IntPoint(a.x() + b.x(), a.y() + b.y());
}

inline IntSize operator-(const IntPoint& a, const IntPoint& b)
{
    return IntSize(a.x() - b.x(), a.y() - b.y());
}

inline IntPoint operator-(const IntPoint& a, const IntSize& b)
{
    return IntPoint(a.x() - b.width(), a.y() - b.height());
}

inline IntPoint operator-(const IntPoint& point)
{
    return IntPoint(-point.x(), -point.y());
}

inline bool operator==(const IntPoint& a, const IntPoint& b)
{
    return a.x() == b.x() && a.y() == b.y();
}

inline bool operator!=(const IntPoint& a, const IntPoint& b)
{
    return a.x() != b.x() || a.y() != b.y();
}

inline IntSize toIntSize(const IntPoint& a)
{
    return IntSize(a.x(), a.y());
}

inline int IntPoint::distanceSquaredToPoint(const IntPoint& point) const
{
    return ((*this) - point).diagonalLengthSquared();
}

} // namespace WebCore

#endif // IntPoint_h


text:
package org.jboss.weld.bootstrap.enablement;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import jakarta.enterprise.inject.spi.Extension;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;

import org.junit.Test;

/**
 *
 * @author Martin Kouba
 *
 */
public class EnablementListViewTest {

    @Test
    public void testBasicOperations() {
        final List<Item> list = new ArrayList<>();
        EnablementListView view = new EnablementListView() {
            @Override
            protected ViewType getViewType() {
                return null;
            }

            @Override
            protected Extension getExtension() {
                return null;
            }

            @Override
            protected List<Item> getDelegate() {
                return list;
            }
        };
        int stringPriority = 1;
        int doublePriority = 300;
        int bigIntegerPriority = 301;
        int floatPriority = (((bigIntegerPriority * Item.ITEM_PRIORITY_SCALE_POWER) - (doublePriority * Item.ITEM_PRIORITY_SCALE_POWER)) / 2)
            + (doublePriority * Item.ITEM_PRIORITY_SCALE_POWER);

        list.add(new Item(Integer.class, 20));
        list.add(new Item(String.class, stringPriority));
        list.add(new Item(Double.class, doublePriority));

        // test contains(), index() and lastIndexOf(), note that list is *not* sorted
        assertTrue(view.contains(Integer.class));
        assertEquals(2, view.indexOf(Double.class));
        assertEquals(0, view.lastIndexOf(Integer.class));

        assertEquals(stringPriority, list.get(1).getOriginalPriority());
        assertEquals(stringPriority, list.get(1).getPriority());
        assertEquals(0, list.get(1).getNumberOfScaling());

        assertEquals(3, view.size());
        assertEquals(Integer.class, view.get(0));
        list.add(new Item(BigInteger.class, bigIntegerPriority));
        assertEquals(4, view.size());

        view.add(BigDecimal.class);
        assertEquals(5, view.size());
        assertEquals(BigDecimal.class, view.get(view.size() - 1));

        Collections.sort(list);
        // String, Integer, Double, BigInteger, BigDecimal
        // Scale!
        view.add(3, Float.class);
        // String, Integer, Double, Float, BigInteger, BigDecimal
        assertEquals(6, list.size());
        assertEquals(doublePriority * Item.ITEM_PRIORITY_SCALE_POWER, list.get(2).getPriority());
        assertEquals(doublePriority, list.get(2).getOriginalPriority());
        assertEquals(1, list.get(2).getNumberOfScaling());
        assertEquals(Float.class, view.get(3));
        assertEquals(floatPriority, list.get(3).getPriority());

        view.remove(3);
        // String, Integer, Double, BigInteger, BigDecimal
        assertEquals(5, list.size());
        assertEquals(bigIntegerPriority * Item.ITEM_PRIORITY_SCALE_POWER, list.get(3).getPriority());

        view.set(0, StringBuilder.class);
        assertEquals(stringPriority * Item.ITEM_PRIORITY_SCALE_POWER, list.get(0).getPriority());

        // remove via List.remove(Object)
        view.remove(Double.class);
        assertEquals(4, list.size());
    }

    @Test
    public void testListIterator() {

        final List<Item> list = new ArrayList<>();
        EnablementListView view = new EnablementListView() {
            @Override
            protected ViewType getViewType() {
                return null;
            }

            @Override
            protected Extension getExtension() {
                return null;
            }

            @Override
            protected List<Item> getDelegate() {
                return list;
            }
        };
        int stringPriority = 20;
        list.add(new Item(Integer.class, 10));
        list.add(new Item(String.class, stringPriority));
        list.add(new Item(Double.class, 30));

        ListIterator<Class<?>> iterator = view.listIterator();
        // -> Integer, String, Double
        assertEquals(Integer.class, iterator.next());
        // Integer, -> String, Double
        iterator.remove();
        // -> String, Double
        iterator.add(Float.class);
        assertEquals(Float.class, iterator.previous());
        // -> Float, String, Double
        assertFalse(iterator.hasPrevious());
        assertTrue(iterator.hasNext());
        assertEquals(Float.class, iterator.next());
        assertEquals(String.class, iterator.next());
        // Float, -> String, Double
        iterator.set(StringBuilder.class);
        assertEquals(3, list.size());
        assertEquals(StringBuilder.class, view.get(1));
        assertEquals(stringPriority, list.get(1).getPriority());
    }

    @Test
    public void testMassListOperations() {
        final List<Item> list = new ArrayList<>();
        EnablementListView view = new EnablementListView() {
            @Override
            protected EnablementListView.ViewType getViewType() {
                return null;
            }

            @Override
            protected Extension getExtension() {
                return null;
            }

            @Override
            protected List<Item> getDelegate() {
                return list;
            }
        };
        list.add(new Item(Integer.class, 10));
        list.add(new Item(String.class, 20));
        list.add(new Item(Double.class, 30));
        list.add(new Item(Float.class, 50));

        // note that these operations indirectly test List.contains()
        // try removeAll()
        Collection<Object> testList = new ArrayList<Object>();
        testList.add(Integer.class);
        testList.add(Float.class);

        view.removeAll(testList);

        assertEquals(2, list.size());
        assertEquals(String.class, list.get(0).getJavaClass());
        assertEquals(Double.class, list.get(1).getJavaClass());

        // re-add missing items
        list.add(new Item(Integer.class, 10));
        list.add(new Item(Float.class, 50));

        // try retainAll()
        view.retainAll(testList);
        assertEquals(2, list.size());
        assertEquals(Integer.class, list.get(0).getJavaClass());
        assertEquals(Float.class, list.get(1).getJavaClass());

        // re-add missing items
        list.add(new Item(String.class, 20));
        list.add(new Item(Double.class, 30));

        // try containsAll()
        assertTrue(view.containsAll(testList));
    }
}


text:
cask "media-center" do
  version "28.00.96"
  sha256 "6678151d64e46b253349d42b24c360f4e26c5b27363184cab85948800454122d"

  url "https://files.jriver.com/mediacenter/channels/v#{version.major}/stable/MediaCenter#{version.no_dots}-Universal.dmg"
  name "JRiver Media Center"
  desc "Manage and play your media"
  homepage "https://www.jriver.com/"

  livecheck do
    url "https://www.jriver.com/download.html"
    strategy :page_match do |page|
      match = page.match(/MediaCenter(\d\d+)(\d\d+)(\d\d+)-Universal\.dmg/i)
      next if match.blank?

      "#{match[1]}.#{match[2]}.#{match[3]}"
    end
  end

  auto_updates true

  app "Media Center #{version.major}.app"

  zap trash: [
    "~/Documents/JRiver/",
    "~/Library/Application Support/J River/",
    "~/Library/Caches/com.jriver.MediaCenter#{version.major}",
    "~/Library/Cookies/com.jriver.MediaCenter#{version.major}*",
    "~/Library/Preferences/com.jriver.MediaCenter#{version.major}.plist",
    "~/Library/Saved Application State/com.jriver.MediaCenter#{version.major}.savedState",
  ]
end


text:
import { ITreeFilterProps, ITreeFilterState, TreeItem } from './treeFilter.Props';
import * as _ from 'lodash';

export interface TreeBranch {
    id: string;
    depth: number;
}

export interface LeafsAndBranches {
    Leafs: Array<string>;
    Branches: Array<TreeBranch>;
}

export interface CheckResult {
    checked: Array<string>;
    partially: Array<string>;
}

export interface ILookupTable {
    [id: string]: TreeItem;
}

export interface TreeLookups {
    parentLookup: ILookupTable;
    itemLookup: ILookupTable;
}

/**
 * Check if current tree item has children by checking if its children property
 * is not null and its length is greater than 0.
 * @param item Current tree item.
 */
export const itemHasChildren = (item: TreeItem) => {
    return item.children != null && item.children.length !== 0;
};

export class ItemOperator {
    static getLookupTableAndParentLookup = (items: Array<TreeItem>): TreeLookups => {
        let parentItems: ILookupTable = {};
        let lookup: ILookupTable = {};

        if (items == null || items.length === 0) {
            return {
                parentLookup: parentItems,
                itemLookup: lookup
            };
        }

        const setParentAndLookup = (parent, children: Array<TreeItem>) => {
            for (let item of children) {
                lookup[item.id] = item;
                parentItems[item.id] = parent;
                if (itemHasChildren(item)) {
                    setParentAndLookup(item, item.children);
                }
            }
        };

        setParentAndLookup(undefined, items);

        return {
            parentLookup: Object.freeze(parentItems),
            itemLookup: Object.freeze(lookup)
        };
    }

    /**
     * Returns ids of all items in tree structure in single array.
     */
    static getAllItemIds = (entryItems: Array<TreeItem>) => {
        const getItemIdsRecursive = (items) => {
            let itemIds = [];

            if (items == null || items.length === 0) {
                return itemIds;
            }

            for (let item of items) {
                itemIds.push(item.id);
                if (item.children) {
                    itemIds = itemIds.concat(getItemIdsRecursive(item.children));
                }
            }

            return itemIds;
        };
        let allItems = getItemIdsRecursive(entryItems);
        return Object.freeze(allItems);
    }

    /**
     * Returns ids of all children that are children of current item or any
     * of its child recursive.
     */
    static getAllChildrenIds = (currentItem: TreeItem): Array<string> => {
        let childrenIds: Array<string> = [];
        if (itemHasChildren(currentItem)) {
            currentItem.children.forEach(childElement => {
                childrenIds.push(childElement.id);
                childrenIds = childrenIds.concat(ItemOperator.getAllChildrenIds(childElement));
            });
        }
        return childrenIds;
    }

    static getAllLeafsAndBranches = (currentItem: TreeItem, originalItem: TreeItem): LeafsAndBranches => {
        let leafs = [];
        let branches: Array<TreeBranch> = [];

        const checkItemRecursive = (item: TreeItem, depth) => {
            const children = item.children;
            if (itemHasChildren(item)) {
                branches.push({ id: item.id, depth: depth });
                children.forEach(childElement => {
                    checkItemRecursive(childElement, depth + 1);
                });
            } else {
                leafs.push(item.id);
            }
        };

        if (itemHasChildren(currentItem)) {
            branches.push({
                id: currentItem.id,
                depth: 0
            });

            currentItem.children.forEach(childElement => {
                checkItemRecursive(childElement, 1);
            });
        } else if (itemHasChildren(originalItem)) { // when all children are filtered with search
            branches.push({
                id: originalItem.id,
                depth: 0
            });

            originalItem.children.forEach(childElement => {
                checkItemRecursive(childElement, 1);
            });
        } else {
            leafs.push(currentItem.id);
        }

        return {
            Leafs: leafs,
            Branches: branches
        };
    }

    /**
     * Filters given array of tree items based on given search string.
     * Values and search query will be converted to lowercase before so this
     * search is case insensitive.
     */
    static filterItems = (items: Array<TreeItem>, query: string): Array<TreeItem> => {
        let filteredItems: Array<TreeItem> = [];
        query = query.toLowerCase();

        if (items == null || items.length === 0) {
            return filteredItems;
        }

        if (query === '') {
            return items;
        }

        for (let item of items) {
            if (item.value.toLowerCase().search(query) !== -1) {
                let newItem: TreeItem = { ...item, children: [], expanded: false };
                filteredItems.push(newItem);
            } else if (itemHasChildren(item)) {
                const filteredChildren = ItemOperator.filterItems(item.children, query);
                if (filteredChildren.length > 0) {
                    let newItem: TreeItem = { ...item, children: filteredChildren, expanded: true };
                    filteredItems.push(newItem);
                }
            }
        }

        return filteredItems;
    }
}