import datetime
import decimal
import json

from django import forms
from django.conf import settings
from django.core.exceptions import ValidationError
from django.test import TestCase, override_settings
from django.utils import timezone
from faker import Factory as FakerFactory
from django.contrib.auth.models import Permission
from django.contrib.contenttypes.models import ContentType

from experimenter.experiments.forms import (
    BugzillaURLField,
    DSIssueURLField,
    ChangeLogMixin,
    CustomModelMultipleChoiceField,
    ExperimentArchiveForm,
    ExperimentCommentForm,
    ExperimentObjectivesForm,
    ExperimentOverviewForm,
    ExperimentReviewForm,
    ExperimentRisksForm,
    ExperimentStatusForm,
    ExperimentSubscribedForm,
    ExperimentResultsForm,
    JSONField,
    NormandyIdForm,
    ExperimentTimelinePopulationForm,
    ExperimentOrderingForm,
)
from experimenter.experiments.models import Experiment
from experimenter.base.tests.factories import CountryFactory, LocaleFactory
from experimenter.experiments.tests.factories import ExperimentFactory, UserFactory
from experimenter.bugzilla.tests.mixins import MockBugzillaMixin
from experimenter.experiments.tests.mixins import MockTasksMixin, MockRequestMixin

from experimenter.notifications.models import Notification

faker = FakerFactory.create()

class TestJSONField(TestCase):

    def test_jsonfield_accepts_valid_json(self):
        valid_json = json.dumps({"a": True, 2: ["b", 3, 4.0]})
        field = JSONField()
        cleaned = field.clean(valid_json)
        self.assertEqual(cleaned, valid_json)

    def test_jsonfield_rejects_invalid_json(self):
        invalid_json = "{this isnt valid"
        field = JSONField()

        with self.assertRaises(ValidationError):
            field.clean(invalid_json)

@override_settings(BUGZILLA_HOST="https://bugzilla.example.com/")
class TestBugzillaURLField(TestCase):

    def test_accepts_bugzilla_url(self):
        field = BugzillaURLField()
        bugzilla_url = "{base}/show_bug.cgi?id=123".format(base=settings.BUGZILLA_HOST)
        cleaned = field.clean(bugzilla_url)
        self.assertEqual(cleaned, bugzilla_url)

    def test_rejects_non_show_bug_bugzilla_url(self):
        field = BugzillaURLField()
        bugzilla_url = "{base}/123".format(base=settings.BUGZILLA_HOST)

        with self.assertRaises(ValidationError):
            field.clean(bugzilla_url)

    def test_rejects_non_bugzilla_url(self):
        field = BugzillaURLField()

        with self.assertRaises(ValidationError):
            field.clean("www.example.com")

@override_settings(DS_ISSUE_HOST="https://jira.example.com/browse/")
class TestDSIssueURLField(TestCase):

    def test_accepts_ds_url_field(self):
        field = DSIssueURLField()
        ds_url = "{base}DS-1234".format(base=settings.DS_ISSUE_HOST)
        cleaned = field.clean(ds_url)
        self.assertEqual(cleaned, ds_url)

    def test_rejects_wrong_project_name(self):
        field = DSIssueURLField()
        ds_url = "{base}AA-1234".format(base=settings.DS_ISSUE_HOST)
        with self.assertRaises(ValidationError):
            field.clean(ds_url)

class TestChangeLogMixin(MockRequestMixin, TestCase):

    def test_mixin_creates_change_log_with_request_user_on_save(self):

        class TestForm(ChangeLogMixin, forms.ModelForm):

            class Meta:
                model = Experiment
                fields = ("name",)

        data = ExperimentFactory.attributes()
        form = TestForm(request=self.request, data=data)

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(experiment.changes.count(), 1)

        change = experiment.changes.get()
        self.assertEqual(change.changed_by, self.user)

    def test_mixin_sets_old_and_new_status(self):
        old_status = Experiment.STATUS_DRAFT
        new_status = Experiment.STATUS_REVIEW
        experiment = ExperimentFactory.create_with_status(old_status)

        self.assertEqual(experiment.changes.count(), 1)

        class TestForm(ChangeLogMixin, forms.ModelForm):

            class Meta:
                model = Experiment
                fields = ("status",)

        form = TestForm(
            request=self.request, data={"status": new_status}, instance=experiment
        )
        self.assertTrue(form.is_valid())

        form.save()

        self.assertEqual(experiment.changes.count(), 2)

        change = experiment.changes.latest()

        self.assertEqual(change.old_status, old_status)
        self.assertEqual(change.new_status, new_status)

    def test_changelog_not_produced_when_no_change(self):

        experiment = ExperimentFactory.create_with_status(
            target_status=Experiment.STATUS_DRAFT
        )
        num_of_changes = experiment.changes.count()

        class TestForm(ChangeLogMixin, forms.ModelForm):

            class Meta:
                model = Experiment
                fields = ("name",)

        form = TestForm(
            request=self.request, data={"name": experiment.name}, instance=experiment
        )
        self.assertTrue(form.is_valid())
        form.save()
        experiment = Experiment.objects.get(id=experiment.id)
        self.assertEqual(experiment.changes.count(), num_of_changes)

    def test_changelog_values(self):
        experiment = Experiment()
        experiment.save()

        country1 = CountryFactory(code="CA", name="Canada")
        country2 = CountryFactory(code="US", name="United States")
        locale1 = LocaleFactory(code="da", name="Danish")
        locale2 = LocaleFactory(code="de", name="German")

        data = {
            "proposed_start_date": timezone.now().date(),
            "proposed_duration": 20,
            "population_percent": "10",
            "firefox_min_version": "56.0",
            "firefox_max_version": "58.0",
            "firefox_channel": Experiment.CHANNEL_BETA,
            "client_matching": "en-us",
            "platform": Experiment.PLATFORM_WINDOWS,
            "locales": [locale1, locale2],
            "countries": [country1, country2],
        }

        form = ExperimentTimelinePopulationForm(
            request=self.request, data=data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        experiment = form.save()
        latest_changes = experiment.changes.latest()

        expected_data = {
            "locales": {
                "new_value": ["da", "de"],
                "old_value": None,
                "display_name": "Locales",
            },
            "platform": {
                "new_value": "All Windows",
                "old_value": None,
                "display_name": "Platform",
            },
            "countries": {
                "new_value": ["CA", "US"],
                "old_value": None,
                "display_name": "Countries",
            },
            "client_matching": {
                "new_value": "en-us",
                "old_value": None,
                "display_name": "Population Filtering",
            },
            "firefox_channel": {
                "new_value": "Beta",
                "old_value": None,
                "display_name": "Firefox Channel",
            },
            "proposed_duration": {
                "new_value": 20,
                "old_value": None,
                "display_name": "Proposed Total Duration (days)",
            },
            "population_percent": {
                "new_value": "10.0000",
                "old_value": None,
                "display_name": "Population Percentage",
            },
            "firefox_min_version": {
                "new_value": "56.0",
                "old_value": None,
                "display_name": "Firefox Min Version",
            },
            "firefox_max_version": {
                "new_value": "58.0",
                "old_value": None,
                "display_name": "Firefox Max Version",
            },
            "proposed_start_date": {
                "new_value": timezone.now().date().strftime("%Y-%m-%d"),
                "old_value": None,
                "display_name": "Proposed Start Date",
            },
        }

        self.assertEqual(expected_data, latest_changes.changed_values)

@override_settings(
    BUGZILLA_HOST="https://bugzilla.example.com/",
    DS_ISSUE_HOST="https://jira.example.com/browse/",
)
class TestExperimentOverviewForm(MockRequestMixin, TestCase):

    def setUp(self):
        super().setUp()
        ds_url = "{base}DS-123".format(base=settings.DS_ISSUE_HOST)
        bug_url = "{base}show_bug.cgi?id=123".format(base=settings.BUGZILLA_HOST)
        self.related_exp = ExperimentFactory.create()

        self.data = {
            "type": Experiment.TYPE_PREF,
            "name": "A new experiment!",
            "short_description": "Let us learn new things",
            "data_science_issue_url": ds_url,
            "owner": self.user.id,
            "analysis_owner": self.user.id,
            "engineering_owner": "Lisa the Engineer",
            "public_name": "A new public experiment!",
            "public_description": "Let us learn new public things",
            "related_to": [self.related_exp],
            "feature_bugzilla_url": bug_url,
            "related_work": "Designs: https://www.example.com/myproject/",
        }

    def test_minimum_required_fields_for_experiment(self):
        bug_url = "https://jira.example.com/browse/DO-123"

        data = {
            "type": Experiment.TYPE_PREF,
            "owner": self.user.id,
            "name": "A new experiment!",
            "short_description": "Let us learn new things",
            "data_science_issue_url": bug_url,
            "public_name": "Public Name",
            "public_description": "Public Description",
        }
        form = ExperimentOverviewForm(request=self.request, data=data)

        self.assertTrue(form.is_valid())

        experiment = form.save()

        self.assertEqual(experiment.owner, self.user)
        self.assertEqual(experiment.status, experiment.STATUS_DRAFT)
        self.assertEqual(experiment.name, data["name"])
        self.assertEqual(experiment.slug, "a-new-experiment")
        self.assertEqual(experiment.short_description, data["short_description"])
        self.assertEqual(experiment.public_name, data["public_name"])
        self.assertEqual(experiment.public_description, data["public_description"])
        self.assertEqual(experiment.changes.count(), 1)

    def test_minimum_required_fields_for_rollout(self):
        data = {
            "type": Experiment.TYPE_ROLLOUT,
            "owner": self.user.id,
            "name": "A new experiment!",
            "short_description": "Let us learn new things",
        }
        form = ExperimentOverviewForm(request=self.request, data=data)
        self.assertTrue(form.is_valid())

        experiment = form.save()

        self.assertEqual(experiment.owner, self.user)
        self.assertEqual(experiment.status, experiment.STATUS_DRAFT)
        self.assertEqual(experiment.name, self.data["name"])
        self.assertEqual(experiment.slug, "a-new-experiment")
        self.assertEqual(experiment.short_description, self.data["short_description"])
        self.assertEqual(experiment.changes.count(), 1)

    def test_form_creates_experiment(self):
        form = ExperimentOverviewForm(request=self.request, data=self.data)
        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(experiment.owner, self.user)
        self.assertEqual(experiment.engineering_owner, self.data["engineering_owner"])
        self.assertEqual(experiment.status, experiment.STATUS_DRAFT)
        self.assertEqual(experiment.name, self.data["name"])
        self.assertEqual(experiment.slug, "a-new-experiment")
        self.assertEqual(experiment.short_description, self.data["short_description"])
        self.assertTrue(self.related_exp in experiment.related_to.all())

        self.assertEqual(experiment.changes.count(), 1)
        change = experiment.changes.get()
        self.assertEqual(change.old_status, None)
        self.assertEqual(change.new_status, experiment.status)
        self.assertEqual(change.changed_by, self.request.user)

    def test_empty_slug_raises_error(self):
        self.data["name"] = "#"

        form = ExperimentOverviewForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_unique_name_with_same_slug_raises_error(self):
        ExperimentFactory.create(slug="slug")
        self.data["name"] = "slug#"

        form = ExperimentOverviewForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_ds_issue_url_required_for_non_rollout(self):
        self.data["type"] = Experiment.TYPE_PREF
        del self.data["data_science_issue_url"]

        form = ExperimentOverviewForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_bugzilla_url_optional_for_rollout(self):
        self.data["type"] = Experiment.TYPE_ROLLOUT
        del self.data["data_science_issue_url"]

        form = ExperimentOverviewForm(request=self.request, data=self.data)

        self.assertTrue(form.is_valid())

    def test_invalid_bugzilla_url(self):
        self.data["data_science_issue_url"] = "https://example.com/notbugzilla"

        form = ExperimentOverviewForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

class TestExperimentTimelinePopulationForm(MockRequestMixin, TestCase):

    def setUp(self):
        super().setUp()

        self.data = {
            "proposed_start_date": timezone.now().date(),
            "proposed_duration": 20,
            "proposed_enrollment": 10,
            "population_percent": "10.0",
            "firefox_channel": Experiment.CHANNEL_NIGHTLY,
            "firefox_min_version": "67.0",
            "firefox_max_version": "69.0",
            "locales": [],
            "countries": [],
            "platform": Experiment.PLATFORM_WINDOWS,
            "client_matching": "en-us",
        }

    def test_no_fields_required(self):
        experiment = ExperimentFactory.create()
        data = {
            "proposed_start_date": "",
            "proposed_duration": "",
            "proposed_enrollment": "",
            "population_percent": "",
            "firefox_channel": "",
            "firefox_min_version": "",
            "firefox_max_version": "",
            "locales": [],
            "countries": [],
            "platform": "",
            "client_matching": "",
        }
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        experiment = form.save()

    def test_form_saves_experiment_timeline_population_data(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, countries=[], locales=[]
        )
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )

        self.assertEqual(experiment.changes.count(), 1)

        experiment = form.save()

        self.assertEqual(experiment.proposed_start_date, self.data["proposed_start_date"])
        self.assertEqual(experiment.proposed_duration, self.data["proposed_duration"])
        self.assertEqual(experiment.proposed_enrollment, self.data["proposed_enrollment"])
        self.assertEqual(experiment.population_percent, decimal.Decimal("10.000"))
        self.assertEqual(experiment.firefox_channel, self.data["firefox_channel"])
        self.assertEqual(experiment.firefox_min_version, self.data["firefox_min_version"])
        self.assertEqual(experiment.firefox_max_version, self.data["firefox_max_version"])
        self.assertEqual(experiment.platform, self.data["platform"])
        self.assertEqual(experiment.client_matching, self.data["client_matching"])

        self.assertEqual(experiment.changes.count(), 2)

    def test_form_saves_rollout_timeline_population_data(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT,
            type=Experiment.TYPE_ROLLOUT,
            countries=[],
            locales=[],
        )

        data = {
            "proposed_start_date": timezone.now().date(),
            "proposed_duration": 20,
            "rollout_playbook": Experiment.ROLLOUT_PLAYBOOK_LOW_RISK,
            "firefox_channel": Experiment.CHANNEL_NIGHTLY,
            "firefox_min_version": "67.0",
            "firefox_max_version": "69.0",
            "locales": [],
            "countries": [],
        }

        form = ExperimentTimelinePopulationForm(
            request=self.request, data=data, instance=experiment
        )

        self.assertEqual(experiment.changes.count(), 1)

        experiment = form.save()

        self.assertEqual(experiment.proposed_start_date, data["proposed_start_date"])
        self.assertEqual(experiment.firefox_min_version, data["firefox_min_version"])
        self.assertEqual(experiment.firefox_max_version, data["firefox_max_version"])
        self.assertEqual(experiment.firefox_channel, data["firefox_channel"])
        self.assertEqual(experiment.population_percent, decimal.Decimal("25.000"))

        self.assertEqual(experiment.changes.count(), 2)

    def test_enrollment_must_be_less_or_equal_duration(self):
        self.data["proposed_enrollment"] = 2
        self.data["proposed_duration"] = 1

        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_large_duration_is_invalid(self):
        self.data["proposed_duration"] = Experiment.MAX_DURATION + 1

        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_large_enrollment_duration_is_invalid(self):
        self.data["proposed_enrollment"] = Experiment.MAX_DURATION + 1

        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())

    def test_start_date_must_be_greater_or_equal_to_current_date(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        self.data["proposed_start_date"] = timezone.now().date() - datetime.timedelta(
            days=1
        )

        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertFalse(form.is_valid())

    def test_locales_choices(self):
        locale1 = LocaleFactory(code="sv-SE", name="Swedish")
        locale2 = LocaleFactory(code="fr", name="French")
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, countries=[], locales=[]
        )
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(
            list(form.fields["locales"].choices),
            [
                (CustomModelMultipleChoiceField.ALL_KEY, "All locales"),
                (locale2.code, str(locale2)),
                (locale1.code, str(locale1)),
            ],
        )

    def test_locales_initials(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, locales=[]
        )
        locale1 = LocaleFactory(code="sv-SE", name="Swedish")
        locale2 = LocaleFactory(code="fr", name="French")
        experiment.locales.add(locale1)
        experiment.locales.add(locale2)
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(form.initial["locales"], [locale2, locale1])

    def test_locales_initials_all_locales(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, locales=[]
        )
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(
            form.initial["locales"], [CustomModelMultipleChoiceField.ALL_KEY]
        )

    def test_clean_locales(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, num_variants=0
        )
        locale1 = LocaleFactory(code="sv-SE", name="Swedish")
        locale2 = LocaleFactory(code="fr", name="French")
        self.data["locales"] = [locale2.code, locale1.code]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        self.assertEqual(set(form.cleaned_data["locales"]), set([locale2, locale1]))

    def test_clean_locales_all(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, num_variants=0
        )
        locale1 = LocaleFactory(code="sv-SE", name="Swedish")
        locale2 = LocaleFactory(code="fr", name="French")
        self.data["locales"] = [
            locale2.code,
            CustomModelMultipleChoiceField.ALL_KEY,
            locale1.code,
        ]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        self.assertEqual(list(form.cleaned_data["locales"]), [])

    def test_clean_unrecognized_locales(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        self.data["locales"] = ["xxx"]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(not form.is_valid())
        self.assertTrue(form.errors["locales"])

    def test_countries_choices(self):
        country1 = CountryFactory(code="SV", name="Sweden")
        country2 = CountryFactory(code="FR", name="France")

        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, countries=[], locales=[]
        )
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(
            list(form.fields["countries"].choices),
            [
                (CustomModelMultipleChoiceField.ALL_KEY, "All countries"),
                (country2.code, str(country2)),
                (country1.code, str(country1)),
            ],
        )

    def test_countries_initials(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, countries=[]
        )
        country1 = CountryFactory(code="SV", name="Sweden")
        country2 = CountryFactory(code="FR", name="France")
        experiment.countries.add(country1)
        experiment.countries.add(country2)
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(form.initial["countries"], [country2, country1])

    def test_countries_initials_all(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, num_variants=0, countries=[]
        )
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertEqual(
            form.initial["countries"], [CustomModelMultipleChoiceField.ALL_KEY]
        )

    def test_clean_countries(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, num_variants=0
        )
        country1 = CountryFactory(code="SV", name="Sweden")
        country2 = CountryFactory(code="FR", name="France")
        self.data["countries"] = [country1.code, country2.code]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        self.assertEqual(
            # form.cleaned_data["countries"] is a QuerySet to exhaust it.
            list(form.cleaned_data["countries"]),
            [country2, country1],
        )

    def test_clean_countries_all(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, num_variants=0
        )
        country1 = CountryFactory(code="SV", name="Sweden")
        country2 = CountryFactory(code="FR", name="France")
        self.data["countries"] = [
            country1.code,
            CustomModelMultipleChoiceField.ALL_KEY,
            country2.code,
        ]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(form.is_valid())
        self.assertEqual(list(form.cleaned_data["countries"]), [])

    def test_clean_unrecognized_countries(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        self.data["countries"] = ["xxx"]
        form = ExperimentTimelinePopulationForm(
            request=self.request, data=self.data, instance=experiment
        )
        self.assertTrue(not form.is_valid())
        self.assertTrue(form.errors["countries"])

    def test_form_is_invalid_if_population_percent_below_0(self):
        self.data["population_percent"] = "-1"
        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())
        self.assertIn("population_percent", form.errors)

    def test_form_is_invalid_if_population_percent_above_100(self):
        self.data["population_percent"] = "101"
        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())
        self.assertIn("population_percent", form.errors)

    def test_form_is_invalid_if_firefox_max_is_lower_than_min(self):
        self.data["firefox_min_version"] = "66.0"
        self.data["firefox_max_version"] = "64.0"
        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertFalse(form.is_valid())
        self.assertIn("firefox_max_version", form.errors)

    def test_form_is_valid_if_firefox_max_is_equal_to_min(self):
        self.data["firefox_min_version"] = "66.0"
        self.data["firefox_max_version"] = "66.0"
        form = ExperimentTimelinePopulationForm(request=self.request, data=self.data)
        self.assertTrue(form.is_valid())

class TestExperimentObjectivesForm(MockRequestMixin, TestCase):

    def test_no_fields_required(self):
        experiment = ExperimentFactory.create()
        form = ExperimentObjectivesForm(
            request=self.request, data={}, instance=experiment
        )
        self.assertTrue(form.is_valid())

    def test_form_saves_objectives(self):
        created_experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        self.assertEqual(created_experiment.changes.count(), 1)

        data = {
            "objectives": "The objective is to experiment!",
            "analysis": "Lets analyze the results!",
            "survey_required": True,
            "survey_urls": "example.com",
            "survey_instructions": "Here are the launch instructions.",
        }

        form = ExperimentObjectivesForm(
            request=self.request, data=data, instance=created_experiment
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(experiment.objectives, data["objectives"])
        self.assertEqual(experiment.analysis, data["analysis"])
        self.assertTrue(experiment.survey_required)
        self.assertEqual(experiment.survey_urls, data["survey_urls"])
        self.assertEqual(experiment.survey_instructions, data["survey_instructions"])

        self.assertEqual(experiment.changes.count(), 2)

class TestExperimentRisksForm(MockRequestMixin, TestCase):

    valid_data = {
        "risk_partner_related": False,
        "risk_brand": True,
        "risk_fast_shipped": True,
        "risk_confidential": True,
        "risk_release_population": True,
        "risk_revenue": True,
        "risk_data_category": True,
        "risk_external_team_impact": True,
        "risk_telemetry_data": True,
        "risk_ux": True,
        "risk_security": True,
        "risk_revision": True,
        "risk_technical": True,
        "risk_technical_description": "It's complicated",
        "risks": "There are some risks",
        "testing": "Always be sure to test!",
        "test_builds": "Latest build",
        "qa_status": "It ain't easy being green",
    }

    def test_no_fields_required(self):
        experiment = ExperimentFactory.create()
        form = ExperimentRisksForm(request=self.request, data={}, instance=experiment)
        self.assertTrue(form.is_valid())

    def test_form_saves_risks(self):
        created_experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        self.assertEqual(created_experiment.changes.count(), 1)

        data = self.valid_data.copy()
        form = ExperimentRisksForm(
            request=self.request, data=data, instance=created_experiment
        )

        self.assertTrue(form.is_valid())

        experiment = form.save()
        self.assertFalse(experiment.risk_partner_related)
        self.assertTrue(experiment.risk_brand)
        self.assertTrue(experiment.risk_fast_shipped)
        self.assertTrue(experiment.risk_confidential)
        self.assertTrue(experiment.risk_release_population)
        self.assertTrue(experiment.risk_technical)
        self.assertEqual(
            experiment.risk_technical_description, data["risk_technical_description"]
        )
        self.assertEqual(experiment.risks, data["risks"])
        self.assertEqual(experiment.testing, data["testing"])
        self.assertEqual(experiment.test_builds, data["test_builds"])
        self.assertEqual(experiment.qa_status, data["qa_status"])

        self.assertEqual(experiment.changes.count(), 2)

class TestExperimentResultsForm(MockRequestMixin, TestCase):

    def test_form_saves_results(self):
        experiment = ExperimentFactory.create()
        self.assertEqual(experiment.changes.count(), 0)
        data = {
            "results_url": "https://example.com",
            "results_initial": "Initially, all went well.",
            "results_lessons_learned": "Lessons were learned.",
        }

        form = ExperimentResultsForm(request=self.request, data=data, instance=experiment)

        self.assertTrue(form.is_valid())

        experiment = form.save()

        self.assertEqual(experiment.results_url, "https://example.com")
        self.assertEqual(experiment.results_initial, "Initially, all went well.")

        self.assertEqual(experiment.changes.count(), 1)

class TestExperimentReviewForm(
    MockRequestMixin, MockBugzillaMixin, MockTasksMixin, TestCase
):

    def test_form_saves_reviews(self):
        user = UserFactory.create()
        content_type = ContentType.objects.get_for_model(Experiment)
        experiment_model_permissions = Permission.objects.filter(
            content_type=content_type, codename__startswith="can_check"
        )
        for permission in experiment_model_permissions:
            user.user_permissions.add(permission)

        self.request.user = user

        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_REVIEW)
        self.assertEqual(experiment.changes.count(), 2)

        self.assertFalse(experiment.review_science)
        self.assertFalse(experiment.review_engineering)
        self.assertFalse(experiment.review_qa_requested)
        self.assertFalse(experiment.review_intent_to_ship)
        self.assertFalse(experiment.review_bugzilla)
        self.assertFalse(experiment.review_qa)
        self.assertFalse(experiment.review_relman)
        self.assertFalse(experiment.review_advisory)
        self.assertFalse(experiment.review_legal)
        self.assertFalse(experiment.review_ux)
        self.assertFalse(experiment.review_security)
        self.assertFalse(experiment.review_vp)
        self.assertFalse(experiment.review_data_steward)
        self.assertFalse(experiment.review_comms)
        self.assertFalse(experiment.review_impacted_teams)

        data = {
            "review_science": True,
            "review_engineering": True,
            "review_qa_requested": True,
            "review_intent_to_ship": True,
            "review_bugzilla": True,
            "review_qa": True,
            "review_relman": True,
            "review_advisory": True,
            "review_legal": True,
            "review_ux": True,
            "review_security": True,
            "review_vp": True,
            "review_data_steward": True,
            "review_comms": True,
            "review_impacted_teams": True,
        }

        form = ExperimentReviewForm(request=self.request, data=data, instance=experiment)

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertTrue(experiment.review_science)
        self.assertTrue(experiment.review_engineering)
        self.assertTrue(experiment.review_qa_requested)
        self.assertTrue(experiment.review_intent_to_ship)
        self.assertTrue(experiment.review_bugzilla)
        self.assertTrue(experiment.review_qa)
        self.assertTrue(experiment.review_relman)
        self.assertTrue(experiment.review_advisory)
        self.assertTrue(experiment.review_legal)
        self.assertTrue(experiment.review_ux)
        self.assertTrue(experiment.review_security)
        self.assertTrue(experiment.review_vp)
        self.assertTrue(experiment.review_data_steward)
        self.assertTrue(experiment.review_comms)
        self.assertTrue(experiment.review_impacted_teams)

        self.assertEqual(experiment.changes.count(), 3)

    def test_added_reviews_property(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_REVIEW)
        self.assertEqual(experiment.changes.count(), 2)

        data = {"review_bugzilla": True, "review_science": True}

        form = ExperimentReviewForm(request=self.request, data=data, instance=experiment)

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(len(form.added_reviews), 2)
        self.assertEqual(len(form.removed_reviews), 0)
        self.assertIn(form.fields["review_bugzilla"].label, form.added_reviews)
        self.assertIn(form.fields["review_science"].label, form.added_reviews)

        self.assertEqual(experiment.changes.count(), 3)

    def test_removed_reviews_property(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_REVIEW, review_bugzilla=True, review_science=True
        )

        self.assertEqual(experiment.changes.count(), 2)

        data = {"review_bugzilla": False, "review_science": False}

        form = ExperimentReviewForm(request=self.request, data=data, instance=experiment)

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(len(form.added_reviews), 0)
        self.assertEqual(len(form.removed_reviews), 2)
        self.assertIn(form.fields["review_bugzilla"].label, form.removed_reviews)
        self.assertIn(form.fields["review_science"].label, form.removed_reviews)

        self.assertEqual(experiment.changes.count(), 3)

    def test_required_reviews(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_REVIEW,
            type=Experiment.TYPE_PREF,
            review_relman=True,
            review_science=True,
        )

        form = ExperimentReviewForm(request=self.request, data={}, instance=experiment)

        self.assertEqual(
            set([f.name for f in form.required_reviews]),
            set(
                [
                    "review_science",
                    "review_advisory",
                    "review_engineering",
                    "review_qa_requested",
                    "review_intent_to_ship",
                    "review_bugzilla",
                    "review_qa",
                    "review_relman",
                ]
            ),
        )

    def test_required_reviews_when_a_risk_partner_related_is_true(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_REVIEW,
            review_relman=True,
            review_science=True,
            risk_partner_related=True,
        )

        form = ExperimentReviewForm(request=self.request, data={}, instance=experiment)

        self.assertIn(form["review_vp"], form.required_reviews)
        self.assertIn(form["review_legal"], form.required_reviews)
        self.assertNotIn(form["review_vp"], form.optional_reviews)
        self.assertNotIn(form["review_legal"], form.optional_reviews)

    def test_optional_reviews(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_REVIEW, review_relman=True, review_science=True
        )

        form = ExperimentReviewForm(request=self.request, data={}, instance=experiment)

        self.assertEqual(
            form.optional_reviews,
            [
                form["review_comms"],
                form["review_data_steward"],
                form["review_impacted_teams"],
                form["review_legal"],
                form["review_security"],
                form["review_ux"],
                form["review_vp"],
            ],
        )

    def test_optional_reviews_when_a_risk_option_is_true(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_REVIEW,
            review_relman=True,
            review_science=True,
            risk_partner_related=True,
        )

        form = ExperimentReviewForm(request=self.request, data={}, instance=experiment)

        self.assertNotIn(form["review_vp"], form.optional_reviews)
        self.assertIn(form["review_vp"], form.required_reviews)

    def test_required_reviews_for_rollout(self):
        experiment = ExperimentFactory.create(type=Experiment.TYPE_ROLLOUT)

        form = ExperimentReviewForm(self.request, instance=experiment)

        self.assertEqual(
            set([f.name for f in form.required_reviews]),
            set(
                [
                    "review_qa",
                    "review_intent_to_ship",
                    "review_qa_requested",
                    "review_advisory",
                    "review_relman",
                ]
            ),
        )

    def test_optional_reviews_for_rollout(self):
        experiment = ExperimentFactory.create(type=Experiment.TYPE_ROLLOUT)

        form = ExperimentReviewForm(self.request, instance=experiment)

        self.assertEqual(
            set([f.name for f in form.optional_reviews]),
            set(
                [
                    "review_impacted_teams",
                    "review_ux",
                    "review_legal",
                    "review_security",
                    "review_vp",
                    "review_comms",
                    "review_data_steward",
                ]
            ),
        )

    def test_cannot_check_review_relman_without_permissions(self):
        user_1 = UserFactory.create()
        user_2 = UserFactory.create()

        content_type = ContentType.objects.get_for_model(Experiment)
        permission = Permission.objects.get(
            content_type=content_type, codename="can_check_relman_signoff"
        )
        user_1.user_permissions.add(permission)

        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_REVIEW)

        self.request.user = user_2
        form = ExperimentReviewForm(
            request=self.request, data={"review_relman": True}, instance=experiment
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()
        self.assertFalse(experiment.review_relman)

        self.request.user = user_1

        form = ExperimentReviewForm(
            request=self.request, data={"review_relman": True}, instance=experiment
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertTrue(experiment.review_relman)

    def test_cannot_check_review_qa_without_permissions(self):
        user_1 = UserFactory.create()
        user_2 = UserFactory.create()

        content_type = ContentType.objects.get_for_model(Experiment)
        permission = Permission.objects.get(
            content_type=content_type, codename="can_check_QA_signoff"
        )
        user_1.user_permissions.add(permission)
        self.assertTrue(user_1.has_perm("experiments.can_check_QA_signoff"))
        self.assertFalse(user_2.has_perm("experiments.can_check_QA_signoff"))

        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_REVIEW)

        self.request.user = user_2
        form = ExperimentReviewForm(
            request=self.request, data={"review_qa": True}, instance=experiment
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()
        self.assertFalse(experiment.review_qa)

        self.request.user = user_1

        form = ExperimentReviewForm(
            request=self.request, data={"review_qa": True}, instance=experiment
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertTrue(experiment.review_qa)

class TestExperimentStatusForm(
    MockBugzillaMixin, MockRequestMixin, MockTasksMixin, TestCase
):

    def test_form_allows_valid_state_transition_and_creates_changelog(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        form = ExperimentStatusForm(
            request=self.request,
            data={"status": experiment.STATUS_REVIEW},
            instance=experiment,
        )
        self.assertTrue(form.is_valid())
        experiment = form.save()
        self.assertEqual(experiment.status, experiment.STATUS_REVIEW)
        change = experiment.changes.latest()
        self.assertEqual(change.old_status, experiment.STATUS_DRAFT)
        self.assertEqual(change.new_status, experiment.STATUS_REVIEW)

    def test_form_rejects_invalid_state_transitions(self):
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        form = ExperimentStatusForm(
            request=self.request,
            data={"status": experiment.STATUS_LIVE},
            instance=experiment,
        )
        self.assertFalse(form.is_valid())

    def test_sets_bugzilla_id_when_draft_becomes_review(self):
        experiment = ExperimentFactory.create_with_status(
            Experiment.STATUS_DRAFT, bugzilla_id=None
        )
        form = ExperimentStatusForm(
            request=self.request,
            data={"status": experiment.STATUS_REVIEW},
            instance=experiment,
        )
        self.assertTrue(form.is_valid())
        experiment = form.save()
        self.mock_tasks_create_bug.delay.assert_called_with(self.user.id, experiment.id)

    def test_adds_bugzilla_comment_and_normandy_slug_when_becomes_ship(self):
        experiment = ExperimentFactory.create_with_status(
            target_status=Experiment.STATUS_REVIEW,
            type=Experiment.TYPE_PREF,
            name="Experiment Name",
            slug="experiment-slug",
            firefox_min_version="57.0",
            firefox_max_version="",
            firefox_channel=Experiment.CHANNEL_NIGHTLY,
            bugzilla_id="12345",
        )
        self.assertEqual(experiment.normandy_slug, None)

        form = ExperimentStatusForm(
            request=self.request,
            data={"status": experiment.STATUS_SHIP},
            instance=experiment,
        )

        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(
            experiment.normandy_slug, "pref-experiment-name-nightly-57-bug-12345"
        )
        self.mock_tasks_update_experiment_bug.delay.assert_called_with(
            self.user.id, experiment.id
        )

class TestExperimentCommentForm(MockRequestMixin, TestCase):

    def test_form_creates_comment(self):
        text = "hello"
        section = Experiment.SECTION_OVERVIEW
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        form = ExperimentCommentForm(
            request=self.request,
            data={"experiment": experiment.id, "section": section, "text": text},
        )
        self.assertTrue(form.is_valid())
        comment = form.save()
        self.assertEqual(comment.experiment, experiment)
        self.assertEqual(comment.section, section)
        self.assertEqual(comment.created_by, self.user)
        self.assertEqual(comment.text, text)

    def test_section_must_be_valid(self):
        text = "hello"
        section = "invalid section"
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        form = ExperimentCommentForm(
            request=self.request,
            data={"experiment": experiment.id, "section": section, "text": text},
        )
        self.assertFalse(form.is_valid())
        self.assertIn("section", form.errors)

    def test_text_is_required(self):
        text = ""
        section = Experiment.SECTION_OVERVIEW
        experiment = ExperimentFactory.create_with_status(Experiment.STATUS_DRAFT)
        form = ExperimentCommentForm(
            request=self.request,
            data={"experiment": experiment.id, "section": section, "text": text},
        )
        self.assertFalse(form.is_valid())
        self.assertIn("text", form.errors)

class TestExperimentArchiveForm(MockRequestMixin, MockTasksMixin, TestCase):

    def test_form_flips_archive_bool(self):

        experiment = ExperimentFactory.create(archived=False)

        form = ExperimentArchiveForm(self.request, instance=experiment, data={})
        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.assertEqual(self.mock_tasks_update_bug_resolution.delay.call_count, 1)
        self.assertTrue(experiment.archived)
        self.assertEqual(experiment.changes.latest().message, "Archived Delivery")

        form = ExperimentArchiveForm(self.request, instance=experiment, data={})
        self.assertTrue(form.is_valid())

        experiment = form.save()
        self.assertEqual(self.mock_tasks_update_bug_resolution.delay.call_count, 2)
        self.assertFalse(experiment.archived)
        self.assertEqual(experiment.changes.latest().message, "Unarchived Delivery")

    def test_form_stays_unarchived_when_live(self):
        self.assertEqual(Notification.objects.count(), 0)
        experiment = ExperimentFactory.create(
            archived=False, status=Experiment.STATUS_LIVE
        )

        form = ExperimentArchiveForm(self.request, instance=experiment, data={})
        self.assertTrue(form.is_valid())
        experiment = form.save()

        self.mock_tasks_update_bug_resolution.delay.assert_not_called()
        self.assertFalse(experiment.archived)
        self.assertEqual(Notification.objects.count(), 1)

class TestExperimentSubscribedForm(MockRequestMixin, TestCase):

    def test_form_adds_subscribers(self):
        experiment = ExperimentFactory.create()

        self.assertFalse(self.user in experiment.subscribers.all())

        form = ExperimentSubscribedForm(self.request, instance=experiment, data={})
        self.assertTrue(form.is_valid())

        experiment = form.save()
        self.assertTrue(self.user in experiment.subscribers.all())

    def test_form_removes_subscribers(self):
        experiment = ExperimentFactory.create(subscribers=[self.user])

        self.assertTrue(self.user in experiment.subscribers.all())

        form = ExperimentSubscribedForm(self.request, instance=experiment, data={})
        self.assertTrue(form.is_valid())

        experiment = form.save()
        self.assertFalse(self.user in experiment.subscribers.all())

class TestNormandyIdForm(MockRequestMixin, TestCase):

    def test_form_not_valid_with_bad_main_id(self):
        experiment = ExperimentFactory.create()

        form = NormandyIdForm(
            self.request,
            instance=experiment,
            data={"normandy_id": "aaaa", "other_normandy_ids": "434"},
        )

        self.assertFalse(form.is_valid())

    def test_form_not_valid_with_bad_other_ids(self):
        experiment = ExperimentFactory.create()

        form = NormandyIdForm(
            self.request,
            instance=experiment,
            data={"normandy_id": "4343", "other_normandy_ids": "434, aaa"},
        )

        self.assertFalse(form.is_valid())

    def test_form_not_valid_when_other_ids_duplicate_main_id(self):
        experiment = ExperimentFactory.create()

        form = NormandyIdForm(
            self.request,
            instance=experiment,
            data={"normandy_id": "4343", "other_normandy_ids": "4343"},
        )

        self.assertFalse(form.is_valid())

    def test_form_valid_when_other_ids_formatted_correctly(self):
        experiment = ExperimentFactory.create()

        form = NormandyIdForm(
            self.request,
            instance=experiment,
            data={"normandy_id": "4343", "other_normandy_ids": "4323, 5671"},
        )

        self.assertTrue(form.is_valid())

    def test_form_changelog_has_message(self):
        experiment = ExperimentFactory.create()

        form = NormandyIdForm(
            self.request,
            instance=experiment,
            data={"normandy_id": "4343", "other_normandy_ids": "443"},
        )

        self.assertTrue(form.is_valid())

        form.save()

        self.assertTrue(experiment.changes.latest().message, "Recipe ID(s) Added")

class TestExperimentOrderingForm(TestCase):

    def test_accepts_valid_ordering(self):
        ordering = ExperimentOrderingForm.ORDERING_CHOICES[1][0]
        form = ExperimentOrderingForm({"ordering": ordering})
        self.assertTrue(form.is_valid())

    def test_rejects_invalid_ordering(self):
        form = ExperimentOrderingForm({"ordering": "invalid ordering"})
        self.assertFalse(form.is_valid())


"""
Data Transform Plugin
"""

import _pickle as cPickle
from datetime import datetime, timedelta
from difflib import SequenceMatcher
import logging
import os
import re

from bson.objectid import ObjectId
import psycopg2
from pymongo import MongoClient

from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin

class NRSRTransformOperator(BaseOperator):
    """
    Data Transform
    """

    def __init__(self, data_type, period, daily,
                 postgres_url, mongo_settings, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.data_type = data_type
        self.period = period
        self.daily = daily

        mongo_client = MongoClient(mongo_settings['uri'])
        mongo_db = mongo_client[mongo_settings['db']]
        self.mongo_col = mongo_db[mongo_settings['col']]
        self.mongo_outcol = mongo_db[mongo_settings['outcol']]
        self.postgres_url = postgres_url

    def _fields_to_dict(self, fields):
        """
        Convert list of fields to mongodb field dict
        """
        return {x: 1 for x in fields}

    def _insert_documents(self, documents, remove=[]):
        """
        Insert into MongoDB
        """

        if remove:
            self.mongo_outcol.remove({'type': {'$in': remove}})

        self.mongo_outcol.insert_many(documents)

    def _get_documents(self, fields_dict, *aggregation, unwind=None, projection=None):
        """
        Get MongoDB Cursor
        """

        filter_dict = {'type': self.data_type}
        if self.period:
            filter_dict['period_num'] = self.period
        if self.daily and self.data_type not in ['daily_club']:
            now = datetime.utcnow() - timedelta(days=30)
            filter_dict['_id'] = {'$gte': ObjectId.from_datetime(now)}

        if unwind and projection:
            docs = self.mongo_col.aggregate([
                {'$match': filter_dict},
                unwind,
                projection
            ])
        elif aggregation:
            aggregation = [{'$match': filter_dict}] + list(aggregation)
            docs = self.mongo_col.aggregate(aggregation)
        else:
            docs = self.mongo_col.find(filter_dict, fields_dict)
        return docs

    def _copy_doc(self, document):
        return cPickle.loads(cPickle.dumps(document, -1))

    def _get_wanted_keys(self, document, keys):
        """Reduce dictionary to wanted keys only"""
        return {k: v for k, v in document.items() if k in keys}

    def transform_members(self):
        """
        Transform MPs
        """

        fields_list = [
            'external_id',
            'forename',
            'surname',
            # 'title',
            'stood_for_party',
            # 'born',
            # 'nationality',
            # 'residence',
            # 'county',
            # 'email',
            'image_urls',
            'period_num',
            'url',
            'memberships',
            'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        new_docs = []

        residence_replacements = {
            'Tvrdošín-Medvedzie': 'Medvedzie',
            'Kostolná-Záriečie': 'Kostolná - Záriečie',
            'Ivánka pri Dunaji': 'Ivanka pri Dunaji',
            'Nový Život - Eliášovce': 'Eliášovce',
            'Michalovce - Čečehov': 'Čečehov',
            'Hnúšťa - Likier': 'Likier'
        }
        county_replacements = {
            'Trenčín': 'Trenčiansky',
            'Nitra': 'Nitriansky',
        }
        nationality_replacements = {
            'slovenská': 'slovak',
            'maďarská': 'hungarian',
            'rómska': 'romani',
            'rusínska': 'rusyn',
            'ruská': 'russian',
            'česká': 'czech',
            'ukrajinská': 'ukrainian',
            '': 'unknown'
        }

        # pg_conn = psycopg2.connect(self.postgres_url)
        # pg_cursor = pg_conn.cursor()

        # wanted_keys = [
        #     'external_id', 'period_num', 'forename', 'surname', 'title',
        #     'email', 'born', 'nationality',
        #     'external_photo_url', 'stood_for_party', 'url', 'type'
        # ]
        wanted_keys = [
            'external_id', 'external_photo_url', 'period_num',
            'forename', 'surname', 'stood_for_party', 'url', 'type'
        ]

        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            for key, val in new_doc.items():
                if isinstance(val, str):
                    new_doc[key] = val.strip()
            new_doc['external_photo_url'] = new_doc['image_urls'][0]
            # GDPR
            # if new_doc['residence'] in residence_replacements:
            #     new_doc['residence'] = residence_replacements[new_doc['residence']]

            # if new_doc['external_id'] == 184:
            #     new_doc['residence'] = 'Bratislava'

            # if new_doc['residence'] == '':
            #     new_doc['residence'] = 'Neuvedené'

            # if new_doc['county'] in county_replacements:
            #     new_doc['county'] = county_replacements[new_doc['county']]

            # if new_doc['nationality'] in nationality_replacements:
            #     new_doc['nationality'] = nationality_replacements[new_doc['nationality']]

            # region = new_doc['county'] if 'kraj' in new_doc['county'] else '{} kraj'.format(new_doc['county'])
            # residence_query = """
            #     SELECT V.id, V.full_name, D.name, R.name
            #     FROM geo_village V
            #     INNER JOIN geo_district D ON V.district_id = D.id
            #     INNER JOIN geo_region R ON (D.region_id = R.id)
            #     WHERE V.full_name = '{village}' AND R.name = '{region}'
            #     """.format(village=new_doc['residence'], region=region)
            # pg_cursor.execute(residence_query)
            # villages = pg_cursor.fetchall()
            # residence_id = None
            # if not villages:
            #     raise Exception("Residence can't be paired for {}".format(new_doc))

            # elif len(villages) > 1 and new_doc['external_id'] == 708:
            #     for village in villages:
            #         if village[2] == 'Žarnovica':
            #             residence_id = village[0]
            #     if not residence_id:
            #         raise Exception(
            #             "Residence doesn't match on {} for {}".format(new_doc, villages))
            # else:
            #     residence_id = villages[0][0]

            # if not residence_id:
            #     raise Exception("residence_id for {} is None".format(new_doc))

            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            # TODO: broken nrsr data workaround
            if new_doc['external_id'] == 1080:
                new_doc['stood_for_party'] = 'OBYČAJNÍ ĽUDIA a nezávislé osobnosti (OĽANO), NOVA, Kresťanská únia (KÚ), ZMENA ZDOLA'
            if new_doc['external_id'] == 874:
                new_doc['stood_for_party'] = 'SMER – sociálna demokracia'
            if new_doc['external_id'] == 966:
                new_doc['stood_for_party'] = 'Sloboda a Solidarita'
            # new_doc['residence_id'] = residence_id
            new_docs.append(new_doc)
        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_member_changes(self):
        """
        Transform MP changes
        """
        fields_list = [
            'external_id',
            'date',
            'period_num',
            'change_type',
            'change_reason',
            'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        new_docs = []
        change_type = {
            "Mandát sa neuplatňuje": 0,
            "Mandát vykonávaný (aktívny poslanec)": 1,
            "Mandát náhradníka zaniknutý": 2,
            "Mandát náhradníka vykonávaný": 3,
            "Mandát náhradníka získaný": 4,
            "Mandát zaniknutý": 5,
            "Mandát nadobudnutý vo voľbách": 6
        }

        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['change_type'] = change_type[doc['change_type']]
            del new_doc['_id']
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_presses(self):
        """
        Transform Press data
        """
        fields_list = [
            'title',
            'press_num',
            'group_num',
            'period_num',
            'press_type',
            'date',
            'attachments_urls',
            'attachments_names',
            'url',
            'type'
        ]

        fields_dict = self._fields_to_dict(fields_list)

        press_type_replacements = {
            'Návrh zákona': 1,
            'Iný typ': 2,
            'Informácia': 7,
            'Správa': 5,
            'Petícia': 3,
            'Medzinárodná zmluva': 4,

        }

        wanted_keys = ['press_type', 'title', 'press_num', 'date', 'period_num', 'url', 'type']
        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['press_type'] = press_type_replacements[new_doc['press_type']]
            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_sessions(self):
        """
        Transform Session data
        """

        fields_list = ['url', 'period_num', 'external_id', 'name', 'program_points', 'type']
        fields_dict = self._fields_to_dict(fields_list)

        state_replacements = {
            'Prerokovaný bod programu': 0,
            'Neprerokovaný bod programu': 1,
            'Presunutý bod programu': 2,
            'Stiahnutý bod programu': 3,
            'Prerušené rokovanie o bode programu': 4,
        }

        wanted_keys = ['external_id', 'period_num', 'name', 'type', 'url', 'program_points', 'session_num']
        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            try:
                new_doc['session_num'] = int(''.join(re.findall(r'\d+', new_doc['name'][:15])))
            except ValueError:
                new_doc['session_num'] = None

            program_points = []
            for point in new_doc['program_points']:
                program_points.append({
                    'state': state_replacements[point['state']],
                    'point_num': point['progpoint'],
                    'press_num': point['parlpress'],
                    'text1': point['text'][0],
                    'text2': point['text'][1],
                    'text3': point['text'][2],
                })
            new_doc['program_points'] = program_points
            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_club_members(self):
        """
        Transform club members
        """

        max_voting = list(
            self.mongo_col.find({
                'type': 'voting'
            }, {
                'datetime': 1
            }).sort([('datetime', -1)]).limit(1))[0]['datetime'].replace(
                hour=12, minute=0, second=0, microsecond=0)

        # TODO(Jozef): the if/else condition won't work on last day of period
        aggregation = [
            {
                '$unwind': '$clubs'
            },
            {
                '$project': {
                    'period_num': 1,
                    'date': 1,
                    'club_name': {
                        '$arrayElemAt': ['$clubs', 0]
                    },
                    'club_values': {
                        '$arrayElemAt': ['$clubs', 1]
                    }
                }
            }, {
                '$unwind': '$club_values'
            }, {
                '$sort': {
                    'club_values': 1,
                    'date': 1
                }
            },
            {
                '$group': {
                    '_id': {
                        'period_num': '$period_num',
                        'member_external_id': '$club_values',
                        'club': '$club_name',
                        'date': '$date',
                    },
                }
            },
            {
                '$project': {
                    '_id': '$_id',
                    'period_num': '$_id.period_num',
                    'member_external_id': '$_id.member_external_id',
                    'date': '$_id.date',
                    'club': '$_id.club',
                }
            },
            {'$sort': {'_id.period_num': 1, '_id.member_external_id': 1, '_id.date': 1}}
        ]
        member_sequence = {}
        docs = list(self._get_documents({}, *aggregation))
        docs_len = len(docs)

        pg_conn = psycopg2.connect(self.postgres_url)
        pg_cursor = pg_conn.cursor()

        active_query = """
        SELECT 1 FROM person_person PP
            INNER JOIN parliament_member PM ON PM.person_id = PP.id
            INNER JOIN parliament_memberactive PMA ON PMA.member_id = PM.id
        WHERE PMA.start <= '{date}' AND (PMA.end > '{date}' OR PMA.end IS NULL)
        AND PP.external_id = {external_id}
        LIMIT 1
        """

        for index, doc in enumerate(docs):
            _id = doc['_id']
            period_num = _id['period_num']
            club_key = _id['club']
            member_key = _id['member_external_id']

            if period_num not in member_sequence:
                member_sequence[period_num] = {}

            if club_key not in member_sequence[period_num]:
                member_sequence[period_num][club_key] = {}

            if member_key not in member_sequence[period_num][club_key]:
                member_sequence[period_num][club_key][member_key] = []

            member_link = member_sequence[period_num][club_key][member_key]

            if index < docs_len - 1:
                next_item = docs[index + 1]['_id']
            else:
                next_item = None

            if member_link:
                # last was ended, create new
                if member_link[-1][2]:
                    member_link.append([_id['date'], None, False])
            else:
                member_link.append([_id['date'], None, False])

            current = member_link[-1]

            if next_item:
                if next_item['member_external_id'] == member_key and next_item['club'] != club_key:
                    if _id['date'] == max_voting:
                        current[1] = None
                    else:
                        current[1] = _id['date']
                    current[2] = True
                elif next_item['member_external_id'] != member_key:
                    pg_cursor.execute(active_query.format(date=max_voting, external_id=member_key))
                    db_result = pg_cursor.fetchall()
                    if not db_result:
                        current[1] = _id['date']
                        current[2] = True

        if pg_conn is not None:
            pg_conn.close()

        new_docs = []
        for period_num, clubs in member_sequence.items():
            for club, members in clubs.items():
                for member, changes in members.items():
                    for change in changes:
                        new_docs.append({
                            'type': self.data_type,
                            'period_num': period_num,
                            'member_external_id': member,
                            'club': club,
                            'start': change[0],
                            'end': change[1],
                        })

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_votings(self):
        """
        Transform votings and votes
        """

        fields_list = [
            'external_id', 'topic', 'datetime', 'session_num', 'voting_num',
            'result', 'period_num', 'press_num', 'url', 'votes', 'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        result_replacements = {
            "Návrh prešiel": 0,
            "Návrh neprešiel": 1,
            "Parlament nebol uznášaniaschopný": 2
        }

        vote_replacements = {
            "-": -1,
            "Z": 0,
            "P": 1,
            "?": 2,
            "N": 3,
            "0": 4
        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['result'] = result_replacements[new_doc['result']]
            for vote in new_doc['votes']:
                vote['vote'] = vote_replacements[vote['vote']]
            del new_doc['_id']
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_bills(self):
        """
        Transform bills
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'press_num',
            'proposer',
            'delivered',
            'current_state',
            'current_result',
            'category_name',
            'url',
            'type',
        ]

        fields_dict = self._fields_to_dict(fields_list)

        current_state_replacements = {
            "Evidencia": 0,
            "Uzavretá úloha": 1,
            "Rokovanie NR SR": 8,
            "Rokovanie gestorského výboru": 9,
            "I. čítanie": 2,
            "II. čítanie": 3,
            "III. čítanie": 4,
            "Redakcia": 5,
            "Výber poradcov k NZ": 10,
            "Stanovisko k NZ": 7
        }

        current_result_replacements = {
            "(NR SR nebude pokračovať v rokovaní o návrhu zákona)": 0,
            "(NZ vzal navrhovateľ späť)": 1,
            "(Zákon vyšiel v Zbierke zákonov)": 7,
            "(NZ nebol schválený)": 5,
            "(Zákon bol vrátený prezidentom)": 8,
            "(Zápis spoločnej správy výborov)": 4,
            "(Zapísané uznesenie výboru)": 9,
            "(NZ postúpil do II. čítania)": 11,
            "(NZ postúpil do redakcie)": 3,
            "(Výber právneho poradcu)": 10,
            "(Pripravená informácia k NZ)": 6
        }

        category_name_replacements = {
            "Novela zákona": 0,
            "Návrh nového zákona": 1,
            "Iný typ": 2,
            "Petícia": 3,
            "Medzinárodná zmluva": 4,
            "Správa": 5,
            "Ústavný zákon": 6,
            "Informácia": 7,
            "Návrh zákona o štátnom rozpočte": 8,
            "Zákon vrátený prezidentom": 9,

        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            if 'category_name' not in doc:
                # empty bill with no details
                continue
            press_title = ''
            press_num = ''
            if 'press_num' in doc:
                presses = list(self.mongo_col.find({
                    'type': 'press', 'press_num': doc['press_num'], 'period_num': doc['period_num']}))
                if not presses:
                    raise Exception("Missing press for {}".format(doc))
                if len(presses) > 1:
                    raise Exception("Multiple presses for {}".format(doc))
                else:
                    press_title = presses[0]['title']
                    press_num = presses[0]['press_num']

            doc['press_num'] = press_num
            names = []
            proposers = []
            doc['proposer_nonmember'] = ''
            proposer_type = 'NULL'
            if 'proposer' in doc:
                if doc['proposer'].startswith('poslanci NR SR'):
                    doc['proposer_nonmember'] = 'poslanci NR SR'
                    proposer_type = 0
                    try:
                        names = re.match(r'^.*\(([^)]+)\)$', doc['proposer']).groups()[0].split(', ')
                    except AttributeError:
                        names = []
                elif doc['proposer'].startswith('vláda'):
                    proposer_type = 1
                    propre = re.match(r'vláda \((.*)\)', doc['proposer'])
                    if propre and len(propre.groups()) == 1:
                        doc['proposer_nonmember'] = propre.groups()[0]
                    else:
                        doc['proposer_nonmember'] = doc['proposer']
                elif doc['proposer'].startswith('výbor'):
                    proposer_type = 2
                    doc['proposer_nonmember'] = doc['proposer']
                query = """
                    SELECT M.id, S.forename, S.surname FROM parliament_member M
                    INNER JOIN parliament_period P ON M.period_id = P.id
                    INNER JOIN person_person S ON M.person_id = S.id
                    WHERE P.period_num = {period_num}
                    AND S.surname = '{surname}' AND S.forename LIKE '{forename}%'
                    """
                pg_conn = psycopg2.connect(self.postgres_url)
                pg_cursor = pg_conn.cursor()
                for name in names:
                    name_list = name.split('.\xa0')
                    pg_cursor.execute(
                        query.format(
                            period_num=doc['period_num'],
                            forename=name_list[0],
                            surname=name_list[1]
                        )
                    )
                    rows = pg_cursor.fetchall()
                    if len(rows) == 1:
                        proposers.append(rows[0][0])
                    elif len(rows) == 0:
                        raise Exception("Missing member in {}".format(query))
                    else:
                        try:
                            similarities = []
                            for row in rows:
                                similarities.append(
                                    SequenceMatcher(
                                        None, '{} {}'.format(row[1], row[2]),
                                        press_title).ratio())
                            max_ratio = max(similarities)
                            proposers.append(rows[similarities.index(max_ratio)][0])
                        except Exception as exc:
                            raise Exception(
                                "Similarities search failed on {} with {}".format(doc, exc))
                pg_conn.close()
                doc['proposers'] = proposers

            new_doc = self._copy_doc(doc)
            new_doc['proposer_type'] = proposer_type
            try:
                new_doc['current_state'] = current_state_replacements[new_doc['current_state']]
            except KeyError:
                new_doc['current_state'] = None
            try:
                new_doc['current_result'] = current_result_replacements[new_doc['current_result']]
            except KeyError:
                new_doc['current_result'] = None
            new_doc['category_name'] = category_name_replacements[new_doc['category_name']]

            new_doc = self._get_wanted_keys(new_doc, [
                'type', 'period_num', 'press_num', 'external_id', 'delivered',
                'current_state', 'current_result', 'proposers',
                'proposer_nonmember', 'proposer_type', 'category_name', 'url'
            ])
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    # def transform_bill_process_steps(self):
    #     """
    #     Transform bill process steps
    #     """
    #     fields_list = [
    #         'type',

    #         'period_num',

    #         'external_id',
    #         'bill_id',
    #         'body_label',
    #         'meeting_session_num',
    #         'meeting_resolution',
    #         'step_result',

    #         'committees_label',
    #         'slk_label',
    #         'coordinator_label',
    #         'coordinator_meeting_date',??
    #         'coordinator_name',??
    #         'discussed_label',
    #         'sent_standpoint',
    #         'sent_label',
    #         'act_num_label'

    #         'url'
    #     ]

    def transform_debate_appearances(self):
        """
        Debate Appearances
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'debater_name',
            'debater_party',
            'debater_role',
            'start',
            'end',
            'session_num',
            'press_num',
            'appearance_type',
            'appearance_type_addition',
            'video_short_url',
            'text',
        ]

        fields_dict = self._fields_to_dict(fields_list)

        appearance_type_replacements = {
            "-": 0,
            "Doplňujúca otázka / reakcia zadávajúceho": 1,
            "Prednesenie interpelácie": 2,
            "Prednesenie otázky": 3,
            "Uvádzajúci uvádza bod": 4,
            "Vstup predsedajúceho": 5,
            "Vystúpenie": 6,
            "Vystúpenie s faktickou poznámkou": 7,
            "Vystúpenie s procedurálnym návrhom": 8,
            "Vystúpenie spoločného spravodajcu": 9,
            "Vystúpenie v rozprave": 10,
            "Zodpovedanie otázky": 11,
        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['appearance_type'] = appearance_type_replacements[doc['appearance_type']]
            name_list = doc['debater_name'].split(', ')
            doc['debater_forename'] = name_list[1]
            doc['debater_surname'] = name_list[0]
            if not 'text' in doc:
                doc['text'] = ''
            else:
                doc['text'] = ''.join(doc['text'])
            if 'debater_party' not in doc:
                doc['debater_party'] = None
            try:
                party = re.match(r'^\(+(.+)+\)', doc['debater_party'])
                if party:
                    doc['debater_party'] = party.groups()[0]
            except TypeError:
                pass

            if 'debater_role' not in doc:
                doc['debater_role'] = ''
            doc['debater_role'] = doc['debater_role'][2:]

            if 'NRSR' in doc['debater_role'] or 'NR SR' in doc['debater_role']:
                doc['parliament_member'] = True
            else:
                doc['parliament_member'] = False

            new_docs.append(
                self._get_wanted_keys(doc, [
                    'type', 'period_num', 'session_num', 'press_num', 'external_id',
                    'start', 'end', 'debater_forename', 'debater_surname', 'debater_role',
                    'parliament_member', 'debater_party', 'appearance_type',
                    'appearance_type_addition', 'video_short_url', 'text'
                ])
            )

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_interpellations(self):
        """
        Transform Interpellations
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'status',
            'asked_by',
            'description',
            'recipients',
            'date',
            'responded_by',
            'interpellation_session_num',
            'response_session_num',
            'responded_by',
            'press_num',
            'url'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        status_replacements = {
            "Príjem odpovede na interpeláciu": 0,
            "Rokovanie o interpelácii": 1,
            "Uzavretá odpoveď na interpeláciu": 2,
            "Interpelácia na expedíciu": 3,
        }
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['status'] = status_replacements[doc['status']]
            asked_list = doc['asked_by'].split(', ')
            doc['asked_by_forename'] = asked_list[1]
            doc['asked_by_surname'] = asked_list[0]

            new_docs.append(
                self._get_wanted_keys(doc, [
                    'type', 'external_id', 'period_num', 'status',
                    'asked_by_forename', 'asked_by_surname', 'description',
                    'recipients', 'date', 'responded_by', 'interpellation_session_num',
                    'response_session_num', 'responded_by', 'press_num', 'url'
                ])
            )

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_amendments(self):
        """Transform amendments"""
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'press_num',
            'session_num',
            'submitter',
            'other_submitters',
            'date',
            'signed_members',
            'voting_external_id',
            'url'
        ]

        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['submitter'] = doc['submitter'].split(', ')[::-1]

            if 'other_submitters' in doc:
                doc['other_submitters'] = [x.split(', ')[::-1] for x in doc['other_submitters']]

            if 'signed_members' in doc:
                doc['signed_members'] = [x.split(', ')[::-1] for x in doc['signed_members']]
            new_docs.append(self._get_wanted_keys(doc, fields_list))

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_committees(self):
        """Committees and memberships"""
        fields_list = [
            '_id',
            'type',
            'period_num',
            'external_id',
            'name',
            'description',
            'url',
            'members'
        ]

        roles = {
            'predsedníčka': 0,
            'predseda': 1,
            'podpredsedníčka': 2,
            'podpredsedkyňa': 2, # incorrect phrase promoted to nrsr.sk
            'podpredseda': 3,
            'členka': 4,
            'člen': 5,
            'náhradná členka': 6,
            'náhradný člen': 7,
            'overovateľka': 8,
            'overovateľ': 9
        }

        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['description'] = ''.join([x.strip() for x in doc['description']])
            for member in doc['members']:
                member['role'] = roles[member['role']]
                member['start'] = doc['_id'].generation_time
            new_docs.append(self._get_wanted_keys(doc, fields_list))
        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_committee_schedules(self):
        """Transform committee sessions and session points"""
        fields_list = [
            'type',
            'committee_name',
            'period_num',
            'date',
            'time',
            'place',
            'points'
        ]
        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        pg_conn = psycopg2.connect(self.postgres_url)
        pg_cursor = pg_conn.cursor()
        for doc in self._get_documents(fields_dict):
            times = doc['time'].split(' - ')
            if len(times) == 1:
                time_start = times[0]
                time_end = None
            else:
                time_start, time_end = times

            pg_cursor.execute(
                """
                SELECT PC.id FROM parliament_committee PC
                LEFT JOIN parliament_period PP ON PP.id = PC.period_id
                WHERE PC.name = '{committee_name}' AND PP.period_num = {period_num}
                """.format(
                    committee_name=doc['committee_name'],
                    period_num=doc['period_num']
                )
            )
            row = pg_cursor.fetchone()

            try:
                dt_start = datetime.strptime('{} {}'.format(doc['date'], time_start),
                                  '%d. %m. %Y %H:%M')
            except ValueError:
                dt_start = datetime.strptime('{} {}'.format(doc['date'], time_start),
                                  '%d.%m.%Y %H:%M')

            try:
                dt_end = datetime.strptime('{} {}'.format(doc['date'], time_end),
                                  '%d. %m. %Y %H:%M') if time_end else None
            except ValueError:
                dt_end = datetime.strptime('{} {}'.format(doc['date'], time_end),
                                  '%d.%m.%Y %H:%M') if time_end else None

            new_doc = {
                'type': doc['type'],
                'period_num': doc['period_num'],
                'place':
                ''.join([x.strip() for x in doc['place']]),
                'start': dt_start,
                'end': dt_end,
                'committee_id':
                row[0]
            }
            points = []
            pg_cursor.execute(
                """
                SELECT PRESS.id, PRESS.press_num FROM parliament_press PRESS
                LEFT JOIN parliament_period PER ON PER.id = PRESS.period_id
                WHERE PER.period_num = {period_num}
                """.format(period_num=doc['period_num'])
            )
            presses = {x[1]: x[0] for x in pg_cursor.fetchall()}
            for point in doc['points']:
                index = None
                text = ''.join([x.strip() for x in point['text']])
                if not text:
                    continue
                is_index = re.match(r'^([0-9]\. )([\s\S]*)$', text)
                if is_index:
                    index = is_index.groups()[0].strip().replace('.', '')
                    text = ''.join(is_index.groups()[1:])

                press_id = None
                if point['press_num']:
                    press_id = presses[str(point['press_num'])]
                points.append({
                    'press_id': press_id,
                    'index': index,
                    'topic': text
                })
            new_doc['points'] = points
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

        if pg_conn is not None:
            pg_conn.close()

    def execute(self, context):
        """Operator Executor"""
        if self.data_type == 'member':
            self.transform_members()
        elif self.data_type == 'member_change':
            self.transform_member_changes()
        elif self.data_type == 'press':
            self.transform_presses()
        elif self.data_type == 'session':
            self.transform_sessions()
        # elif self.data_type == 'club':
        #     data_frame = self.transform_clubs()
        elif self.data_type == 'daily_club':
            self.transform_club_members()
        elif self.data_type == 'voting':
            self.transform_votings()
        elif self.data_type == 'bill':
            self.transform_bills()
        # elif self.data_type == 'bill_step':
        #     self.transform_bill_process_steps()
        elif self.data_type == 'debate_appearance':
            self.transform_debate_appearances()
        elif self.data_type == 'interpellation':
            self.transform_interpellations()
        elif self.data_type == 'amendment':
            self.transform_amendments()
        elif self.data_type == 'committee':
            self.transform_committees()
        elif self.data_type == 'committeeschedule':
            self.transform_committee_schedules()

class NRSRTransformPlugin(AirflowPlugin):

    name = 'nrsr_transform_plugin'
    operators = [NRSRTransformOperator]
    hooks = []
    executors = []
    macros = []
    admin_views = []
    flask_blueprints = []
    menu_links = []


import tkinter as tk
from tkinter import ttk
from tkinter import scrolledtext
#====main window=====
app = tk.Tk()
app.title("Progress bars")
app.geometry('400x600')

vertical = tk.DoubleVar(value = 15)
scl = ttk.Scale(
    app,
    command = lambda value: print(vertical.get()),
    from_ = 0,
    to = 30,
    length = 300,
    orient = 'vertical',
    variable = vertical
)
scl.pack()

# ====progress bar=====
progress = ttk.Progressbar(
    app,
    variable = vertical,
    maximum = 30,
    orient = 'horizontal',
    length = 400
)
progress.pack()

# ====scrolled text ======
sc = scrolledtext.ScrolledText(app, height = 10)
sc.pack()
# =======numeric progress========
np = tk.IntVar()
npp = ttk.Progressbar(app, orient = 'vertical', variable = np)
npp.pack()
npp.start()

indicatr = ttk.Label(app, textvariable = np)
indicatr.pack()

# =======another scale=======
ascale2 = ttk.Scale (app, variable = np, from_ = 0, to = 100)
ascale2.pack()

# =====run app======
app.mainloop()