from datetime import datetime, date
from typing import Collection, Any
from decimal import Decimal

from rdr_service import clock
from rdr_service.code_constants import CONSENT_FOR_STUDY_ENROLLMENT_MODULE, EMPLOYMENT_ZIPCODE_QUESTION_CODE, \
    PMI_SKIP_CODE, \
    STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, ZIPCODE_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE, \
    CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE, EHR_CONSENT_QUESTION_CODE
from rdr_service.dao.consent_dao import ConsentDao
from rdr_service.etl.model.src_clean import SrcClean, Observation, PidRidMapping, Person, Measurement, Death, \
    EHRConsentStatus
from rdr_service.code_constants import CONSENT_FOR_STUDY_ENROLLMENT_MODULE, EMPLOYMENT_ZIPCODE_QUESTION_CODE, PMI_SKIP_CODE,\
    STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, ZIPCODE_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE,\
    WEAR_CONSENT_QUESTION_CODE, WEAR_YES_ANSWER_CODE, WEAR_CONSENT_MODULE
from rdr_service.etl.model.src_clean import SrcClean, Observation, PidRidMapping, Person, Measurement, Death, \
    WearConsent
from rdr_service.model.code import Code
from rdr_service.model.consent_file import ConsentFile
from rdr_service.model.consent_response import ConsentResponse
from rdr_service.model.measurements import PhysicalMeasurements
from rdr_service.model.measurements import Measurement as RdrMeasurement
from rdr_service.model.participant import Participant
from rdr_service.model.deceased_report import DeceasedReport
from rdr_service.model.api_user import ApiUser
from rdr_service.dao.physical_measurements_dao import PhysicalMeasurementsDao
from rdr_service.dao.curation_etl_dao import CdrEtlRunHistoryDao, CdrEtlSurveyHistoryDao
from rdr_service.participant_enums import QuestionnaireResponseStatus, QuestionnaireResponseClassificationType, \
    PhysicalMeasurementsCollectType, OriginMeasurementUnit, DeceasedNotification, DeceasedReportStatus
from rdr_service.tools.tool_libs.curation import CurationExportClass
from tests.helpers.unittest_base import BaseTestCase
from tests.helpers.tool_test_mixin import ToolTestMixin
from tests import test_data
from rdr_service.clock import FakeClock

TIME = datetime(2000, 1, 10)
TIME_2 = datetime(2022, 5, 10)

WEAR_NO_ANSWER_CODE = "wear_no"

class CurationEtlTest(ToolTestMixin, BaseTestCase):
    def setUp(self):
        super(CurationEtlTest, self).setUp(with_consent_codes=True)
        self._setup_data()
        self.history_dao = CdrEtlRunHistoryDao()
        self.pm_dao = PhysicalMeasurementsDao()

    def _setup_data(self):
        self.participant = self.data_generator.create_database_participant()
        self.data_generator.create_database_participant_summary(
            participant=self.participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))

        self.module_code = self.data_generator.create_database_code(value='src_clean_test')
        self.question_code_list = []
        self.indexed_answers = []
        self.questionnaire = self.data_generator.create_database_questionnaire_history()
        for question_index in range(4):
            question_code = self.data_generator.create_database_code(value=f'q_{question_index}')
            self.question_code_list.append(question_code)
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=self.questionnaire.questionnaireId,
                questionnaireVersion=self.questionnaire.version,
                codeId=question_code.codeId
            )
            answer_code = self.data_generator.create_database_code(value=f'a_{question_index}')
            self.indexed_answers.append((question_index, 'valueCodeId', answer_code.codeId))

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=self.questionnaire.questionnaireId,
            questionnaireVersion=self.questionnaire.version,
            codeId=self.module_code.codeId
        )

        self.questionnaire_response = self._setup_questionnaire_response(self.participant, self.questionnaire,
                                                                         indexed_answers=self.indexed_answers)

    def _setup_questionnaire_response(self, participant, questionnaire, authored=datetime(2020, 3, 15),
                                      created=datetime(2020, 3, 15), indexed_answers=None, ignored_answer_indexes=None,
                                      status=QuestionnaireResponseStatus.COMPLETED,
                                      classification_type=QuestionnaireResponseClassificationType.COMPLETE):

        if not ignored_answer_indexes:
            ignored_answer_indexes = []

        questionnaire_response = self.data_generator.create_database_questionnaire_response(
            participantId=participant.participantId,
            questionnaireId=questionnaire.questionnaireId,
            questionnaireVersion=questionnaire.version,
            authored=authored,
            created=created,
            status=status,
            classificationType=classification_type
        )

        if indexed_answers is None:
            # If no answers were specified then answer all questions with 'test answer'
            indexed_answers = [
                (question_index, 'valueString', 'test answer')
                for question_index in range(len(questionnaire.questions))
            ]

        for question_index, answer_field_name, answer_string in indexed_answers:
            question = questionnaire.questions[question_index]
            self.data_generator.create_database_questionnaire_response_answer(
                questionnaireResponseId=questionnaire_response.questionnaireResponseId,
                questionId=question.questionnaireQuestionId,
                **{answer_field_name: answer_string},
                ignore=question_index in ignored_answer_indexes
            )

        return questionnaire_response

    def _exists_in_src_clean(self, column_name:str, value:Any) -> bool:
        value_exists = self.session.query(
            getattr(SrcClean,column_name)
        ).filter(
            getattr(SrcClean,column_name) == value
        ).distinct().scalar()
        return bool(value_exists
                    )

    @staticmethod
    def run_cdm_data_generation(cutoff=None, vocabulary='gs://curation-vocabulary/aou_vocab_20220201/',
                                participant_origin='all', participant_list_file=None, include_surveys=None,
                                exclude_surveys=None, exclude_participants=None, omit_surveys=False,
                                omit_measurements=False, exclude_in_person_pm=False, exclude_remote_pm=False,
                                prep_bq=False):
        CurationEtlTest.run_tool(CurationExportClass, tool_args={
            'command': 'cdm-data',
            'cutoff': cutoff,
            'vocabulary': vocabulary,
            'participant_origin': participant_origin,
            'participant_list_file': participant_list_file,
            'include_surveys': include_surveys,
            'exclude_surveys': exclude_surveys,
            'exclude_participants': exclude_participants,
            'omit_surveys': omit_surveys,
            'omit_measurements': omit_measurements,
            "exclude_in_person_pm": exclude_in_person_pm,
            "exclude_remote_pm": exclude_remote_pm,
            "prep_bq": prep_bq
        })

    @staticmethod
    def run_exclude_code_command(operation, code_value, code_type):
        CurationEtlTest.run_tool(CurationExportClass, tool_args={
            'command': 'exclude-code',
            'operation': operation,
            'code_value': code_value,
            'code_type': code_type
        })

    def test_locking(self):
        """Make sure that building the CDM tables doesn't take exclusive locks"""

        # Take an exclusive lock on the participant, one of the records known to be part of the insert query
        self.session.query(Participant).filter(
            Participant.participantId == self.participant.participantId
        ).with_for_update().one()

        # This will time out if the tool tries to take an exclusive lock on the participant
        self.run_cdm_data_generation()

    def _src_clean_record_found_for_response(self, questionnaire_response_id):
        response_record = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == questionnaire_response_id
        ).one_or_none()
        return response_record is not None

    def test_latest_questionnaire_response_used(self):
        """The latest questionnaire response received for a module should be used"""
        # Note: this only applies to modules that shouldn't roll up answers (ConsentPII should be rolled up)

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10)
        )

        # Check that we are only be seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def _create_consent_questionnaire(self):
        module_code = self.session.query(Code).filter(Code.value == CONSENT_FOR_STUDY_ENROLLMENT_MODULE).one()
        consent_question_codes = [
            self.data_generator.create_database_code(value=f'consent_q_code_{question_index}')
            for question_index in range(4)
        ]
        consent_question_codes += self.session.query(Code).filter(Code.value.in_([
            STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE, DATE_OF_BIRTH_QUESTION_CODE
        ])).all()

        consent_questionnaire = self.data_generator.create_database_questionnaire_history()
        for consent_question_code in consent_question_codes:
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=consent_questionnaire.questionnaireId,
                questionnaireVersion=consent_questionnaire.version,
                codeId=consent_question_code.codeId
            )

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=consent_questionnaire.questionnaireId,
            questionnaireVersion=consent_questionnaire.version,
            codeId=module_code.codeId
        )

        return consent_questionnaire

    def test_consent_response_answers_roll_up(self):
        """
        For the consent survey, all of the most recent answers for a code should be used
        even if they were in previous responses.
        """

        consent_questionnaire = self._create_consent_questionnaire()

        self._setup_questionnaire_response(self.participant, consent_questionnaire)
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (1, 'valueString', 'NewLastName'),
                (3, 'valueString', 'new-email')
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (2, 'valueString', 'updated address'),
                (3, 'valueString', 'corrected-email')
            ],
            authored=datetime(2020, 8, 1)
        )

        # Check that the newest answer is in the src_clean, even if it wasn't from the latest response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(consent_questionnaire.questions):
            expected_answer = 'test answer'
            if question_index == 1:
                expected_answer = 'NewLastName'
            elif question_index == 2:
                expected_answer = 'updated address'
            elif question_index == 3:
                expected_answer = 'corrected-email'

            # Since there was an initial response with an answer for every question, then every question
            # should have an answer in the export (even though partial responses updated some of them).
            # There also shouldn't be multiple answers from the participant for any of the survey
            # questions in the export.
            src_clean_answer_query = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one()
            self.assertEqual(expected_answer, src_clean_answer_query.value_string)

    def test_consent_address_roll_up(self):
        """
        For the consent survey, any answers for the first line of the street address should also
        override previous answers for StreetAddress2
        """

        consent_questionnaire = self._create_consent_questionnaire()

        # Set up a response that answers all the questions, including the two address lines
        self._setup_questionnaire_response(self.participant, consent_questionnaire)
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            authored=datetime(2020, 5, 1)
        )

        # Enter another response that just updates the first line of the street address
        expected_final_address = '42 Wallaby Way'
        self._setup_questionnaire_response(
            self.participant,
            consent_questionnaire,
            indexed_answers=[
                (4, 'valueString', expected_final_address)  # Assuming the 4th question is the first line of the address
            ],
            authored=datetime(2020, 8, 1)
        )

        # Check that the only address answer in src_clean is the updated line 1 for the street address (which will
        # also replace line 2 for the the first response)
        self.run_cdm_data_generation()

        # Load all src_clean rows for lines 1 and 2 of the address
        address_answers = self.session.query(SrcClean).join(
            Code, Code.codeId == SrcClean.question_code_id
        ).filter(
            Code.value.in_([STREET_ADDRESS_QUESTION_CODE, STREET_ADDRESS2_QUESTION_CODE])
        ).all()

        # Make sure we only get the updated address and nothing from the original response
        self.assertEqual(
            1,
            len(address_answers),
            'The updated street address should overwrite the previous answers for line 1 and 2 of the address'
        )
        self.assertEqual(expected_final_address, address_answers[0].value_string)

    def test_in_progress_responses_are_filtered_out_of_export(self):
        """
        We will filter in-progress questionnaire responses out during the ETL process until the curation team is
        ready to start receiving them.
        """

        # Create an in-progress questionnaire response that should not appear in src_clean
        participant = self.data_generator.create_database_participant()
        self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            status=QuestionnaireResponseStatus.IN_PROGRESS
        )
        self.run_cdm_data_generation()

        # Check that src_clean doesn't have any records for the in-progress questionnaire response
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertEmpty(src_clean_answers)

    def test_later_in_progress_response_not_used(self):
        """
        Make sure later, in-progress responses don't make us filter out full and valid responses that should be used
        """

        # Create a questionnaire response that might be used instead of the default for the test suite
        in_progress_response = self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10),
            status=QuestionnaireResponseStatus.IN_PROGRESS
        )
        self.run_cdm_data_generation()

        # Make sure src_clean only has data from the full response
        in_progress_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == in_progress_response.questionnaireResponseId
        ).all()
        self.assertEmpty(in_progress_answers)

        complete_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == self.questionnaire_response.questionnaireResponseId
        ).all()
        self.assertNotEmpty(complete_answers)

    def test_duplicate_record_in_temp_questionnaire_response_filtered_out(self):
        """
        Test that duplicate record in cdm.tmp_questionnaire_response is not included in export
        """

        # Create a new questionnaire response
        participant = self.data_generator.create_database_participant()
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )

        # Create another response that is a duplicate
        duplicate_response = self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            classification_type=QuestionnaireResponseClassificationType.DUPLICATE
        )

        self.run_cdm_data_generation()

        # Make sure no answers from the duplicate response made it into SrcClean
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertFalse(any(
            answer_record.questionnaire_response_id == duplicate_response.questionnaireResponseId
            for answer_record in src_clean_answers
        ))

    def test_invalid_record_in_temp_questionnaire_response_filtered_out(self):
        """
        Test that invalid record in cdm.tmp_questionnaire_response is not included in export
        """

        # Create a new questionnaire response
        participant = self.data_generator.create_database_participant()
        invalid_response = self._setup_questionnaire_response(
            participant,
            self.questionnaire,
            classification_type=QuestionnaireResponseClassificationType.INVALID
        )

        self.run_cdm_data_generation()

        # Make sure no answers from the response made it into SrcClean
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.participant_id == participant.participantId
        ).all()
        self.assertFalse(any(
            answer_record.questionnaire_response_id == invalid_response.questionnaireResponseId
            for answer_record in src_clean_answers
        ))

    def test_zip_code_maps_to_string_field(self):
        """
        There are some questionnaire responses that have the zip code transmitted to us in the valueInteger
        field. Curation is expecting zip codes to be exported to them as strings. This checks to make sure that
        they're mapped correctly.
        """

        # Two codes have used value_integer for transmitting the value
        employment_zipcode_code = self.data_generator.create_database_code(value=EMPLOYMENT_ZIPCODE_QUESTION_CODE)
        address_zipcode_code = self.data_generator.create_database_code(value=ZIPCODE_QUESTION_CODE)
        skip_code = self.data_generator.create_database_code(value=PMI_SKIP_CODE)

        # Create a questionnaire with zip code questions
        zip_code_questionnaire = self.data_generator.create_database_questionnaire_history()
        for index in range(2):  # Creating four questions to test PMI_SKIP and when the value_string is correctly used
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=zip_code_questionnaire.questionnaireId,
                questionnaireVersion=zip_code_questionnaire.version,
                codeId=employment_zipcode_code.codeId
            )
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=zip_code_questionnaire.questionnaireId,
                questionnaireVersion=zip_code_questionnaire.version,
                codeId=address_zipcode_code.codeId
            )
        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=zip_code_questionnaire.questionnaireId,
            questionnaireVersion=self.questionnaire.version,
            codeId=self.module_code.codeId
        )

        # Set up a response with zip code values transmitted in various ways
        expected_zip_code_answers = [
            (0, 'valueInteger', '90210'),
            (1, 'valueInteger', '12345'),
            (2, 'valueString', '12121'),
            (3, 'valueCodeId', skip_code.codeId),
        ]
        zip_code_response = self._setup_questionnaire_response(
            self.participant,
            zip_code_questionnaire,
            indexed_answers=expected_zip_code_answers
        )

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == zip_code_response.questionnaireResponseId
        ).all()

        for index, answer_field, expected_value in expected_zip_code_answers:
            src_cln: SrcClean = src_clean_answers[index]
            if answer_field == 'valueCodeId':  # An answer of skipping the zip code
                self.assertEqual(expected_value, src_cln.value_code_id)
            else:
                self.assertEqual(expected_value, src_cln.value_string)
                self.assertIsNone(src_cln.value_number)

    def test_exclude_participants_age_under_18(self):
        """
        Curation team request to exclude participants that were under 18 at the time of study consent.
        """
        self._create_consent_questionnaire()
        self.run_cdm_data_generation(omit_measurements=True)
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(4, len(src_clean_answers))

        dob = datetime(1982, 1, 11)
        self.data_generator.create_database_participant_summary(participant=self.participant, dateOfBirth=dob)
        self.run_cdm_data_generation(omit_measurements=True)  # PM query errors out when no PIDs are supplied.
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(0, len(src_clean_answers))

    def test_etl_exclude_code(self):
        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(4, len(src_clean_answers))
        self.session.commit()

        exclude_question_str = self.question_code_list[0].value + ',' + self.question_code_list[1].value
        self.run_exclude_code_command('add', exclude_question_str, 'question')
        exclude_answer_str = 'a_' + str(self.indexed_answers[2][0])
        self.run_exclude_code_command('add', exclude_answer_str, 'answer')

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(1, len(src_clean_answers))
        self.session.commit()

        self.run_exclude_code_command('add', self.module_code.value, 'module')
        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(0, len(src_clean_answers))

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_etl_include_exclude_survey_history(self):
        exclude_question_str = self.question_code_list[0].value + ',' + self.question_code_list[1].value
        self.run_exclude_code_command('add', exclude_question_str, 'question')
        exclude_answer_str = 'a_' + str(self.indexed_answers[2][0])
        self.run_exclude_code_command('add', exclude_answer_str, 'answer')

        self.run_cdm_data_generation()
        src_clean_answers = self.session.query(SrcClean).all()
        self.assertEqual(1, len(src_clean_answers))
        self.session.commit()

        include_exclude_history_dao = CdrEtlSurveyHistoryDao()
        records = include_exclude_history_dao.get_all()
        self.assertEqual(len(records), 6)

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_etl_history(self):
        with FakeClock(TIME_2):
            self.run_cdm_data_generation(cutoff='2022-04-01')
        records = self.history_dao.get_all()
        self.assertEqual(len(records), 1)
        self.assertEqual(records[0].cutoffDate, date(2022, 4, 1))
        self.assertEqual(records[0].startTime, TIME_2)
        self.assertEqual(records[0].endTime, TIME_2)
        self.assertEqual(records[0].vocabularyPath, 'gs://curation-vocabulary/aou_vocab_20220201/')
        self.assertEqual(records[0].filterOptions['participant_origin'],'all')

        self.clear_table_after_test('cdr_etl_survey_history')
        self.clear_table_after_test('cdr_etl_run_history')
        self.clear_table_after_test('cdr_excluded_code')

    def test_cutoff_date(self):
        participant1 = self.data_generator.create_database_participant()
        # consent before cutoff 2022-04-01, should be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant1, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant1,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        participant2 = self.data_generator.create_database_participant()
        # consent after cutoff 2022-04-01, should not be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant2, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 4, 20),
            dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant2,
            self.questionnaire,
            authored=datetime(2022, 4, 20)
        )

        participant3 = self.data_generator.create_database_participant()
        # withdrawal before cutoff 2022-04-01, should not be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant3, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            withdrawalStatus=2, withdrawalAuthored=datetime(2022, 2, 20), dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant3,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        participant4 = self.data_generator.create_database_participant()
        # withdrawal after cutoff 2022-04-01, should be included in the result
        self.data_generator.create_database_participant_summary(
            participant=participant4, consentForStudyEnrollmentFirstYesAuthored=datetime(2022, 2, 1),
            withdrawalStatus=2, withdrawalAuthored=datetime(2022, 4, 20), dateOfBirth=datetime(1982, 1, 9))
        self._setup_questionnaire_response(
            participant4,
            self.questionnaire,
            authored=datetime(2022, 2, 1)
        )

        self.run_cdm_data_generation(cutoff='2022-04-01')

        src_clean_answers_p1 = self.session.query(SrcClean)\
            .filter(SrcClean.participant_id == participant1.participantId).all()
        self.assertEqual(4, len(src_clean_answers_p1))

        src_clean_answers_p2 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant2.participantId).all()
        self.assertEqual(0, len(src_clean_answers_p2))

        src_clean_answers_p3 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant3.participantId).all()
        self.assertEqual(0, len(src_clean_answers_p3))

        src_clean_answers_p4 = self.session.query(SrcClean) \
            .filter(SrcClean.participant_id == participant4.participantId).all()
        self.assertEqual(4, len(src_clean_answers_p4))

    def test_ignored_answers_are_marked_invalid(self):
        """
        Any answers that have the ignore field set to True should give the skip code
        (to be marked invalid in the finalization step)
        """
        questionnaire_response = self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (0, 'valueString', 'first_answer'),
                (1, 'valueString', 'ignored answer'),
                (2, 'valueBoolean', True),
                (3, 'valueDate', datetime.now())
            ],
            ignored_answer_indexes=[1, 2, 3]
        )

        # Check that we are only be seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        src_clean_answers: Collection[SrcClean] = self.session.query(SrcClean).filter(
            SrcClean.questionnaire_response_id == questionnaire_response.questionnaireResponseId
        ).all()

        # Should be getting 4 records still (even though some of them are blank)
        self.assertEqual(4, len(src_clean_answers))
        for index, answer in enumerate(src_clean_answers):
            if index == 0:
                self.assertEqual('first_answer', answer.value_string)
            else:
                # Make sure none of the answer fields have an answer value
                self.assertFalse(any([
                    answer is not None
                    for answer in [
                        answer.value_string,
                        answer.value_number,
                        answer.value_date,
                        answer.value_boolean,
                        answer.value_code_id
                    ]
                ]))
                self.assertEqual(PMI_SKIP_CODE, answer.value_ppi_code)

    def test_cutoff_date_questionnaire_response_used(self):
        """The latest questionnaire response received before the cutoff date for a module should be used"""
        # Note: this only applies to modules that shouldn't roll up answers (ConsentPII should be rolled up)

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'intermediate answer')
            ],
            authored=datetime(2020, 5, 10),
            created=datetime(2020, 5, 10)
        )

        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update2'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2022, 5, 10),
            created=datetime(2022, 5, 10)
        )

        # Check that we are only seeing the answers from the questionnaire response before cutoff date
        self.run_cdm_data_generation(cutoff='2022-04-01')
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'intermediate answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)
        self.session.commit()

        # Returns the newest answers without cutoff
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update2'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def test_exclude_profile_update_questionnaire_response(self):
        """Questionnaire Responses with PROFILE_UPDATE classification should be excluded"""

        # Create a questionnaire response that would be used instead of the default for the test suite
        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'update'),
                (3, 'valueString', 'final answer')
            ],
            authored=datetime(2020, 6, 10),
            created=datetime(2020, 6, 10)
        )

        self._setup_questionnaire_response(
            self.participant,
            self.questionnaire,
            indexed_answers=[
                (1, 'valueString', 'profile update'),
            ],
            authored=datetime(2021, 6, 10),
            created=datetime(2021, 6, 10),
            classification_type=QuestionnaireResponseClassificationType.PROFILE_UPDATE
        )

        # Check that we are only seeing the answers from the latest questionnaire response
        self.run_cdm_data_generation()
        for question_index, question in enumerate(self.questionnaire.questions):
            expected_answer = None
            if question_index == 1:
                expected_answer = 'update'
            elif question_index == 3:
                expected_answer = 'final answer'

            src_clean_answer = self.session.query(SrcClean).filter(
                SrcClean.question_code_id == question.codeId
            ).one_or_none()
            if expected_answer is None:
                self.assertIsNone(src_clean_answer)
            else:
                self.assertEqual(expected_answer, src_clean_answer.value_string)

    def test_participant_origin_filter(self):
        """ Test the participant origin filter selects the intended participants"""
        participant_ce = self.data_generator.create_database_participant(participantOrigin='careevolution')
        self.data_generator.create_database_participant_summary(
            participant=participant_ce,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
        self._setup_questionnaire_response(
            participant_ce,
            self.questionnaire
        )

        participant_vibrent = self.data_generator.create_database_participant(participantOrigin='vibrent')
        self.data_generator.create_database_participant_summary(
            participant=participant_vibrent,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
        self._setup_questionnaire_response(
            participant_vibrent,
            self.questionnaire
        )

        self.run_cdm_data_generation(participant_origin='careevolution')
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertTrue(ce_ppt_exists)
        self.assertFalse(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('careevolution', run_history.filterOptions['participant_origin'])

        self.session.commit()

        self.run_cdm_data_generation(participant_origin='vibrent')
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertFalse(ce_ppt_exists)
        self.assertTrue(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('vibrent', run_history.filterOptions['participant_origin'])

        self.session.commit()

        self.run_cdm_data_generation()
        vibrent_ppt_exists = self._exists_in_src_clean('participant_id', participant_vibrent.participantId)
        ce_ppt_exists = self._exists_in_src_clean('participant_id', participant_ce.participantId)

        self.assertTrue(ce_ppt_exists)
        self.assertTrue(vibrent_ppt_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual('all', run_history.filterOptions['participant_origin'])

    def test_participant_list(self):
        pids = list(range(10000,10010))
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
            self._setup_questionnaire_response(
                participant,
                self.questionnaire
            )

        self.run_cdm_data_generation(
            participant_origin=None,
            participant_list_file=test_data.data_path('test_curation_participant_list.txt')
        )

        test_participants = [10002, 10004, 10007]
        for pid in pids:
            pid_exists = self._exists_in_src_clean("participant_id", pid)
            if pid in test_participants:
                self.assertTrue(pid_exists)
            else:
                self.assertFalse(pid_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertIn('test-data/test_curation_participant_list.txt',
                      run_history.filterOptions['participant_list_file'])

    def _create_questionnaire(self, survey_name, question_code_list=None):
        module_code = self.data_generator.create_database_code(value=survey_name)
        question_codes = [
            self.data_generator.create_database_code(value=f'{survey_name}_q_code_{question_index}')
            for question_index in range(4)
        ]

        if question_code_list:
            question_codes += self.session.query(Code).filter(Code.value.in_([
                question_code_list
            ])).all()

        questionnaire = self.data_generator.create_database_questionnaire_history()
        for question_code in question_codes:
            self.data_generator.create_database_questionnaire_question(
                questionnaireId=questionnaire.questionnaireId,
                questionnaireVersion=questionnaire.version,
                codeId=question_code.codeId
            )

        self.data_generator.create_database_questionnaire_concept(
            questionnaireId=questionnaire.questionnaireId,
            questionnaireVersion=questionnaire.version,
            codeId=module_code.codeId
        )

        return questionnaire

    def test_include_surveys(self):
        survey_names = ['Q1', 'Q2', 'Q3', 'Q4']
        for survey_name in survey_names:
            questionnaire = self._create_questionnaire(survey_name)
            self._setup_questionnaire_response(self.participant, questionnaire)

        self.run_cdm_data_generation(include_surveys=test_data.data_path('test_curation_surveys.txt'))

        survey_results = []
        for survey_name in survey_names:
            survey_results.append((survey_name, self._exists_in_src_clean('survey_name', survey_name)))

        for result in survey_results:
            if result[0] in ['Q1', 'Q4']:
                self.assertFalse(result[1])
            elif result[0] in ['Q2', 'Q3']:
                self.assertTrue(result[1])

        last_run = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual(['Q2', 'Q3'], last_run.filterOptions['include_surveys'])

    def test_exclude_surveys(self):
        survey_names = ['Q1', 'Q2', 'Q3', 'Q4']
        for survey_name in survey_names:
            questionnaire = self._create_questionnaire(survey_name)
            self._setup_questionnaire_response(self.participant, questionnaire)

        self.run_cdm_data_generation(exclude_surveys=test_data.data_path('test_curation_surveys.txt'))

        survey_results = []
        for survey_name in survey_names:
            survey_results.append((survey_name, self._exists_in_src_clean('survey_name', survey_name)))

        for result in survey_results:
            if result[0] in ['Q1', 'Q4', 'src_clean_test']:
                self.assertTrue(result[1], f"{result[0]} expected in src_clean but not found")
            elif result[0] in ['Q2', 'Q3']:
                self.assertFalse(result[1])

        last_run = self.history_dao.get_last_etl_run_info(self.session)
        self.assertEqual(['Q2', 'Q3'], last_run.filterOptions['exclude_surveys'])

    def test_exclude_participants(self):
        pids = list(range(10000,10010))
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10))
            self._setup_questionnaire_response(
                participant,
                self.questionnaire
            )

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_participants=test_data.data_path('test_curation_participant_list.txt')
        )

        excluded_participants = [10002, 10004, 10007]
        for pid in pids:
            pid_exists = self._exists_in_src_clean("participant_id", pid)
            if pid in excluded_participants:
                self.assertFalse(pid_exists)
            else:
                self.assertTrue(pid_exists)

        run_history = self.history_dao.get_last_etl_run_info(self.session)
        self.assertIn('test-data/test_curation_participant_list.txt',
                         run_history.filterOptions['participant_exclude_file'])

    def test_survey_src_id(self):
        consent_questionnaire = self._create_consent_questionnaire()

        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        obs_src_id = self.session.query(
            Observation.src_id
        ).filter(
            Observation.person_id == participant.participantId
        ).first()[0]

        prm_src_id = self.session.query(
            PidRidMapping.src_id
        ).filter(
            PidRidMapping.person_id == participant.participantId
        ).first()[0]

        person_src_id = self.session.query(
            Person.src_id
        ).filter(
            Person.person_id == participant.participantId
        ).first()[0]

        self.assertEqual('test_portal', obs_src_id)
        self.assertEqual('test_portal', prm_src_id)
        self.assertEqual('test_portal', person_src_id)

    def _test_ce_src_id(self):
        # Care Evolution participant origin should be saved as 'ce' in src_id column
        consent_questionnaire = self._create_consent_questionnaire()

        participant = self.data_generator.create_database_participant(participantOrigin='careevolution')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            self.questionnaire
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        obs_src_id = self.session.query(
            Observation.src_id
        ).filter(
            Observation.person_id == participant.participantId
        ).first()[0]

        self.assertEqual('ce', obs_src_id)

    def _setup_pm(self, participant_id: int):
        """ Creates in-person and remote physical measurements for a participant_id"""
        resource = {"entry": [
            {"resource":
                 {"date": datetime.now().isoformat()}
             }
        ]}

        in_person_pm_data = {
            "physicalMeasurementsId": 1,
            "participantId": participant_id,
            "createdSiteId": 1,
            "finalizedSiteId": 2,
            "origin": 'hpro',
            "collectType": PhysicalMeasurementsCollectType.SITE,
            "originMeasurementUnit": OriginMeasurementUnit.UNSET
        }

        record: PhysicalMeasurements = PhysicalMeasurements(**in_person_pm_data)
        self.pm_dao.store_record_fhir_doc(record, resource)
        pm_record: PhysicalMeasurements = self.pm_dao.insert(record)

        meas_in_person_height = RdrMeasurement(
            measurementId=10001,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="height",
            measurementTime=datetime.now(),
            valueDecimal=162.0,
            valueUnit="cm",
        )
        meas_in_person_weight = RdrMeasurement(
            measurementId=10002,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="weight",
            measurementTime=datetime.now(),
            valueDecimal=63.0,
            valueUnit="kg",
        )
        self.session.add(meas_in_person_weight)
        self.session.add(meas_in_person_height)

        remote_pm_data = {
            "physicalMeasurementsId": 2,
            "participantId": participant_id,
            "createdSiteId": 1,
            "finalizedSiteId": 2,
            "origin": 'test-portal',
            "collectType": PhysicalMeasurementsCollectType.SELF_REPORTED,
            "originMeasurementUnit": OriginMeasurementUnit.UNSET
        }

        record: PhysicalMeasurements = PhysicalMeasurements(**remote_pm_data)
        self.pm_dao.store_record_fhir_doc(record, resource)
        pm_record: PhysicalMeasurements = self.pm_dao.insert(record)

        meas_remote_height = RdrMeasurement(
            measurementId=10005,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="height",
            measurementTime=datetime.now(),
            valueDecimal=165.0,
            valueUnit="cm",
        )
        meas_remote_weight = RdrMeasurement(
            measurementId=10006,
            physicalMeasurementsId=pm_record.physicalMeasurementsId,
            codeSystem="http://terminology.pmi-ops.org/CodeSystem/physical-measurements",
            codeValue="weight",
            measurementTime=datetime.now(),
            valueDecimal=62.0,
            valueUnit="kg",
        )

        self.session.add(meas_remote_weight)
        self.session.add(meas_remote_height)
        self.session.commit()

    def test_pm_src_id(self):
        consent_questionnaire = self._create_consent_questionnaire()
        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )

        self._setup_pm(participant.participantId)

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        inperson_src_id = self.session.query(
            Measurement.src_id
        ).filter(
            Measurement.person_id == participant.participantId,
            Measurement.measurement_type_concept_id == 44818701
        ).first()[0]

        remote_src_id = self.session.query(
            Measurement.src_id
        ).filter(
            Measurement.person_id == participant.participantId,
            Measurement.measurement_type_concept_id == 32865
        ).first()[0]

        self.assertEqual('healthpro', inperson_src_id)
        self.assertEqual('test-portal', remote_src_id)

    def test_include_physical_measurements(self):
        consent_questionnaire = self._create_consent_questionnaire()
        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )
        self._setup_questionnaire_response(
            participant,
            consent_questionnaire,
            indexed_answers=[
                (6, 'valueDate', datetime(1982, 1, 9))
                # Assuming the 6th question is the date of birth
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_pm(participant.participantId)
        in_person_height = (44818701, 'height', Decimal('162.000000'), 'cm')
        in_person_weight = (44818701, 'weight', Decimal('63.000000'), 'kg')
        remote_height = (32865, 'height', Decimal('165.000000'), 'cm')
        remote_weight = (32865, 'weight', Decimal('62.000000'), 'kg')

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=False,
            exclude_remote_pm=False
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Check if in-person and remote measurements are in table
        self.assertIn(in_person_height, measurements)
        self.assertIn(in_person_weight, measurements)
        self.assertIn(remote_height, measurements)
        self.assertIn(remote_weight, measurements)

        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=True,
            exclude_remote_pm=False
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Only remote measurements should be present
        self.assertNotIn(in_person_height, measurements)
        self.assertNotIn(in_person_weight, measurements)
        self.assertIn(remote_height, measurements)
        self.assertIn(remote_weight, measurements)

        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            exclude_in_person_pm=False,
            exclude_remote_pm=True
        )

        measurements = self.session.query(Measurement.measurement_type_concept_id,
                                          Measurement.measurement_source_value,
                                          Measurement.value_as_number,
                                          Measurement.unit_source_value).all()

        # Only in-person measurements should be present
        self.assertIn(in_person_height, measurements)
        self.assertIn(in_person_weight, measurements)
        self.assertNotIn(remote_height, measurements)
        self.assertNotIn(remote_weight, measurements)

    def _exists_in_death_table(self, participant_id: int) -> bool:
        return bool(self.session.query(
            Death.person_id
        ).filter(
            Death.person_id == participant_id
        ).distinct().scalar())

    def test_death_table(self):
        # Create API User for DeceasedReport
        api_user = ApiUser(id=1, username='test', system='test')
        self.session.add(api_user)

        pids = list(range(10000, 10010))
        consent_questionnaire = self._create_consent_questionnaire()
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            self._setup_questionnaire_response(
                participant,
                consent_questionnaire,
                indexed_answers=[
                    (6, 'valueDate', datetime(1982, 1, 9))
                    # Assuming the 6th question is the date of birth
                ],
                authored=datetime(2020, 5, 1)
            )

        # Only approved deceased reports should be in Death
        deceased_reports = [
            DeceasedReport(
                participantId=pids[0],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[1],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.PENDING,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[2],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.DENIED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            )
        ]

        for deceased_report in deceased_reports:
            self.session.add(deceased_report)
        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
        )

        self.assertTrue(self._exists_in_death_table(pids[0]))
        self.assertFalse(self._exists_in_death_table(pids[1]))
        self.assertFalse(self._exists_in_death_table(pids[2]))

    def test_death_table_cutoff(self):
        # Create API User for DeceasedReport
        api_user = ApiUser(id=1, username='test', system='test')
        self.session.add(api_user)

        pids = [10000, 10001]
        consent_questionnaire = self._create_consent_questionnaire()
        for pid in pids:
            participant = self.data_generator.create_database_participant(participantId=pid)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            self._setup_questionnaire_response(
                participant,
                consent_questionnaire,
                indexed_answers=[
                    (6, 'valueDate', datetime(1982, 1, 9))
                    # Assuming the 6th question is the date of birth
                ],
                authored=datetime(2020, 5, 1)
            )

        # Approved Deceased report authored after cutoff should not be in Death
        deceased_reports = [
            DeceasedReport(
                participantId=pids[0],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 3, 1)
            ),
            DeceasedReport(
                participantId=pids[1],
                dateOfDeath=date(2023, 2, 1),
                status=DeceasedReportStatus.APPROVED,
                notification=DeceasedNotification.EHR,
                authorId=1,
                authored=datetime(2023, 5, 1)
            )
        ]

        for deceased_report in deceased_reports:
            self.session.add(deceased_report)
        self.session.commit()

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2023-04-01'
        )

        self.assertTrue(self._exists_in_death_table(pids[0]))
        self.assertFalse(self._exists_in_death_table(pids[1]))

    def test_ehr_consent_table(self):
        consent_dao = ConsentDao()
        self.data_generator.create_database_code(value=EHR_CONSENT_QUESTION_CODE)
        ehr_consent = self._create_questionnaire(CONSENT_FOR_ELECTRONIC_HEALTH_RECORDS_MODULE,
                                                        [EHR_CONSENT_QUESTION_CODE])
        ehr_yes = self.data_generator.create_database_code(value="Yes")
        ehr_no = self.data_generator.create_database_code(value="No")

        questionnaire_response_ids = []
        submission_date = datetime(2023, 4, 29, 0, 0)

        for pid in range(10000,10006):
            participant = self.data_generator.create_database_participant(participantId=pid,
                                                                          researchId=pid+100)
            self.data_generator.create_database_participant_summary(
                participant=participant,
                dateOfBirth=datetime(1982, 1, 9),
                consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
            )
            if pid == 10000:
                self._setup_questionnaire_response(
                    participant,
                    ehr_consent,
                    indexed_answers=[
                        (4, 'valueCodeId', ehr_no.codeId)
                    ],
                    authored=submission_date
                )
            else:
                qr = self._setup_questionnaire_response(
                    participant,
                    ehr_consent,
                    indexed_answers=[
                        (4, 'valueCodeId', ehr_yes.codeId)
                    ],
                    authored=submission_date
                )
                questionnaire_response_ids.append(qr.questionnaireResponseId)

        consent_responses = [
            ConsentResponse(
                id=2,
                questionnaire_response_id=questionnaire_response_ids[1], # PID 10002
            ),
            ConsentResponse(
                id=3,
                questionnaire_response_id=questionnaire_response_ids[2], # PID 10003
            ),
            ConsentResponse(
                id=4,
                questionnaire_response_id=questionnaire_response_ids[3], # PID 10004
            ),
            ConsentResponse(
                id=5,
                questionnaire_response_id=questionnaire_response_ids[4], # PID 10005
            )
        ]
        for consent_response in consent_responses:
            self.session.add(consent_response)
        self.session.commit()

        with clock.FakeClock(datetime(2023, 5, 2)):
            consent_dao.insert(ConsentFile(
                    consent_response_id=2, # PID 10002
                    sync_status=2
                ))

        with clock.FakeClock(datetime(2023, 4, 30)):
            consent_files = [
                ConsentFile(
                    consent_response_id=3, # PID 10003
                    sync_status=2
                ),
                ConsentFile(
                    consent_response_id=4, # PID 10004
                    sync_status=1
                ),
            ]
            for consent_file in consent_files:
                consent_dao.insert(consent_file)

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2023-05-01'
        )

        expected_results = [
            (10000, 10100, 'SUBMITTED_NO', submission_date), # Submitted No answer
            (10001, 10101, 'SUBMITTED', submission_date), # Submitted Yes before validation implemented
            (10002, 10102, 'SUBMITTED_NOT_VALIDATED', submission_date), # Submitted Yes, not validated before cutoff date
            (10003, 10103, 'SUBMITTED', submission_date), # Submitted Yes, valid consent
            (10004, 10104, 'SUBMITTED_INVALID', submission_date), # Submitted Yes, invalid consent
            (10005, 10105, 'SUBMITTED_NOT_VALIDATED', submission_date) # Submitted yes, Consent Response exists, Consent file does not, not yet validated
        ]

        consent_table = self.session.query(
            EHRConsentStatus.person_id,
            EHRConsentStatus.research_id,
            EHRConsentStatus.consent_for_electronic_health_records,
            EHRConsentStatus.consent_for_electronic_health_records_authored
        ).all()

        for result in expected_results:
            self.assertIn(result, consent_table)

    def test_wear_consent_table(self):
        wear_yes_code = self.data_generator.create_database_code(value=WEAR_YES_ANSWER_CODE)
        wear_no_code = self.data_generator.create_database_code(value=WEAR_NO_ANSWER_CODE)
        self.data_generator.create_database_code(value=WEAR_CONSENT_QUESTION_CODE)

        participant = self.data_generator.create_database_participant(participantOrigin='test_portal')
        self.data_generator.create_database_participant_summary(
            participant=participant,
            dateOfBirth=datetime(1982, 1, 9),
            consentForStudyEnrollmentFirstYesAuthored=datetime(2000, 1, 10)
        )

        wear_questionnaire = self._create_questionnaire(WEAR_CONSENT_MODULE, [WEAR_CONSENT_QUESTION_CODE])

        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_yes_code.codeId)
            ],
            authored=datetime(2020, 5, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_no_code.codeId)
            ],
            authored=datetime(2020, 6, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_yes_code.codeId)
            ],
            authored=datetime(2020, 7, 1)
        )
        self._setup_questionnaire_response(
            participant,
            wear_questionnaire,
            indexed_answers=[
                (4, 'valueCodeId', wear_no_code.codeId)
            ],
            authored=datetime(2020, 8, 1)
        )

        self.run_cdm_data_generation(
            participant_origin='all',
            cutoff='2020-07-15'
        )

        wear_table = self.session.query(
            WearConsent.person_id,
            WearConsent.authored,
            WearConsent.consent_status
        ).all()

        self.assertIn((participant.participantId, datetime(2020, 5, 1), WEAR_YES_ANSWER_CODE), wear_table)
        self.assertIn((participant.participantId, datetime(2020, 6, 1), WEAR_NO_ANSWER_CODE), wear_table)
        self.assertIn((participant.participantId, datetime(2020, 7, 1), WEAR_YES_ANSWER_CODE), wear_table)
        # Wear consent after cutoff date shouldn't be present.
        self.assertNotIn((participant.participantId, datetime(2020, 8, 1), WEAR_NO_ANSWER_CODE), wear_table)


import subprocess
import os

p = subprocess.run(["ping", "lanacion.com"], capture_output=True, encoding="cp850")
# print(p.stdout)

texto = p.stdout

pos_inicio = texto.find("[") + 1
pos_fin = texto.find("]") - 1
final = pos_inicio

print(texto[pos_inicio:pos_fin])


#!/usr/bin/env python3
"""Political Insults Game - Processor"""
import json
import logging
import re
import time
from typing import List
from datetime import datetime

import requests
from flask import Flask, request

from main import fight
from person import Person

app = Flask(__name__)

# pylint: disable=consider-using-f-string
# pylint: disable=logging-fstring-interpolation

# Opening page to input details.
# List to store received people.
people: List[dict] = []
# List to store fight
fight_log: List[str] = []

# Returns the results

# How to return webpages in delay - read about it.
@app.route('/fight')
def get_fight():
    """Return fight"""
    # Return results.
    printing_fight = json.dumps(fight_log)
    logging.debug(f"Cleaning up {printing_fight} from unneeded things")
    formatted_fight = re.sub(r"[\[\]]", "", printing_fight)
    return json.dumps(formatted_fight)

@app.route('/people', methods=['POST'])
def add_grids():
    """Create objects for fight"""
    # Add person to list.
    people.append(request.get_json())
    logging.debug(f"Received {people}")

    # Put the people into Person objects, calling make_person.
    p_1: Person = Person(gender=people[0]['Gender 1'], pronouns=people[0]['Pronouns 1'],
                         age=people[0]['Age 1'], skin_color=people[0]['Skin Color 1'],
                         political_ideology=people[0]['Political Ideology 1'])
    p_2: Person = Person(gender=people[0]['Gender 2'], pronouns=people[0]['Pronouns 2'],
                         age=people[0]['Age 2'], skin_color=people[0]['Skin Color 2'],
                         political_ideology=people[0]['Political Ideology 2'])
    logging.debug(f"{people} were put into {p_1} and {p_2}")
    # Getting current date and time
    now = datetime.now()
    dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
    start = time.time()
    # Fight
    fight_string: str = \
        fight(one_who_is_making_statement=p_1, one_who_might_get_triggered=p_2, return_string='')
    end = time.time()
    total_time = end - start
    logging.debug(f"fight is {fight_string}")
    # Adding log to database
    total_time_string: str = str("%.2f" % total_time)
    requests.post("http://political_insults_database:3000/database",
                  json={"Time": total_time_string, "Date": dt_string,
                        "Log": fight_string}, timeout=10)
    logging.debug("Added to database")
    # Put fight log inside string and return that.
    fight_log.append(fight_string)
    logging.debug(f"adding {fight_string} to the log")
    return '', 204

if __name__ == "__main__":
    app.run(debug=True, port=8000, host='political_insults_processor')


from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.shortcuts import render, redirect, get_object_or_404
from django.contrib import messages
from django.conf import settings
from django.http import JsonResponse, HttpResponse

from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_POST

import redis

from actions.utils import create_action
from common.decorators import ajax_required
from .forms import ImageCreateForm
from .models import Image

# conn to Redis
rds_db = redis.StrictRedis(host=settings.REDIS_HOST,
                           port=settings.REDIS_PORT,
                           db=settings.REDIS_DB)

@login_required
def image_create(request):
    if request.method == 'POST':
        form = ImageCreateForm(data=request.POST)

        if form.is_valid():
            cleaned = form.cleaned_data

            new_item = form.save(commit=False)
            new_item.user = request.user

            new_item.save()
            create_action(request.user, 'bookmarked image', new_item)

            messages.success(request, 'Image added 😁 !')

            return redirect(new_item.get_absolute_url())

    else:
        form = ImageCreateForm(data=request.GET)

    return render(request,
                  'images/image/create.html', { 'section': 'images',
                                                'form'   : form })

def image_detail(request, id, slug):
    image = get_object_or_404(Image, id=id, slug=slug)

    # the syntax 'xx:xx:xx' ollows the conventions
    #   which is for naming Redis keys (I'll dig more later (maybe))
    total_views = rds_db.incr(f'image:{image.id}:views')

    # increment image ranking by 1
    rds_db.zincrby('image_ranking', image.id, 1)

    return render(request,
                  'images/image/detail.html', { 'section'    : 'images',
                                                'image'      : image,
                                                'total_views': total_views })

@login_required
def image_ranking(request):
    image_ranking = rds_db.zrange('image_ranking', 0, -1, desc=True)[:10]

    image_ranking_ids = [
        int(id) for id in image_ranking
    ]

    most_viewed = list(Image.objects.filter(id__in=image_ranking_ids))
    most_viewed.sort(key=lambda x: image_ranking_ids.index(x.id))

    return render(request,
                  'images/image/ranking.html', { 'section'    : 'images',
                                                 'most_viewed': most_viewed })

@ajax_required
@login_required
@require_POST
def image_like(request):
    """
        Add a link to the 'image detail' page
            to let users click on it to "like" an image :)

        This func is kinda passive (process only)
            it get stuff from the {% block domready %} in <detail.html>

        ----- ----- ----- -----

        This file is TIGHTLY connected with MULTIPLE files !!
            i.e. the 'ajax' js file & the templates (detail.html)

        Lemme explain the code down below

            variables
                image_id    id      pk 'id' auto gen_ed by Django (DB-Side)
                action              add who-likes-the-image

            JsonResponse
                nothing special, just another if-case

        ----- ----- ----- -----

        The newly added `@ajax_required`
            is mainly for "users cannot access specific url directly"

        Test it by 'YOUR_FULL_URL/images/like/'
            it should be a '400 (Bad request)' (it's inside the devtools)
    """

    image_id = request.POST.get('id')
    action = request.POST.get('action')

    if image_id and action:
        try:
            image = Image.objects.get(id=image_id)

            if action == 'like':
                image.users_like_for_img.add(request.user)

                create_action(request.user, 'likes', image)

            else:
                image.users_like_for_img.remove(request.user)

            return JsonResponse({ 'status': 'ok' })

        except Exception:
            pass

    return JsonResponse({ 'status': 'ko' })

@login_required
def image_list(request):
    all_images = Image.objects.all()

    paginator = Paginator(all_images, 8)
    page = request.GET.get('page')

    try:
        all_images = paginator.page(page)
    except PageNotAnInteger:
        all_images = paginator.page(1)
    except EmptyPage:
        if request.is_ajax():
            return HttpResponse('')

    all_images = paginator.page(paginator.num_pages)

    if request.is_ajax():
        return render(request,
                      'images/image/list_ajax.html', { 'section': 'images',
                                                       'images' : all_images })

    return render(request,
                  'images/image/list.html', { 'section': 'images',
                                              'images' : all_images })


#Bonus lesson (3.5) - try&except, string modification

#Normally, if something goes wrong the program crashes. For example if you try to divide by 0
print("Let's divide 2 numbers!")
numerouno=int(input('Type first number (a): '))
numerodos=int(input('Type second number (a): '))
print(numerouno/numerodos)
#as you can see, if you type letters not numbers or 0 as the second number, the program crashes.
#after all it's impossible to do math on text or divide by 0.
#now let's try again, but this time use Try and Except
#let's put all of this in a loop first, because why not:
print('checkpoint')
NewNumeroUno=0
NewNumeroDos=0
while True:
    NewNumeroUno =input('Type first number (b): ') #I can remove conversion to numbers from input
    NewNumeroDos=input('Type second number (b): ')
    try: #try runs first
        print(float(NewNumeroUno)/float(NewNumeroDos)) #convert to numbers and divide
        NewNumeroUno=0 #let's reset the variables
        NewNumeroDos=0
    except: #if something in try fails, stuff in except runs

        if NewNumeroUno.lower()=='end' or NewNumeroDos.lower()=='end':
            print("loop closed")
            break #way for you to close this loop and go further (type end)
        print("Error! Try different numbers")
        NewNumeroUno=0
        NewNumeroDos=0
#now if the calculations fail, the program will display the error message and still work instead of crashing

#String modifications
#Here I will show you how to make a String shorter or get specific part of it
word = str(input("Type text: "))
word = word[1:] #this way you can remove X letters from a word
print(word) #display the word but now without the first letter
slowo = str(input("Type some longer text: "))
digit = int(input("How many first letters remove?: "))
print(slowo[digit:]) #display the word but now without 'digit' amount of letters
palabra = str(input("Type some longer text again: "))
liczba = int(input("Which letter from this word should I display?: "))
print(palabra[liczba]) #display 'liczba' letter from the string.
#remember that in computer science array starts at 0
liczba = int(input("Which letter from this word should I display again?: "))
print(palabra[liczba-1]) #but if we do this, we can type the index number like if the array started at 1


from flask import Flask, request, jsonify
from exception import SignatureException
import file_handler as fh
import os
from dotenv import load_dotenv
import bugsnag
from bugsnag.flask import handle_exceptions

app = Flask(__name__)
app.secret_key = "secret key"
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024  # 16MB max image size

IMG_FILE = 'img'
IMG_FILE64 = 'img64'
IMG_FILE_BYTES = 'img_bytes'
IMG_RESPONSE_ENCODING = 'resp_enc'
IMG_SIZES = 'img_sizes'
ALLOWED_KEYS = (IMG_FILE, IMG_FILE64, IMG_FILE_BYTES)
ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')

load_dotenv()
STAGE = os.environ.get("STAGE", os.environ.get("FLASK_ENV", "development"))
if STAGE != "development" and "BUGSNAG_API_KEY" in os.environ and os.environ.get("BUGSNAG_API_KEY"):
    bugsnag.configure(
        api_key=os.environ.get("BUGSNAG_API_KEY"),
        project_root="",
        app_version="0.0.1",
        release_stage=STAGE,
    )
    handle_exceptions(app)

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def get_value(k):
    if request.form is not None and k in request.form:
        return request.form[k]
    if request.json is not None and k in request.json:
        return request.json[k]
    return None

def is_file():
    return IMG_FILE in request.files

def is_file64():
    return get_value(IMG_FILE64) is not None

def is_file_bytes():
    return get_value(IMG_FILE_BYTES) is not None

def get_response_encoding():
    return get_value(IMG_RESPONSE_ENCODING) or fh.ENCODING_BASE64

def get_img_sizes():
    return get_value(IMG_SIZES) or {"img": (500, 50)}

def create_response(imgs):
    resp = jsonify(imgs)
    resp.status_code = 201
    return resp

def handle_file():
    file = request.files[IMG_FILE]
    if file.filename == '':
        resp = jsonify({'message': 'No file selected for uploading'})
        resp.status_code = 400
        return resp
    if file and allowed_file(file.filename):
        imgs = fh.get_from_file(request.files[IMG_FILE], get_response_encoding(), get_img_sizes())
        return create_response(imgs)
    else:
        resp = jsonify({'message': 'Allowed file types are {0}'.format(ALLOWED_EXTENSIONS)})
        resp.status_code = 400
        return resp

@app.route('/extract-signature', methods=['POST'])
def extract_signature():
    if is_file():
        return handle_file()

    elif is_file64():
        file64 = get_value(IMG_FILE64)
        imgs = fh.get_from_base64(file64, get_response_encoding(), get_img_sizes())
        return create_response(imgs)

    elif is_file_bytes():
        file_bytes = get_value(IMG_FILE_BYTES)
        imgs = fh.get_from_bytes(file_bytes, get_response_encoding(), get_img_sizes())
        return create_response(imgs)

    resp = jsonify({'message': 'No image in the request. Use {} in either form or json request'.format(ALLOWED_KEYS)})
    resp.status_code = 400
    return resp

@app.errorhandler(SignatureException)
def handle_image_exception(error):
    out = error.to_dict(get_response_encoding())
    response = jsonify(out)
    response.status_code = error.status_code
    return response

@app.errorhandler(Exception)
def handle_exception(e):
    bugsnag.notify(e)
    resp = jsonify({"errors": [{"error_code": "unexpected_error", "message": "An error has occurred while processing the image. Please try again in a few moments."}]})
    resp.status_code = 500
    return resp

if __name__ == "__main__":
    app.run(host="0.0.0.0", port="7000")


import os
wbit="1"                        #precision of weight
abit="1"                        #precision of activation
complementary='false'           #Negative activation or not
train_model='vgg_cifar10_train' #vgg_cifar10_train or resnet_train
save_dir="results"              #saved folder of training accuracy
save_model="Linear_test.pkl"    #name of saved model(use .pkl as file extenstion)

cmd = "python3 main_train.py --model {0} --w_bit {1} --a_bit {2} --comp {3} --save {4} --save_model {5}"\
    .format(train_model,wbit,abit,complementary,save_dir,save_model)

os.system(cmd)


from math import dist
def dist(p1, p2):
    return (p1[0] - p2[0])**2 + (p1[1] - p2[1])**2 + (p1[2] - p2[2])**2
def max_radius(points):
    points = sorted(points)

    k = 5
    spheres = []

    for i in range(1, min(k, len(points))):
        spheres.append((dist(points[0], points[i]) / 2,
                        [points[0][0], points[i][0]],
                        [points[0][1], points[i][1]],
                        [points[0][2], points[i][2]]))

    used = [0] * len(points)

    def dist_to_spheres(point, spheres):
        min_dist = float('inf')
        for s in spheres:
            x_dist = min((point[0] - x)**2 for x in s[1])
            y_dist = min((point[1] - y)**2 for y in s[2])
            z_dist = min((point[2] - z)**2 for z in s[3])
            dist = x_dist + y_dist + z_dist
            min_dist = min(min_dist, dist - s[0])
        return min_dist

    for i in range(k, len(points)):
        min_dist = dist_to_spheres(points[i], spheres)

        for s in spheres:
            s[0] = max(s[0], min_dist / 2.0)
            s[1].append(points[i][0])
            s[2].append(points[i][1])
            s[3].append(points[i][2])

        used[i] = 1
    return [s[0] for s in spheres]
print(max_radius([[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]]))


import matplotlib.pyplot as plt
import numpy as np
import os
import seaborn as sbn
import pandas as pd
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from matplotlib import font_manager
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

from mpl_toolkits.mplot3d import Axes3D

font_path = 'C:\Windows\Fonts\STXIHEI.TTF'
# font = FontProperties(fname=r"simsun.ttc", size=18)
font_manager.fontManager.addfont(font_path)
prop = font_manager.FontProperties(fname=font_path)
plt.rcParams['font.family'] = prop.get_name()

# set some initial paths

# path to the directory where this script lives
thisdir = os.path.abspath('')

# path to the scripts directory of the repository
scriptsdir = os.path.split(thisdir)[0]

# path to the main directory of the repository
maindir = os.path.split(scriptsdir)[0]

# path to the results subdirectory
resultsdir = os.path.join(os.path.split(scriptsdir)[0], 'results')

# path to the data subdirectory
datadir = os.path.join(maindir, 'data')

# path to the output_source subsubdirectory
output_source_dir = os.path.join(datadir, 'output_source')

# %% 年龄计数，合成的人口分布
def get_ages(location, country, level, num_agebrackets=18):
    """
    Get the age count for the synthetic population of the location.

    Args:
        location (str)        : name of the location
        country (str)         : name of the country
        level (str)           : name of level (country or subnational)
        num_agebrackets (int) : the number of age brackets

    Returns:
        dict: A dictionary of the age count.
    """

    if country == 'Europe':
        country = location
        level = 'country'
    if level == 'country':
        file_name = country + '_' + level + '_level_age_distribution_' + '%i' % num_agebrackets + '.csv'
    else:
        file_name = country + '_' + level + '_' + location + '_age_distribution_' + '%i' % num_agebrackets + '.csv'
    file_path = os.path.join(datadir, 'origin_resource', 'age_distributions', file_name)
    df = pd.read_csv(file_path, delimiter=',', header=None)
    df.columns = ['age', 'age_count']
    return dict(zip(df.age.values.astype(int), df.age_count.values))

# 刻度标签数组
def get_label(num_agebrackets=18):
    if num_agebrackets == 85:
        age_brackets = [str(i) + '-' + str((i + 1)) for i in range(0, 84)] + ['85+']
    elif num_agebrackets == 18:
        age_brackets = [str(5 * i) + '-' + str(5 * (i + 1) - 1) for i in range(17)] + ['85+']
    return age_brackets

def read_cumulative_incidence(location,p,p_v,l,l_v,k,k_v,num_agebrackets):
    file_name = f"{location}_cumulative_incidence_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.csv"
    file_path = os.path.join(output_source_dir, 'age-structured_output_source', f'{p}_{p_v}', location, file_name)

    M = np.loadtxt(file_path, delimiter=',')
    return M

def read_cumulative_incidence_age(location,p,p_v,l,l_v,k,k_v,num_agebrackets):
    file_name = f"{location}_cumulative_incidence_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.csv"
    file_path = os.path.join(output_source_dir, 'age-structured_output_source', "bracket", p+"_"+str(p_v), location, file_name)

    M = np.loadtxt(file_path, delimiter=',')
    return M

def plot_p_cumulative_incidence(location, p, l, l_v, k, k_v, num_agebrackets, isSave=False):
    pv_arr = [0.1, 0.5, 0.9]
    data1 = read_cumulative_incidence(location, p, pv_arr[0],l,l_v,k,k_v,num_agebrackets).T
    data2 = read_cumulative_incidence(location, p, pv_arr[1],l,l_v,k,k_v,num_agebrackets).T
    data3 = read_cumulative_incidence(location, p, pv_arr[2],l,l_v,k,k_v,num_agebrackets).T

    fig, axes = plt.subplots(nrows=1, ncols=1, sharex=True, sharey=True, figsize=(5, 4))
    plt.subplots_adjust(left=0.15, right=0.9, bottom=0.15, top=0.90, wspace=0.1, hspace=0.1)
    axes.plot(data1[0], data1[1], label="p = " + str(pv_arr[0]))
    axes.plot(data2[0], data2[1], label="p = " + str(pv_arr[1]))
    axes.plot(data3[0], data3[1], label="p = " + str(pv_arr[2]))
    axes.set_xlabel('时间（天）', labelpad=10, fontsize=14)
    axes.set_ylabel('发病率（%）', labelpad=15, fontsize=14)
    plt.ylim(0, 100)
    plt.xlim(0, 700)
    plt.rcParams.update({'font.size': 12})
    axLine, axLabel = axes.get_legend_handles_labels()
    fig.legend(axLine, axLabel, loc='upper center', shadow=False, frameon=False, ncol=3)
    # plt.show()

    if isSave:
        file_name = f'{location}_cumulative_incidence_age_{num_agebrackets}_p_{l}_{l_v}_{k}_{k_v}.pdf'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='pdf')

        file_name = f'{location}_cumulative_incidence_age_{num_agebrackets}_p_{l}_{l_v}_{k}_{k_v}.eps'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='eps')

        file_name = f'{location}_cumulative_incidence_age_{num_agebrackets}_p_{l}_{l_v}_{k}_{k_v}.svg'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='svg')

def plot_cumulative_incidence_age(location, p, l, l_v, k, k_v,num_agebrackets=18, isSave=False):
    data = read_cumulative_incidence_age(location, p, p_v,l,l_v,k,k_v,num_agebrackets).T
    print(data)
    age_brackets = get_label()
    age_brackets = np.array(age_brackets).reshape(3, 6)
    print(age_brackets)
    left = 0.08
    right = 0.97
    top = 0.9
    bottom = 0.15
    fig, axes = plt.subplots(3, 6, figsize=(9, 4))
    fig.subplots_adjust(left=left, right=right, top=top, bottom=bottom, wspace=0.3, hspace=0.8)
    for i in range(3):
        axes[i][0].set_ylabel('发病率（%）', fontsize=14)
        for j in range(6):
            axes[2][j].set_xlabel('时间（天）', fontsize=14)
            axes[i][j].set_title(f'年龄组:{age_brackets[i][j]}', fontsize=12)
            axes[i][j].xaxis.set_major_locator(plt.MaxNLocator(3))
            axes[i][j].yaxis.set_major_locator(plt.MaxNLocator(2))
            axes[i][j].set_ylim((0, 50))
            axes[i][j].tick_params(labelsize=10)
            axes[i][j].plot(data[0][0:600], data[(i+1)*(j+1)][0:600])
            axes[i][j].axhline(25, linewidth=0.5, linestyle='--', color='r')

    plt.show()

    if isSave:
        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.pdf'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='pdf')

        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.eps'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='eps')

        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.svg'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='svg')

def plot_cumulative_incidence_l_k(location, p, p_v, l, l_v, k, k_v,num_agebrackets=18, isSave=False):
    data = []
    for l_v in range(11):
        for k_v in range(11):
            val = read_cumulative_incidence(location, p, p_v, l, l_v/10, k, k_v/10, num_agebrackets).T[1][-1]
            data.append(val)
    data = np.array(data).reshape(11, 11)

    print(data.round(1))
    left = 0.08
    right = 0.97
    top = 0.9
    bottom = 0.15

    fig = plt.figure(figsize=(9, 4))
    ax = fig.gca(projection='3d')  # 三维坐标轴

    # plt.show()

    if isSave:
        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.pdf'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='pdf')

        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.eps'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='eps')

        file_name = f'{location}_cumulative_incidence_all_age_{num_agebrackets}_{p}={p_v:.2f}_{l}={l_v:.2f}_{k}={k_v:.2f}.svg'
        fig_path = os.path.join(resultsdir, 'age-structured_result', file_name)
        fig.savefig(fig_path, format='svg')

location = 'Shanghai'
country = 'China'
level = 'subnational'
num_agebrackets = 18
p = 'p'
w = 'w'
# beta = 'beta'
q = 'q'
l = 'l'
# param = 'sigma_inverse'
k = 'k'
# param = 'gamma_inverse'
state = 'all_states'

# Dou-SEAIQ model parameters
p_v = 0.1  # 0.1 0.5  0.9   # percent of unvaccinated
w_v = 0.5  # 感染率降低系数
beta_v = 0.3  # 感染率
q_v = 0.5  # 有症状的概率
l_v = 0.8  # 0-1 接种导致有症状概率降低系数
sigma_inverse = 5.2  # mean latent period
k_v = 0.5  # 0-1 无症状被发现的概率降低系数
gamma_inverse = 1.6  # mean quarantined period

ages = get_ages(location, country, level, num_agebrackets)
total_population = sum(ages.values())

# 累计发病率
plot_p_cumulative_incidence(location, p, l, l_v, k, k_v,num_agebrackets, isSave=True)
# plot_cumulative_incidence_age(location, p, l, l_v, k, k_v,num_agebrackets, isSave=False)

# plot_cumulative_incidence_l_k(location, p, p_v, l, l_v, k, k_v, num_agebrackets=18, isSave=False)


import pickle

def fast_response(text):
    """load a response from the saved trained model"""
    vectorizer = pickle.load(open('../models/vectorizerRF.sav', 'rb'))
    classifier = pickle.load(open('../models/classifierRF.sav', 'rb'))
    if text:
        text_vector = vectorizer.transform([text])
        result = classifier.predict(text_vector)
        new_data = result[0]
        print('new_data', new_data)
    #     with open('../document.csv', 'a') as new_data_document:
    #         new_data_document.write(new_data)
    #     return (
    #         make_response(
    #             jsonify({'sentiment': result[0], 'text': text, 'status_code': 200}), 200)
    #     )
    # return make_response(jsonify({'error': 'sorry! unable to parse', 'status_code': 500}), 500)

if __name__ == '__main__':
    fast_response("bad")


import datetime
import webbrowser

import speech_recognition as sr
import pyttsx3
import wikipedia
import os
import random
import smtplib

engine=pyttsx3.init('sapi5')
voices=engine.getProperty('voices')
engine.setProperty('voice', voices[0].id)

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

""" Wish you according to the time """

def wishme():
    hour=int(datetime.datetime.now().hour)
    if(hour>=1 and hour<12):
        speak("Good Morning Sir!")

    elif(hour>=12 and hour<18):
        speak("Good Afternoon Sir!")

    else:
        speak("Good Evening Sir!")

    speak("I'm your jarvis . How may I help you !")

"""Take the command from the user and return it as string """

def takeCommand():

    r=sr.Recognizer()
    with sr.Microphone() as source:
        print("Listening .....")
        r.pause_threshold = 1
        r.energy_threshold = 200
        audio = r.listen(source)

    try:
        print("Recognizing .... ")
        query = r.recognize_google(audio, language='en-in')
        print(f"User said : {query}/n")

    except Exception as e:
        print("Please say that again ... ")
        return "None"
    return query
def sendMail(to,content):
    server=smtplib.SMTP('smtp.gmail.com',587)
    server.ehlo()
    server.starttls()
    server.login("youremail","yourmail password")
    server.sendmail("your mail",to,content)
    server.close()

if __name__ == '__main__':
    wishme()

    while True:
        query = takeCommand().lower()
        if 'wikipedia' in query:
            speak("Searching wikipedia....")
            query=query.replace("wikipedia","")
            results=wikipedia.summary(query,sentence=2)
            speak("According to wikipedia")
            print(results)
            speak(results)

        elif 'open youtube' in query:
            webbrowser.open("youtube.com")

        elif 'open google' in query:
            webbrowser.open("google.com")

        elif 'open stackflow' in query:
            webbrowser.open("stackflow.com")

        elif 'play song' in query:
            music_dir='G:\\2022 Songs'
            songs=os.listdir(music_dir)
            print(songs)
            randomno=random.randint(0,len(songs))
            os.startfile(os.path.join(music_dir,songs[randomno]))

        elif 'time' in query:
            strTime=datetime.datetime.now().strftime("%H:%M:%S")
            speak(f"Sir ,the time is {strTime}")

        elif 'open code' in query:
            codepath="C:\\Users\\sahil\\AppData\\Local\\Programs\\Microsoft VS Code\\Code.exe"
            os.startfile(codepath)

        elif 'send mail' in query:
            try:
                speak("What should I send ?")
                content=takeCommand()
                to="sahilbeny24@gmail.com"
                sendEmail(to,content)
                speak("Email sent scuccessfully.")
            except Exception as e:
                speak("Sorry my friend I was unable to send mail.")

        else:
            print("Sorry I can't Help with that.")

        if "quit" in query or "stop" in query or "exit" in query:
            speak("Thanks for using Jarvis . See you again")
            print("MADE BY --- SAHIL BENIWAL ")
            print(".....Thanks a lot .....")
            exit()


import json
import boto3
from decimal import Decimal

def lambda_handler(event, context):
    candidateId = event['query']['candidateId']
    candidateResult = getTableData('CandidateTestResults', {'candidateId':candidateId})
    candidateData = getTableData('CandidateRegistrationDetails', {'id': candidateId})
    candidateResult['name'] = candidateData['name']
    candidateResult['email'] = candidateData['email']
    return {
        'statusCode': 200,
        'headers': {
            'Access-Control-Allow-*': '*'

        },
        'body': json.dumps(candidateResult, cls=DecimalEncoder)
    }

def getDynamoDbResource():
    return boto3.resource('dynamodb')

def getItem(table_name, item):
    dynamodb = getDynamoDbResource()
    table = dynamodb.Table(table_name)
    return table.get_item(Key=item)

def getTableData(tableName, item):
    dynamodbObj = getItem(tableName, item)
    return dynamodbObj['Item']

class DecimalEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return str(obj)
        return json.JSONEncoder.default(self, obj)