import logging
from time import sleep

from langchain.llms import OpenAI
from scrapy import Request, Spider
from selenium import webdriver
from selenium.webdriver.common.keys import Keys

from conf import (
    CONNECTION_REQUEST_LLM_PROMPT,
    DEFAULT_CONNECTION_MESSAGE,
    MAX_PROFILES_TO_CONNECT,
    MAX_PROFILES_TO_SCRAPE,
    OPENAI_API_KEY,
    ROLES_KEYWORDS,
    SELECTIVE_SCRAPING,
    SEND_CONNECTION_REQUESTS,
)
from linkedin.integrations.linkedin_api import extract_profile_from_url
from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none
from linkedin.items import LinkedinUser
from linkedin.middlewares.selenium import SeleniumSpiderMixin

logger = logging.getLogger(__name__)

SLEEP_TIME_BETWEEN_CLICKS = 1.5

roles_keywords_lowercase = [role.lower() for role in ROLES_KEYWORDS]


def remove_non_bmp_characters(text):
    return "".join(c for c in text if 0x0000 <= ord(c) <= 0xFFFF)


def remove_primary_language(text):
    lines = text.split("\n")
    filtered_lines = [line for line in lines if "primary language" not in line.lower()]
    return "\n".join(filtered_lines)


def is_your_network_is_growing_present(driver):
    got_it_button = get_by_xpath_or_none(
        driver,
        '//button[@aria-label="Got it"]',
        wait_timeout=0.5,
    )
    return got_it_button is not None


def is_email_verifier_present(driver):
    email_verifier = get_by_xpath_or_none(
        driver,
        "//label[@for='email']",
        wait_timeout=0.5,
    )
    return email_verifier is not None


def send_connection_request(driver, message):
    sleep(SLEEP_TIME_BETWEEN_CLICKS)

    # Click the "Add a note" button
    add_note_button = get_by_xpath_or_none(
        driver,
        "//button[contains(@aria-label, 'note')]",
    )
    click(driver, add_note_button) if add_note_button else logger.warning(
        "Add note button unreachable"
    )
    sleep(SLEEP_TIME_BETWEEN_CLICKS)

    # Write the message in the textarea
    message_textarea = get_by_xpath_or_none(
        driver,
        "//textarea[@name='message' and @id='custom-message']",
    )
    message_textarea.send_keys(message[:300]) if message_textarea else logger.warning(
        "Textarea unreachable"
    )
    sleep(SLEEP_TIME_BETWEEN_CLICKS)

    # Click the "Send" button
    send_button = get_by_xpath_or_none(
        driver,
        "//button[@aria-label='Send now']",
    )
    click(driver, send_button) if send_button else logger.warning(
        "Send button unreachable"
    )
    sleep(SLEEP_TIME_BETWEEN_CLICKS)
    return True


def skip_connection_request(connect_button):
    return not (connect_button and SEND_CONNECTION_REQUESTS)


def contains_keywords(user_profile):
    headline = user_profile["headline"].lower()
    return any(role in headline for role in roles_keywords_lowercase)


def skip_profile(user_profile):
    return SELECTIVE_SCRAPING and not contains_keywords(user_profile)


def generate_connection_message(llm: OpenAI, user_profile):
    from langchain import PromptTemplate

    prompt_template = PromptTemplate.from_template(CONNECTION_REQUEST_LLM_PROMPT)

    prompt = prompt_template.format(profile=user_profile)
    logger.debug(f"Generate message with prompt:\n{prompt}:")
    msg = llm.predict(prompt).strip()
    msg = remove_primary_language(msg).strip()
    msg = remove_non_bmp_characters(msg).strip()
    logger.info(f"Generated Icebreaker:\n{msg}")
    return msg


def extract_connect_button(user_container):
    connect_button = get_by_xpath_or_none(
        user_container,
        ".//button[contains(@aria-label, 'connect')]/span",
        wait_timeout=5,
    )
    return (
        connect_button if connect_button else logger.debug("Connect button not found")
    )


def increment_index_at_end_url(response):
    # incrementing the index at the end of the url
    url = response.request.url
    next_url_split = url.split("=")
    index = int(next_url_split[-1])
    next_url = "=".join(next_url_split[:-1]) + "=" + str(index + 1)
    return index, next_url


def extract_user_url(user_container):
    # Use this XPath to select the <a> element
    link_elem = get_by_xpath_or_none(
        user_container,
        ".//a[contains(@class, 'app-aware-link') and contains(@href, '/in/')]",
    )

    if not link_elem:
        logger.warning("Can't extract user URL")
        return None

    user_url = link_elem.get_attribute("href")
    logger.debug(f"Extracted user URL: {user_url}")
    return user_url


def click(driver, element):
    driver.execute_script("arguments[0].scrollIntoView();", element)
    driver.execute_script("arguments[0].click();", element)


def press_exit(driver):
    webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()


class SearchSpider(Spider, SeleniumSpiderMixin):
    """
    Abstract class for generic search on linkedin.
    """

    allowed_domains = ("linkedin.com",)

    def __init__(self, start_url, driver=None, name=None, *args, **kwargs):
        super().__init__(name=name, *args, **kwargs)
        self.start_url = start_url
        self.driver = driver or build_driver()
        self.user_profile = None
        self.profile_counter = 0
        self.connections_sent_counter = 0
        self.llm = (
            OpenAI(
                max_tokens=90,
                model_name="text-davinci-003",
                openai_api_key=OPENAI_API_KEY,
            )
            if SEND_CONNECTION_REQUESTS
            else None
        )

    def wait_page_completion(self, driver):
        """
        Abstract function, used to customize how the specific spider must wait for a search page completion.
        """
        get_by_xpath_or_none(driver, "//*[@id='global-nav']/div", wait_timeout=5)

    def parse_search_list(self, response):
        continue_scrape = True
        driver = self.get_driver_from_response(response)
        if self.check_if_no_results_found(driver):
            logger.warning("No results found. Stopping crawl.")
            return

        for user_container in self.iterate_containers(driver):
            if is_your_network_is_growing_present(driver):
                press_exit(driver)
            user_profile_url = extract_user_url(user_container)
            if user_profile_url is None:
                continue
            logger.debug(f"Found user URL:{user_profile_url}")
            self.user_profile = extract_profile_from_url(
                user_profile_url, driver.get_cookies()
            )
            if self.should_stop(response):
                continue_scrape = False
                break

            connect_button = extract_connect_button(user_container)
            if skip_profile(self.user_profile):
                logger.info(f"Skipped profile: {user_profile_url}")
            else:
                message = (
                    generate_connection_message(self.llm, self.user_profile)
                    if OPENAI_API_KEY
                    else DEFAULT_CONNECTION_MESSAGE
                )
                self.user_profile["connection_msg"] = (
                    message if OPENAI_API_KEY else None
                )
                if skip_connection_request(connect_button):
                    logger.info(f"Skipped connection request: {user_profile_url}")
                else:
                    click(driver, connect_button)
                    if is_email_verifier_present(driver):
                        press_exit(driver)
                    else:
                        conn_sent = send_connection_request(driver, message=message)
                        logger.info(
                            f"Connection request sent to {user_profile_url}\n{message}"
                        ) if conn_sent else None
                        self.connections_sent_counter += 1

                yield LinkedinUser(linkedinUrl=user_profile_url, **self.user_profile)
                self.profile_counter += 1

        if continue_scrape:
            next_url = self.get_next_url(response)
            yield self.create_next_request(next_url, response)

    def get_driver_from_response(self, response):
        return response.meta.pop("driver")

    def check_if_no_results_found(self, driver):
        no_result_found_xpath = (
            "//div[contains(@class, 'search-reusable-search-no-results')]"
        )
        return (
            get_by_xpath_or_none(
                driver=driver, xpath=no_result_found_xpath, wait_timeout=3
            )
            is not None
        )

    def get_next_url(self, response):
        index, next_url = increment_index_at_end_url(response)
        return next_url

    def create_next_request(self, next_url, response):
        return Request(
            url=next_url,
            priority=-1,
            callback=self.parse_search_list,
            meta=response.meta,
        )

    def iterate_containers(self, driver):
        for i in range(1, 11):
            container_xpath = f"//li[contains(@class, 'result-container')][{i}]"
            container_elem = get_by_xpath_or_none(
                driver, container_xpath, wait_timeout=2
            )
            if container_elem:
                logger.debug(f"Loading {i}th user")
                driver.execute_script("arguments[0].scrollIntoView();", container_elem)
                self.sleep()
                yield container_elem

    def should_stop(self, response):
        max_num_profiles = self.profile_counter >= MAX_PROFILES_TO_SCRAPE
        if max_num_profiles:
            logger.info(
                "Stopping Reached maximum number of profiles to scrape. Stopping crawl."
            )

        max_num_connections = self.connections_sent_counter >= MAX_PROFILES_TO_CONNECT
        if max_num_connections:
            logger.info(
                "Stopping Reached maximum number of profiles to connect. Stopping crawl."
            )

        return max_num_profiles

langchain.llms.OpenAI langchain.PromptTemplate.from_template
[((689, 716), 'logging.getLogger', 'logging.getLogger', (['__name__'], {}), '(__name__)\n', (706, 716), False, 'import logging\n'), ((1186, 1271), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//button[@aria-label="Got it"]"""'], {'wait_timeout': '(0.5)'}), '(driver, \'//button[@aria-label="Got it"]\', wait_timeout=0.5\n    )\n', (1206, 1271), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((1397, 1468), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//label[@for=\'email\']"""'], {'wait_timeout': '(0.5)'}), '(driver, "//label[@for=\'email\']", wait_timeout=0.5)\n', (1417, 1468), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((1590, 1622), 'time.sleep', 'sleep', (['SLEEP_TIME_BETWEEN_CLICKS'], {}), '(SLEEP_TIME_BETWEEN_CLICKS)\n', (1595, 1622), False, 'from time import sleep\n'), ((1682, 1753), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//button[contains(@aria-label, \'note\')]"""'], {}), '(driver, "//button[contains(@aria-label, \'note\')]")\n', (1702, 1753), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((1900, 1932), 'time.sleep', 'sleep', (['SLEEP_TIME_BETWEEN_CLICKS'], {}), '(SLEEP_TIME_BETWEEN_CLICKS)\n', (1905, 1932), False, 'from time import sleep\n'), ((1997, 2085), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//textarea[@name=\'message\' and @id=\'custom-message\']"""'], {}), '(driver,\n    "//textarea[@name=\'message\' and @id=\'custom-message\']")\n', (2017, 2085), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((2233, 2265), 'time.sleep', 'sleep', (['SLEEP_TIME_BETWEEN_CLICKS'], {}), '(SLEEP_TIME_BETWEEN_CLICKS)\n', (2238, 2265), False, 'from time import sleep\n'), ((2315, 2379), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//button[@aria-label=\'Send now\']"""'], {}), '(driver, "//button[@aria-label=\'Send now\']")\n', (2335, 2379), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((2514, 2546), 'time.sleep', 'sleep', (['SLEEP_TIME_BETWEEN_CLICKS'], {}), '(SLEEP_TIME_BETWEEN_CLICKS)\n', (2519, 2546), False, 'from time import sleep\n'), ((3058, 3117), 'langchain.PromptTemplate.from_template', 'PromptTemplate.from_template', (['CONNECTION_REQUEST_LLM_PROMPT'], {}), '(CONNECTION_REQUEST_LLM_PROMPT)\n', (3086, 3117), False, 'from langchain import PromptTemplate\n'), ((3504, 3612), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['user_container', '""".//button[contains(@aria-label, \'connect\')]/span"""'], {'wait_timeout': '(5)'}), '(user_container,\n    ".//button[contains(@aria-label, \'connect\')]/span", wait_timeout=5)\n', (3524, 3612), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((4142, 4254), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['user_container', '""".//a[contains(@class, \'app-aware-link\') and contains(@href, \'/in/\')]"""'], {}), '(user_container,\n    ".//a[contains(@class, \'app-aware-link\') and contains(@href, \'/in/\')]")\n', (4162, 4254), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((5656, 5729), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', '"""//*[@id=\'global-nav\']/div"""'], {'wait_timeout': '(5)'}), '(driver, "//*[@id=\'global-nav\']/div", wait_timeout=5)\n', (5676, 5729), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((8691, 8783), 'scrapy.Request', 'Request', ([], {'url': 'next_url', 'priority': '(-1)', 'callback': 'self.parse_search_list', 'meta': 'response.meta'}), '(url=next_url, priority=-1, callback=self.parse_search_list, meta=\n    response.meta)\n', (8698, 8783), False, 'from scrapy import Request, Spider\n'), ((5095, 5109), 'linkedin.integrations.selenium.build_driver', 'build_driver', ([], {}), '()\n', (5107, 5109), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((5251, 5339), 'langchain.llms.OpenAI', 'OpenAI', ([], {'max_tokens': '(90)', 'model_name': '"""text-davinci-003"""', 'openai_api_key': 'OPENAI_API_KEY'}), "(max_tokens=90, model_name='text-davinci-003', openai_api_key=\n    OPENAI_API_KEY)\n", (5257, 5339), False, 'from langchain.llms import OpenAI\n'), ((8349, 8434), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', ([], {'driver': 'driver', 'xpath': 'no_result_found_xpath', 'wait_timeout': '(3)'}), '(driver=driver, xpath=no_result_found_xpath, wait_timeout=3\n    )\n', (8369, 8434), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((9022, 9083), 'linkedin.integrations.selenium.get_by_xpath_or_none', 'get_by_xpath_or_none', (['driver', 'container_xpath'], {'wait_timeout': '(2)'}), '(driver, container_xpath, wait_timeout=2)\n', (9042, 9083), False, 'from linkedin.integrations.selenium import build_driver, get_by_xpath_or_none\n'), ((4675, 4705), 'selenium.webdriver.ActionChains', 'webdriver.ActionChains', (['driver'], {}), '(driver)\n', (4697, 4705), False, 'from selenium import webdriver\n'), ((7808, 7871), 'linkedin.items.LinkedinUser', 'LinkedinUser', ([], {'linkedinUrl': 'user_profile_url'}), '(linkedinUrl=user_profile_url, **self.user_profile)\n', (7820, 7871), False, 'from linkedin.items import LinkedinUser\n')]