"""
A cluster with N nodes attached to a LAN. With a NFS server and dataset long-term persistancy.

Hardware: c6420
Image: UBUNTU18-64-STD
"""

import geni.portal as portal
import geni.rspec.pg as rspec

# Only Ubuntu images supported.
imageList = [
    ('urn:publicid:IDN+emulab.net+image+emulab-ops//UBUNTU18-64-STD', 'UBUNTU 18.04'),
    ('urn:publicid:IDN+wisc.cloudlab.us+image+gaia-PG0:gpu-u1804.node-1', 'UBUNTU 18.04 CUDA 10'),
]

pc = portal.Context()
pc.defineParameter("num_nodes", "Number of nodes", portal.ParameterType.INTEGER, 5, [])
pc.defineParameter("user_name", "user name to run additional setup script",
                   portal.ParameterType.STRING, "peifeng")
pc.defineParameter("setup", "additional setup script",
                   portal.ParameterType.STRING, "setup.sh")
pc.defineParameter("os_image", "Select OS image",
                   portal.ParameterType.IMAGE,
                   imageList[0], imageList, advanced=True)
pc.defineParameter("nfs_hw", "NFS hardware to use", portal.ParameterType.NODETYPE, "c8220", advanced=True)
pc.defineParameter("node_hw", "Node hardware to use", portal.ParameterType.NODETYPE, "c6420", advanced=True)
#pc.defineParameter("dataset", "Dataset backing the NFS storage",
                   #portal.ParameterType.STRING,
                   #"urn:publicid:IDN+clemson.cloudlab.us:gaia-pg0+ltdataset+automl", advanced=True)
params = pc.bindParameters()

# Do not change these unless you change the setup scripts too.
nfsServerName = "nfs"
nfsLanName = "nfsLan"
nfsDirectory = "/nfs"

request = pc.makeRequestRSpec()

# add lan
lan = request.LAN(nfsLanName)
lan.best_effort = True
lan.vlan_tagging = True
lan.link_multiplexing = True

# nfs server with special block storage server
nfsServer = request.RawPC(nfsServerName)
nfsServer.disk_image = params.os_image
nfsServer.hardware_type = params.nfs_hw
lan.addInterface(nfsServer.addInterface())
nfsServer.addService(rspec.Execute(shell="bash", command="/local/repository/nfs-server.sh"))

# Special node that represents the ISCSI device where the dataset resides
dsnode = request.RemoteBlockstore("dsnode", nfsDirectory)
dsnode.dataset = params.dataset
dslink = request.Link("dslink")
dslink.addInterface(dsnode.interface)
dslink.addInterface(nfsServer.addInterface())
# Special attributes for this link that we must use.
dslink.best_effort = True
dslink.vlan_tagging = True
dslink.link_multiplexing = True

# normal nodes
for i in range(params.num_nodes):
    node = request.RawPC("node-{}".format(i + 1))
    node.disk_image = params.os_image
    node.hardware_type = params.node_hw
    bs = node.Blockstore("bs-{}".format(i + 1), "/opt")
    bs.size = "100GB"
    lan.addInterface(node.addInterface("if1"))
    node.addService(rspec.Execute(shell="bash", command="/local/repository/nfs-client.sh"))
    if len(params.setup) > 0:
        node.addService(
            rspec.Execute(
                shell="bash",
                command="/local/repository/{} {}".format(params.setup, params.user_name)
            )
        )

pc.printRequestRSpec(request)


"""
Simple search engine with a data cleaner functionality
"""
import os

# set base ref path and load env
BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
CORPUS_DIR = f'{BASE_DIR}/corpus'
INDEX_DIR = f'{BASE_DIR}/index'

STYLE_UNDERLINE = '\033[4m'
STYLE_BOLD = '\033[1m'
STYLE_HEADER = '\033[95m'
STYLE_BLUE = '\033[94m'
STYLE_CYAN = '\033[96m'
STYLE_GREEN = '\033[92m'
STYLE_WARNING = '\033[93m'
STYLE_FAIL = '\033[91m'
STYLE_CLEAR = '\033[0m'

__author__ = "Owusu K"
__version__ = "0.1.0"


"""SE(3)-equivariant convolution between two single-type features."""

import logging

import torch
from torch import nn

from tfold_se3.utils import get_tensor_size
from tfold_se3.utils import get_peak_memory
from tfold_se3.modules.se3_trans.radial_func import RadialFunc

class PairwiseConv(nn.Module):
    """SE(3)-equivariant convolution between two single-type features."""

    def __init__(self, d_in, n_chns_in, d_out, n_chns_out, n_dims_edge=0):
        """SE(3)-equivariant convolution between a pair of feature types.

        Args:
        * d_in: input features' degree
        * n_chns_in: number of channels of input features
        * d_out: output features' degree
        * n_chns_out: number of channels of output features
        * n_dims_edge: number of dimensions for edge embeddings

        Returns: n/a
        """

        # initialization
        super().__init__()
        self.d_in = d_in
        self.n_chns_in = n_chns_in
        self.d_out = d_out
        self.n_chns_out = n_chns_out
        self.n_dims_edge = n_dims_edge
        self.n_freqs = 2 * min(d_in, d_out) + 1

        # NN-parameterized radial function
        self.radial_fn = RadialFunc(self.n_freqs, self.n_chns_in, self.n_chns_out, self.n_dims_edge)

    def forward(self, inputs, basis_dict):
        """Perform the forward pass.

        Args:
        * inputs: input radial features of size BS x (D_e + 1)
        * basis_dict: dict of equivariant bases, indexed by (d_i, d_o)

        Returns:
        * outputs: output features of size BS x (C_o * (2 * d_o + 1)) x (C_i * (2 * d_i + 1))
        """

        # define abbreviations
        di = self.d_in
        do = self.d_out
        ci = self.n_chns_in
        co = self.n_chns_out
        di_e = 2 * di + 1
        do_e = 2 * do + 1
        nf = self.n_freqs  # which equals to 2 * min(di, do) + 1

        # perform the forward pass
        radial = self.radial_fn(inputs).view(-1, co * ci, nf)
        basis = basis_dict[(self.d_in, self.d_out)].view(-1, do_e * di_e, nf).permute(0, 2, 1)
        try:
            outputs = torch.bmm(radial, basis).view(-1, co, ci, do_e, di_e).permute(
                0, 1, 3, 2, 4).contiguous().view(-1, co * do_e, ci * di_e)
        except:
            logging.error('radial: %s / %.2f MB', str(radial.shape), get_tensor_size(radial))
            logging.error('basis: %s / %.2f MB', str(basis.shape), get_tensor_size(basis))
            logging.error('GPU memory (peak): %.2f MB', get_peak_memory())
            raise

        return outputs

    def __repr__(self):
        """Get the string representation."""

        repr_str = 'PairwiseConv: ' + ', '.join([
            'd_in=%d' % self.d_in,
            'n_chns_in=%d' % self.n_chns_in,
            'd_out=%d' % self.d_out,
            'n_chns_out=%d' % self.n_chns_out,
            'n_dims_edge=%d' % self.n_dims_edge,
            'n_freqs=%d' % self.n_freqs,
        ])

        return repr_str


import os
import stat
import sys
import tempfile

import pytest
import requests
from git import Repo
from mockito import verify, patch, spy2, mock, ANY, when

import opengrok_tools.mirror
from opengrok_tools.scm import Repository
from opengrok_tools.scm.git import GitRepository
from opengrok_tools.scm.repofactory import get_repository
from opengrok_tools.scm.repository import RepositoryException
from opengrok_tools.utils.command import Command
from opengrok_tools.utils.exitvals import (
    CONTINUE_EXITVAL, FAILURE_EXITVAL
)
from opengrok_tools.utils.mirror import check_project_configuration, \
    check_configuration, mirror_project, run_command, get_repos_for_project, \
    HOOKS_PROPERTY, PROXY_PROPERTY, IGNORED_REPOS_PROPERTY, \
    PROJECTS_PROPERTY, DISABLED_CMD_PROPERTY, DISABLED_PROPERTY, \
    CMD_TIMEOUT_PROPERTY, HOOK_TIMEOUT_PROPERTY, DISABLED_REASON_PROPERTY
from opengrok_tools.utils.patterns import COMMAND_PROPERTY, PROJECT_SUBST

def test_empty_project_configuration():
    config = {}
    assert check_project_configuration(config)

def test_none_project_configuration():
    assert check_project_configuration(None)

def test_invalid_project_config_proxy():
    config = {"foo": {PROXY_PROPERTY: True}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_option():
    config = {"foo": {"totally_unknown_option": "value"}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_regex():
    config = {"[]": {PROXY_PROPERTY: True}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_hookdir():
    config = {"foo": {HOOKS_PROPERTY: {"pre": "value"}}}
    assert not check_project_configuration(config, hookdir="/nonexistentdir")

def test_invalid_project_config_ignoredrepos():
    config = {"foo": {IGNORED_REPOS_PROPERTY: {"foo": "bar"}}}
    assert not check_project_configuration(config)

def test_invalid_project_config_hooks():
    with tempfile.TemporaryDirectory() as tmpdir:
        config = {"foo": {HOOKS_PROPERTY: {"pre": "nonexistentfile.sh"}}}
        assert not check_project_configuration(config, hookdir=tmpdir)

def test_invalid_project_config_hooknames():
    with tempfile.TemporaryDirectory() as tmpdir:
        config = {"foo": {HOOKS_PROPERTY: {"blah": "value"}}}
        assert not check_project_configuration(config, hookdir=tmpdir)

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_invalid_project_config_nonexec_hook():
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, "foo.sh"), 'w+') as tmpfile:
            tmpfile.write("foo\n")
            config = {"foo": {HOOKS_PROPERTY: {"pre": "foo.sh"}}}
            assert not check_project_configuration(config, hookdir=tmpdir)

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_valid_project_config_hook():
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, "foo.sh"), 'w+') as tmpfile:
            tmpfile.write("foo\n")
            st = os.stat(tmpfile.name)
            os.chmod(tmpfile.name, st.st_mode | stat.S_IEXEC)
            config = {"foo": {HOOKS_PROPERTY: {"pre": "foo.sh"}}}
            assert check_project_configuration(config, hookdir=tmpdir)

def test_invalid_config_option():
    assert not check_configuration({"nonexistent": True})

def test_valid_config():
    assert check_configuration({
        PROJECTS_PROPERTY: {"foo": {PROXY_PROPERTY: True}},
        PROXY_PROPERTY: "proxy"
    })

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_incoming_retval(monkeypatch):
    """
    Test that the special CONTINUE_EXITVAL value bubbles all the way up to
    the mirror.py return value.
    """

    class MockResponse:

        # mock json() method always returns a specific testing dictionary
        @staticmethod
        def json():
            return "true"

        @staticmethod
        def raise_for_status():
            pass

    with tempfile.TemporaryDirectory() as source_root:
        repo_name = "parent_repo"
        repo_path = os.path.join(source_root, repo_name)
        cloned_repo_name = "cloned_repo"
        cloned_repo_path = os.path.join(source_root, cloned_repo_name)
        project_name = "foo"  # does not matter for this test

        os.mkdir(repo_path)

        def mock_get_repos(*args, **kwargs):
            return [get_repository(cloned_repo_path,
                                   "git", project_name)]

        def mock_get(*args, **kwargs):
            return MockResponse()

        # Clone a Git repository so that it can pull.
        repo = Repo.init(repo_path)
        with repo.config_writer() as git_config:
            git_config.set_value('user', 'email', 'someone@example.com')
            git_config.set_value('user', 'name', 'John Doe')

        new_file_path = os.path.join(repo_path, 'foo')
        with open(new_file_path, 'w'):
            pass
        assert os.path.isfile(new_file_path)
        index = repo.index
        index.add([new_file_path])
        index.commit("add file")
        repo.clone(cloned_repo_path)

        with monkeypatch.context() as m:
            m.setattr(sys, 'argv', ['prog', "-I", project_name])

            # With mocking done via pytest it is necessary to patch
            # the call-site rather than the absolute object path.
            m.setattr("opengrok_tools.mirror.get_config_value",
                      lambda x, y, z: source_root)
            m.setattr("opengrok_tools.utils.mirror.get_repos_for_project",
                      mock_get_repos)
            m.setattr("opengrok_tools.utils.mirror.do_api_call", mock_get)

            assert opengrok_tools.mirror.main() == CONTINUE_EXITVAL

def test_empty_project_config():
    assert check_project_configuration({'foo': None})

def test_disabled_command_api():
    """
    Test that mirror_project() calls call_rest_api() if API
    call is specified in the configuration for disabled project.
    """
    with patch(opengrok_tools.utils.mirror.call_rest_api,
               lambda a, b, c: mock(spec=requests.Response)):
        project_name = "foo"
        config = {
            DISABLED_CMD_PROPERTY: {
                COMMAND_PROPERTY: [
                    "http://localhost:8080/source/api/v1/foo",
                    "POST", "data"]
            },
            PROJECTS_PROPERTY: {
                project_name: {DISABLED_PROPERTY: True}
            }
        }

        assert mirror_project(config, project_name, False,
                              None, None) == CONTINUE_EXITVAL
        verify(opengrok_tools.utils.mirror). \
            call_rest_api(config.get(DISABLED_CMD_PROPERTY),
                          PROJECT_SUBST, project_name)

def test_disabled_command_api_text_append(monkeypatch):
    """
    Test that message text is appended if DISABLED_PROPERTY is a string.
    """

    text_to_append = "foo bar"

    def mock_call_rest_api(command, b, c):
        disabled_command = config.get(DISABLED_CMD_PROPERTY)
        assert disabled_command
        command_args = disabled_command.get(COMMAND_PROPERTY)
        assert command_args
        data = command_args[2]
        assert data
        text = data.get("text")
        assert text
        assert text.find(text_to_append)

        return mock(spec=requests.Response)

    with monkeypatch.context() as m:
        m.setattr("opengrok_tools.utils.mirror.call_rest_api",
                  mock_call_rest_api)

        project_name = "foo"
        data = {'messageLevel': 'info', 'duration': 'PT5M',
                'tags': ['%PROJECT%'],
                'text': 'disabled project'}
        config = {
            DISABLED_CMD_PROPERTY: {
                COMMAND_PROPERTY: [
                    "http://localhost:8080/source/api/v1/foo",
                    "POST", data]
            },
            PROJECTS_PROPERTY: {
                project_name: {
                    DISABLED_REASON_PROPERTY: text_to_append,
                    DISABLED_PROPERTY: True
                }
            }
        }

        mirror_project(config, project_name, False, None, None)

def test_disabled_command_run():
    """
    Make sure that mirror_project() results in calling run_command().
    """
    spy2(opengrok_tools.utils.mirror.run_command)
    project_name = "foo"
    config = {
        DISABLED_CMD_PROPERTY: {
            COMMAND_PROPERTY: ["cat"]
        },
        PROJECTS_PROPERTY: {
            project_name: {DISABLED_PROPERTY: True}
        }
    }

    assert mirror_project(config, project_name, False,
                          None, None) == CONTINUE_EXITVAL
    verify(opengrok_tools.utils.mirror).run_command(ANY, project_name)

def test_disabled_command_run_args():
    """
    Make sure that run_command() calls Command.execute().
    """
    cmd = mock(spec=Command)
    project_name = "foo"
    run_command(cmd, project_name)
    verify(cmd).execute()

def test_mirror_project_timeout(monkeypatch):
    """
    Test mirror_project() timeout inheritance/override from global
    configuration to get_repos_for_project(). The test merely verifies
    that the timeout values are passed between the expected functions,
    not whether it actually affects the execution.
    """
    cmd_timeout = 3
    hook_timeout = 4

    def mock_get_repos(*args, **kwargs):
        mock_get_repos.called = True

        assert kwargs[CMD_TIMEOUT_PROPERTY] == cmd_timeout

        # Technically this function should return list of Repository objects
        # however for this test this is not necessary.
        return ['foo']

    def mock_process_hook(hook_ident, hook, source_root, project_name_arg,
                          proxy, hook_timeout_arg):
        assert hook_timeout_arg == hook_timeout

        # We want to terminate mirror_project() once this function runs.
        # This way mirror_project() will return FAILURE_EXITVAL
        return False

    def test_mirror_project(config):
        retval = mirror_project(config, project_name, False,
                                "http://localhost:8080/source", "srcroot")
        assert retval == FAILURE_EXITVAL

        # TODO: is there better way to ensure that get_repos_for_project()
        #       was actually called ?
        assert mock_get_repos.called

    with monkeypatch.context() as m:
        mock_get_repos.called = False
        m.setattr("opengrok_tools.utils.mirror.get_repos_for_project",
                  mock_get_repos)
        m.setattr("opengrok_tools.utils.mirror.process_hook",
                  mock_process_hook)

        project_name = "foo"
        # override testing
        global_config_1 = {
            PROJECTS_PROPERTY: {
                project_name: {
                    CMD_TIMEOUT_PROPERTY: cmd_timeout,
                    HOOK_TIMEOUT_PROPERTY: hook_timeout
                }
            },
            CMD_TIMEOUT_PROPERTY: cmd_timeout * 2,
            HOOK_TIMEOUT_PROPERTY: hook_timeout * 2
        }
        # inheritance testing
        global_config_2 = {
            CMD_TIMEOUT_PROPERTY: cmd_timeout,
            HOOK_TIMEOUT_PROPERTY: hook_timeout
        }

        test_mirror_project(global_config_1)
        test_mirror_project(global_config_2)

def test_get_repos_for_project(monkeypatch):
    """
    Test argument passing between get_repos_for_project() and get_repository()
    """
    project_name = 'foo'
    proxy_dict = {"http_proxy": "http://foo.bar:80",
                  "https_proxy": "http://foo.bar:80"}
    git_cmd_path = "/foo/git"
    commands = {"git": git_cmd_path}
    timeout = 314159
    test_repo = "/" + project_name

    def mock_get_repos(*args):
        return [test_repo]

    def mock_get_repo_type(*args):
        return "Git"

    with tempfile.TemporaryDirectory() as source_root:
        # Note that it is actually not necessary to create real
        # Git repository for the test to work. This is due to
        # the way how Repository objects are created.
        with monkeypatch.context() as m:
            m.setattr("opengrok_tools.utils.mirror.get_repos",
                      mock_get_repos)
            m.setattr("opengrok_tools.utils.mirror.get_repo_type",
                      mock_get_repo_type)

            repos = get_repos_for_project(project_name, None, source_root,
                                          commands=commands,
                                          proxy=proxy_dict,
                                          command_timeout=timeout)
            assert len(repos) == 1
            assert isinstance(repos[0], GitRepository)
            git_repo = repos[0]
            assert git_repo.timeout == timeout
            assert git_repo.command == git_cmd_path
            assert git_repo.env.items() >= proxy_dict.items()

            # Now ignore the repository
            repos = get_repos_for_project(project_name, None, source_root,
                                          ignored_repos=['.'])
            assert len(repos) == 0

DEFAULT_COMMAND = 'default-command'

@pytest.mark.parametrize(['expected_command', 'config'], [
    (DEFAULT_COMMAND, None),
    ('/usr/bin/git', '/usr/bin/git'),
    (DEFAULT_COMMAND, {}),
    (DEFAULT_COMMAND, {'incoming': '/bin/false'}),
    (DEFAULT_COMMAND, []),
    ('/usr/local/bin/git', {'command': '/usr/local/bin/git'}),
    (
            '/usr/local/bin/git',
            {'command': '/usr/local/bin/git', 'incoming': '/bin/false'}
    )
])
def test_mirroring_custom_repository_command(expected_command, config):
    assert expected_command == Repository._repository_command(config, lambda: DEFAULT_COMMAND)

@pytest.mark.parametrize(
    ('touch_binary'), [
        pytest.param('/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/touch'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/touch'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_invoke_command(touch_binary):
    checking_file = 'incoming.txt'
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': [touch_binary, checking_file]
        }, None, None, None)
        assert repository.incoming() is False
        assert checking_file in os.listdir(repository_root)

@pytest.mark.parametrize(
    ('echo_binary'), [
        pytest.param('/bin/echo',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/echo'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/echo',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/echo'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_changes(echo_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': [echo_binary, 'new incoming changes!']
        }, None, None, None)
        assert repository.incoming() is True

@pytest.mark.parametrize(
    ('true_binary'), [
        pytest.param('/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/true'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/true'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_no_changes(true_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': true_binary
        }, None, None, None)
        assert repository.incoming() is False

@pytest.mark.parametrize(
    ('false_binary'), [
        pytest.param('/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/false'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/false'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_error(false_binary):
    with pytest.raises(RepositoryException):
        with tempfile.TemporaryDirectory() as repository_root:
            repository = GitRepository(mock(), repository_root, 'test-1', {
                'incoming': false_binary
            }, None, None, None)
            repository.incoming()

def test_mirroring_incoming_invoke_original_command():
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1',
                                   None, None, None, None)
        with when(repository).incoming_check().thenReturn(0):
            repository.incoming()
            verify(repository).incoming_check()

@pytest.mark.parametrize(
    ('touch_binary'), [
        pytest.param('/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/touch'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/touch'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_invoke_command(touch_binary):
    checking_file = 'sync.txt'
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': [touch_binary, checking_file]
        }, None, None, None)
        assert repository.sync() == 0
        assert checking_file in os.listdir(repository_root)

@pytest.mark.parametrize(
    ('true_binary'), [
        pytest.param('/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/true'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/true'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_success(true_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': true_binary
        }, None, None, None)
        assert 0 == repository.sync()

@pytest.mark.parametrize(
    ('false_binary'), [
        pytest.param('/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/false'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/false'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_error(false_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': false_binary
        }, None, None, None)
        assert 1 == repository.sync()

def test_mirroring_sync_invoke_original_command():
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1',
                                   None, None, None, None)
        with when(repository).reposync().thenReturn(0):
            repository.sync()
            verify(repository).reposync()


import numpy as np
import json
from pydantic.dataclasses import dataclass

@dataclass
class Detector:
    sample_rate: int = 0
    center_freq: int = 0
    # Your custom params are below, call them whatever you want
    param1: int = 1
    param2: str = 'test2'
    param3: float = 5.67

    def detect(self, samples):
        print(samples[0:10])
        print(self.sample_rate)
        print(self.center_freq)
        print(self.param1)
        print(self.param2)
        print(self.param3)

        # Your detection (and optionally, classification) code here

        # For the return, make a list, then for each detected emission, add one of these dicts to the list:
        annotations = []
        an = {}
        an['core:freq_lower_edge'] = 1 # Hz
        an['core:freq_upper_edge'] = 2 # Hz
        an['core:sample_start'] = 3
        an['core:sample_count'] = 4
        an["core:description"] = "Unknown"
        annotations.append(an)
        return annotations

if __name__ == "__main__":
    # Example of how to test your detector locally
    fname = "C:\\Users\\marclichtman\\Downloads\\synthetic"
    with open(fname + '.sigmf-meta', 'r') as f:
        meta_data = json.load(f)
    sample_rate = meta_data["global"]["core:sample_rate"]
    center_freq = meta_data["captures"][0]['core:frequency']
    samples = np.fromfile(fname + '.sigmf-data', dtype=np.complex64)
    params = {'sample_rate': sample_rate, 'center_freq': center_freq, 'param1': 1, 'param2': 'test2', 'param3': 5.67}
    detector = Detector(**params)
    annotations = detector.detect(samples)
    print(annotations)


# from ast import Assert

# class Item:

#     # def __init__(self) -> None:
#     #     pass
#     pay_rate=0.8
#     #
#     def __init__(self, name:str, price:float, number:int):

#         # all class methods need to recieve a self argument as it is passed
#         # by default in the call statement

#         assert price>=0,f"price not greater than zero  {price}"
#         assert number>=0

#         # assert statements are used to assert conditions upon receiving data

#         print("hello")
#         self.name = name
#         self.price = price
#         self.number = number

#     def cal_total_price(self):
#         print(self.price * self.number)

# item1 = Item("anand", -100, 45)

# item1.cal_total_price()

# item2 = Item('hello', 1234, 5)

# item2.cal_total_price()

n = 10
nb=len(bin(n))-2
for i in range(0,n+1):
    li=nb-(len(bin(i))-2)
    d=bin(i)
    print('a'*(li),d[2:],sep='')


import json
import pickle

student_ID = "student_ID"
student_name = "student_name"
student_age = "student_age"
student_address = "student_address"
total_course_info = "total_course_info"
num_of_course_learned = "num_of_course_learned"
total_course_info = "total_course_info"
learning_course_info = "learning_course_info"
course_code = "course_code"
course_name = "course_name"
course_teacher = "course_teacher"
course_open_date = "course_open_date"
course_close_date = "course_close_date"

json_file_name = "ITT_Student.json"
g_json_bigdata_id_index_file = "ITT_Student_ID_Index.txt"
g_json_bigdata_left_id_list = []
g_json_bigdata = []
student_index_list = []
# temp_total_course_info = []

def add_student_total_course_info(student_index) :
    while 1 :
        order_1 = input("현재 수강 과목 정보를 추가하시겠습니까?(예=1 아니오=1)\n\
입력동작 : ")
        if order_1 == '2':
            break
        elif order_1 == '1':
            course_code_input = input("학생이 수강중인 강의코드를 입력하십시오.(강의코드 알파벳 2자리 + 개걍 년,월,일 6자리)\n\
    (e.g. IB171106) : ")
            course_name_input = input("해당 강의코드의 강의명을 입력하십시오.\n\
    (e.g. IoT 빅데이터 실무반) : ")
            course_teacher_input = input("해당 강의코드의 강사명을 입력하십시오.\n\
    (e.g. 이현구) : ")
            course_open_date_input = input("해당 강의코드의 개강날짜를 입력하십시오.\n\
    (e.g. 2017-11-06) : ")
            course_close_date_input = input("해당 강의코드의 종강날짜를 입력하십시오.\n\
    (e.g. 2018-09-05) : ")
            g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][course_code_input] = \
                {
                    course_name: course_name_input, course_teacher: course_teacher_input,
                    course_open_date: course_open_date_input, course_close_date: course_close_date_input
                }
    return None

def add_student_info(student_index) :
    student_name_input = input("학생 이름을 입력해 주십시오.\n\
(e.g. 김상엽) : ")
    student_age_input = input("학생 나이를 입력해 주십시오.\n\
(e.g. 30) : ")
    student_address_input = input("학생 주소를 입력해 주십시오.\n\
(e.g.  경북 경산시 조영동) : ")
    num_of_course_learned_input = input("과거 수강 횟수를 입력해 주십시오.\n\
(e.g. 0) : ")
    g_json_bigdata[0][student_index] =\
        {
            student_name : student_name_input,
            student_age : student_age_input,
            student_address :student_address_input,
            total_course_info :
                {
                    num_of_course_learned : num_of_course_learned_input,
                    learning_course_info : {}
                }
        }
    add_student_total_course_info(student_index)
    return None

def show_student_info(student_index) :
    print("<학생정보>")
    print("-학생아이디 : %s" %student_index)
    print("-학생이름 : %s" %g_json_bigdata[0][student_index][student_name])
    print("-학생나이 : %s" %g_json_bigdata[0][student_index][student_age])
    print("-학생주소 : %s" %g_json_bigdata[0][student_index][student_address])
    print("-수강정보")
    print("--과거수강횟수 : %s" %g_json_bigdata[0][student_index][total_course_info][num_of_course_learned])
    temp_attending_class_keys_list = list(g_json_bigdata[0][student_index][total_course_info][learning_course_info][0].keys())
    for y in temp_attending_class_keys_list:
        print("--현재수강과목코드 : %s" % y)
        print("--현재수강강의이름 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_name])
        print("--현재수강강의교사 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_teacher])
        print("--현재수강강의개강 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_open_date])
        print("--현재수강강의종강 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_close_date])
        print()
    return None

def get_student_index_list() :
    global student_index_list
    student_index_list = list(g_json_bigdata[0].keys())
    return None

def show_all() :
    for x in student_index_list :
        print(show_student_info(x))
        print("------------------------------------------------------------------------")
    return None

def check_codition(stored_data, input_data) :
    divide_stored_data = list(str(stored_data))
    divide_inputted_data = list(str(input_data))
    append_or_not = '0'
    starting_index = -1
    for x in range(len(divide_stored_data)) :
        if divide_stored_data[x] == divide_inputted_data[0] :
            starting_index = x
            break
    if starting_index == -1 : return append_or_not
    elif starting_index != -1 :
        for x in range(len(divide_inputted_data)) :
            try :
                if divide_stored_data[starting_index+x] == divide_inputted_data[x] : append_or_not = '1'
                else :
                    append_or_not = '0'
                    return append_or_not
            except :
                append_or_not = '0'
                return append_or_not
        return append_or_not

def show_searched_student_info(searched_student_name_list) :
    if len(searched_student_name_list) == 1:
        show_student_info(searched_student_name_list[0])
    elif len(searched_student_name_list) > 1:
        print("2개 이상의 결과가 검색되어 아이디와 이름만 보여줍니다.")
        for x in searched_student_name_list:
            print("아이디 : %s    이름 : %s " %(x, g_json_bigdata[0][x][student_name]))
    elif searched_student_name_list == []:
        print("수강중인 학생이 없습니다.")
    return None

def search_with_student_primary_key(input_data) :
    searched_student_name_list = []
    for x in student_index_list :
        if check_codition(x, input_data) == '1' :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_into_one_depth(input_data_category, input_data) :
    searched_student_name_list = []
    for x in student_index_list:
        if check_codition(g_json_bigdata[0][x][input_data_category], input_data) == '1':
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_with_past_registrarion(student_ragistrantion) :
    searched_student_name_list = []
    for x in student_index_list :
        if check_codition(g_json_bigdata[0][x][total_course_info][num_of_course_learned], student_ragistrantion) == '1' :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_with_now_studying() :
    searched_student_name_list = []
    for x in student_index_list :
        if list(g_json_bigdata[0][x][total_course_info][learning_course_info].keys()) != [] :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_into_two_depth(input_data_category, input_data) :
    searched_student_name_list = []
    for x in student_index_list:
        student_class_code_list = list(g_json_bigdata[0][x][total_course_info][learning_course_info].keys())
        for y in student_class_code_list:
            if check_codition(g_json_bigdata[0][x][total_course_info][learning_course_info][y][input_data_category], input_data) == '1':
                searched_student_name_list.append(x)
                break
    show_searched_student_info(searched_student_name_list)
    return None

def delet_with_student_primary_key(student_index) :
    for x in student_index_list :
        if student_index == x :
            del(g_json_bigdata[0][x])
            g_json_bigdata_left_id_list.append(student_index)
            return None

def update_student_info(primary_key) :
    for x in student_index_list :
        if primary_key == x[1] :
            while 1 :
                print("선택하신 아이디의 정보는 아래와 같습니다.")
                print("1:학생이름 : %s" % g_json_bigdata[0][x][student_name])
                print("2:학생나이 : %s" % g_json_bigdata[0][x][student_age])
                print("3:학생주소 : %s" % g_json_bigdata[0][x][student_address])
                print("-수강정보")
                print("4:과거수강횟수 : %s" % g_json_bigdata[0][x][total_course_info][num_of_course_learned])
                temp_attending_class_keys_list = list(g_json_bigdata[0][x][total_course_info][num_of_course_learned].keys())
                print("5:현재수강강의 코드 : ", end = '')
                print(temp_attending_class_keys_list)
                print("6:현재수강강의 추가 : ")
                print("7:수정 끝")
                order_1 = input("바꾸려는 정보를 선택하십시오. : ")
                if order_1 == '7' : break
                elif order_1 == '1' :
                    new_name = input("이름을 입력하십시오.\ne.g)김상엽 : ")
                    g_json_bigdata[0][x][student_name] = new_name
                elif order_1 == '2' :
                    new_age = input("나이를 입력하십시오.\ne.g)30 : ")
                    g_json_bigdata[0][x][student_age] = new_age
                elif order_1 == '3' :
                    new_address = input("주소를 입력하십시오.\ne.g)경북 경산시 조영동 : ")
                    g_json_bigdata[0][x][student_address] = new_address
                elif order_1 == '4' :
                    new_p_registration = input("과거수강횟수를 입력하십시오.\ne.g)0 : ")
                    g_json_bigdata[0][x][total_course_info][num_of_course_learned] = new_p_registration
                elif order_1 == '5' :
                    c_code_from = input("바꾸려는 강의를 코드를 입력하십시오. : ")
                    print(">>현재수강강의이름 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_name])
                    print(">>현재수강강의교사 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_teacher])
                    print(">>현재수강강의개강 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_open_date])
                    print(">>현재수강강의종강 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_close_date])
                    order_2 = input("현재 수강 과목 정보를 수정하시겠습니까?(1:예 2:아니오 3:삭제) : ")
                    if order_2 == '2' : pass
                    elif order_2 == '1' :
                        print()
                        del(g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from])
                        add_student_total_course_info(primary_key)
                    elif order_2 == '3' : del(g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from])
                elif order_1 == '6' :
                    print()
                    add_student_total_course_info(primary_key)
    return None

def main_function() :
    global student_index_list
    print("학생 정보 관리 프로그램입니다.".center(30))
    while 1 :
        order_1 = input("\n동작을 선택해 주십시오.\n\
1:학생 정보입력\n2:학생 정보조회\n3:학생 정보수정\n4:학생 정보삭제\n5:프로그램 종료\n동작입력 : ")
        if order_1 == '5' : break
        elif order_1 == '1' :
            print()
            while 1 :
                print()
                g_json_bigdata.append(add_student_info(student_index_list.pop()))
                student_index_list = get_student_index_list()
                order_2 = input("계속 하시겠습니까?(1:예 2:아니오) : ")
                if order_2 == '2' : break
                elif order_2 == '1' : continue
        elif order_1 == '2' :
            print()
            while 1 :
                order_2 = input("\n1:전체정보출력\n2:ID로조회\n3:이름으로조회\n4:나이로조회\n5:주소로조회\n6:과거수강횟수로조회\n\
7:현재강의를수강하는학생으로조회\n8:현재수강과목의강의명으로조회\n9:현재수강과목의강사로조회\n10:상위메뉴로\n동작입력 : ")
                if order_2 == '10' : break
                elif order_2 == '1' :
                    print()
                    show_all()
                elif order_2 == '2' :
                    print()
                    input_data = input("아이디를 입력하세요. : ")
                    search_with_student_primary_key(input_data)
                elif order_2 == '3' :
                    print()
                    input_data = input("이름을 입력하세요. : ")
                    search_into_one_depth(student_name, input_data)
                elif order_2 == '4' :
                    print()
                    input_data = input("나이를 입력하세요. : ")
                    search_into_one_depth(student_age, input_data)
                elif order_2 == '5' :
                    print()
                    input_data = input("주소를 입력하세요. : ")
                    search_into_one_depth(student_address, input_data)
                elif order_2 == '6' :
                    print()
                    input_data = input("과거수강횟수를 입력하세요. : ")
                    search_with_past_registrarion(input_data)
                elif order_2 == '7' :
                    print()
                    search_with_now_studying()
                elif order_2 == '8' :
                    print()
                    input_data = input("현재수강과목의강의명을 입력하세요. : ")
                    search_into_two_depth(course_name, input_data)
                elif order_2 == '9' :
                    print()
                    input_data = input("현재수강과목의강사명을 입력하세요. : ")
                    search_into_two_depth(course_teacher, input_data)
        elif order_1 == '3' :
            print()
            input_data = input("수정하려는 아이디를 입력하십시오. : ")
            update_student_info(input_data)
            student_index_list = get_student_index_list()
        elif order_1 == '4' :
            print()
            input_data = input("삭제하려는 아이디를 입력하십시오. : ")
            delet_with_student_primary_key(input_data)
            student_index_list = get_student_index_list()
    return None

while 1 :
    try :
        with open(json_file_name, encoding='UTF8') as json_file :
            json_object = json.load(json_file)
            json_string = json.dumps(json_object)
            g_json_bigdata = json.loads(json_string)
        with open(g_json_bigdata_id_index_file, 'rb') as json_index:
            student_index_list = pickle.load(json_index)
        main_function()
        with open('ITT_Student.json', 'w', encoding='utf8') as outfile:
            jsonResult = g_json_bigdata
            readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
            outfile.write(readable_result)
        with open(g_json_bigdata_id_index_file, 'wb') as json_index:
            pickle.dump(student_index_list, json_index)
        break

    except FileNotFoundError :
        file_not_found = input("%s파일을 찾을 수 없습니다. 다음 동작을 선택해 주십시오.\n1:신규생성 2:경로설정" %json_file_name)
        if file_not_found == '2' :
            path = input("경로를 입력해 주십시오.\ne.g)D:\\Pycharm_projects\\Emigration\\Jason_Practice\\ : ")
            try:
                with open(path + json_file_name, encoding='UTF8') as json_file:
                    json_object = json.load(json_file)
                    json_string = json.dumps(json_object)
                    g_json_big_data = json.loads(json_string)
                with open(path + g_json_bigdata_id_index_file, 'rb') as index_file:
                    left_primary_key_index_list = pickle.load(index_file)
                student_primary_key_list = get_student_index_list()
                main_function()
                with open(path + json_file_name, 'w', encoding='utf8') as outfile:
                    jsonResult = g_json_big_data
                    readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
                    outfile.write(readable_result)
                with open(path + g_json_bigdata_id_index_file, 'wb') as json_index:
                    pickle.dump(left_primary_key_index_list, json_index)
                break
            except:
                print("해당 경로에도 파일이 없습니다. 기본 경로에서 신규생성을 합니다.")
                file_not_found = '1'
        if file_not_found == '1' :
            temp_id_index_str = ""
            for x in range(1,1000) :
                temp_id_index_str += "ITT" + "{0:0>3}".format(str(x)) + '\n'
            temp_id_index_list = list(reversed(temp_id_index_str[:-1].split('\n')))
            with open('ITT_Student.json', 'w', encoding='utf8') as outfile:
                json_big_dict = []
                while 1:
                    order_1 = input("학생정보를 추가하시겠습니까?(1:예 2:아니오) : ")
                    if order_1 == '2':
                        break
                    elif order_1 == '1':
                        g_json_bigdata.append(add_student_info(temp_id_index_list.pop()))
                jsonResult = g_json_bigdata
                readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
                outfile.write(readable_result)
                print('ITT_Student.json SAVED')
            with open(g_json_bigdata_id_index_file, 'wb') as index_file:
                pickle.dump(student_index_list, index_file)
                print("%s파일 생성 완료" %g_json_bigdata_id_index_file)
        else : print("동작 선택 좀 똑디 하소.")

김상엽_빅데이터저장_코딩테스트.py


import subprocess
import sys
import os
from os import walk
import xml.etree.ElementTree as ET
import csv
import pygsheets

# If modifying these scopes, delete the file token.json.
#SCOPES = 'https://www.googleapis.com/auth/spreadsheets.readonly'
# The ID and range of a sample spreadsheet.
SPREADSHEET_ID = '1VN8nzFbih88_PpIBE0QM3lK4TCqvWTCCUnBnuCYgnBc'
SAMPLE_RANGE_NAME = 'Summary!A2:G'
CTS_DIR= '/home/ubuntu/Documents/android-cts'

def main():
	result_values = ["1","b","c","x","y","z"]
	print("hello")
	update_google_sheet(result_values)
	print("goobye")

def update_google_sheet(result_values):
    '''
    This method use to get result from test_result.xml to update google sheet report
    '''
    SPREADSHEET_ID = '1VN8nzFbih88_PpIBE0QM3lK4TCqvWTCCUnBnuCYgnBc'
    gc = pygsheets.authorize()
    print("flag")
    # Open spreadsheet and then workseet
    sh = gc.open_by_key(SPREADSHEET_ID)
    wks_list = sh.worksheets()
    module_ws = wks_list[2]# Modules tab
    print(len(wks_list))
    module_ws.append_table(values=result_values, start="A1",end=None, dimension='ROWS', overwrite=True)

if __name__ == '__main__':
    main()


import wiringpi as wp
import time

OUTPUT = 1
INPUT = 0
HIGH = 1
LOW = 0

wp.wiringPiSetupGpio()
wp.pinMode(1,0)

while True:
	value1 = wp.analogRead(25)
	time.sleep(0.1)
	value2 = wp.analogRead(25)
	speed = value2

	print(value)


from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
from django.db.models import Q
from django.core.paginator import Paginator
from .models import Book
import json

def index(request):
    context = {
        'message': 'TEST TEST TEST'
    }
    return render(request, 'Libraryapp/index.html', context)

def search(request):
    books = Book.objects.all().order_by('title')
    text = request.GET.get('text', '')
    if text != '':
        books = Book.objects.filter(Q(title__icontains=text)
                                    |Q(author__icontains=text)).order_by('title')

    page = request.GET.get('page', 1)
    books_per_page = 5
    pagintor = Paginator(books, books_per_page)
    books = pagintor.page(page)
    books_data = []
    for book in books:
        books_data.append({
            'title': book.title,
            'author': book.author,
            'image': book.image,
            'year': book.year,
            'pages': book.pages,
            'url': book.url,
            'country': book.country,
            'language': book.language,
        })

    return JsonResponse({'books': books_data})


from math import log, sqrt, exp
from scipy import stats

def bsm_call_value(S0, K, T, r, sigma):
    """
    Valuation of European call option in Black-Scholes-Merton model

    Args:
        S0: initial stock/index level
        K : strike price
        T : maturity date (in year fractions)
        r : constant risk-free short rate
        sigma: volatility factor in diffusion term

    Returns:
        value: present value of the European call option
    """
    S0 = float(S0)
    d1 = (log(S0 / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))
    d2 = (log(S0 / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))

    # stats.norm.cdf is the cumulative distribution function for the normal distribution
    value = (S0 * stats.norm.cdf(d1, 0.0, 1.0) -
             K * exp(-r * T) * stats.norm.cdf(d2, 0.0, 1.0))

    return value

def bsm_vega(S0, K, T, r, sigma):
    """
    Vega of European option in BSM model

    Args:
        S0: initial stock / index level
        K : strike price
        T : maturity date (in year fractions)
        r : constant risk-free short rate
        sigma : volatility factor in diffusion term

    Returns:
        vega: partial derivative of BSM formula w.r.t sigma, i.e. vega
    """
    S0 = float(S0)
    d1 = (log(S0 / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))
    vega = S0 * stats.norm.pdf(d1, 0.0, 1.0) * sqrt(T)
    return vega

def bsm_call_imp_vol(S0, K, T, r, C0, sigma_est, it=100):
    """
    Implied volatility of European call option in BSM model

    Args:
        S0: initial stock/index level
        K : strike price
        T : maturity date (in year fractions)
        r : constant risk-free rate
        sigma_est : estimate of impl volatility
        it: number of iterations

    Returns:
        sigma_est: numerically estimated implied volatility
    """
    for i in range(it):
        sigma_est -= ((bsm_call_value(S0, K, T, r, sigma_est) - C0) /
                      bsm_vega(S0, K, T, r, sigma_est))
    return sigma_est


#US population
year_start = 1950
years_list = []
max_deviation = [0]
# print(max_deviation)

txt_file = open('data/USPopulation.txt', 'r')                     #open txt file
population_list = [int(i) for i in txt_file.readlines()]          #create list and integer stroke

for i in population_list:                                         #create years list 50' to 90'
    years_list.append(year_start)
    year_start += 1

for i in range(len(population_list)):                             #создаем список с приростом
    if i == 40:
        break
    dev = population_list[i + 1] - population_list[i]
    max_deviation.append(dev)

max_num = max(max_deviation)                                    #вычисляем максимальное изменение
index_max_num = max_deviation.index(max_num)                    #получаем индекс года
year_max_dev = years_list[index_max_num]                        #получаем год максимального прироста

print(max_num)
print(year_max_dev)

Tonny_geddis_book_problem/other/temp_2 (Конфликтующая копия с компьютера MacBook13 2022-08-25).py


import re
from arsenal.misc import force

class ParseError(Exception):
    """ Custom exception class used by this module. """
    pass

class Span(object):
    __slots__ = ('label','begins','ends')
    def __init__(self, label, begins, ends):
        self.label = label
        self.begins = begins
        self.ends = ends
    def __repr__(self):
        return 'Span(label=%r, begins=%r, ends=%r)' % (self.label, self.begins, self.ends)
    def __eq__(self, other):
        if isinstance(other, Span):
            return (self.label == other.label and self.begins == other.begins and self.ends == other.ends)
        else:
            return len(other) == 3 and (self.label == other[0] and self.begins == other[1] and self.ends == other[2])
    def __iter__(self):
        return iter((self.label, self.begins, self.ends))

WhitespaceLexer = re.compile('\S+')

Lexer = re.compile('|'.join(["http://\S+",                               # keep urls together.. might include invalid URL characters
                             "\S+@\S+",
                             "[0-9][0-9]?\s*\([0-9]?\)",                 # e.g. "7(4)" common in volume
                             "\(\s*[0-9][0-9]?\s*\)",                    # e.g. "(4)" common in volume
                             "[0-9]+\s*-\s*[0-9]+",                      # possible page number
                             "[0-9]+(?:st|nd|rd|th)",
                             "[A-Z]\.",
                             "Ph\.?[Dd]\.",
                             #"[Vv]ol(?:\.|ume)\s*[0-9]+",               # make "Vol. 3" one token.
                             "(?:Vol|Proc|Dept|Univ|No|Inc|Dr)\s*\.",
                             "pp\.",
                             "\(\s*[0-9][0-9][0-9][0-9][a-z]?\s*\)",     # e.g. "(1994)" year in parens
                             "[Ee]d(?:s?\.|itors?)",
                             #"\w+-\w+",
                             "\+[A-Z]+\+",
                             "[a-zA-Z]+(?:'s)?",
                             #"[0-9]+(?:\.[0-9]+)?",                      # 231 or 2.0, but not 2.
                             "[0-9]+",
                             "[()\"'\-\.,]",
                             "\S+"]))

TaggedText = re.compile("<([a-z0-9_]+)>([\w\W]+?)</([a-z0-9_]+)>|([^<>\s]+)", re.IGNORECASE)

def fromSGML(f, linegrouper="\n", bioencoding=False):
    for line in re.split(linegrouper, open(f).read()):
        if bioencoding:
            seq = sgml2bio(line)
        else:
            seq = sgml2seq(line)
        if seq:
            yield seq

@force
def sgml2segmentation(x, lexer=WhitespaceLexer):
    """
    >>> sgml2segmentation('<title>Cat in the Hat</title><author>Dr. Seuss</author>')
    [('title', ['Cat', 'in', 'the', 'Hat']), ('author', ['Dr.', 'Seuss'])]
    """
    x = x.strip().replace("\n", " +L+ ")
    for (tag, tagged, close, outside) in TaggedText.findall(x):
        if tag != close:
            raise ParseError("opening (%s) and closing (%s) tags do not match in sequence\n    %r\n" % (tag, close, x))
        if tagged:
            yield (tag, lexer.findall(tagged))
        else:
            for w in lexer.findall(outside):
                yield ("O", [w])

@force
def sgml2bio(x):
    """
    >>> sgml2bio('<title>Cat in the Hat</title><author>Dr. Seuss</author>')
    [('B-title', 'Cat'), ('I-title', 'in'), ('I-title', 'the'), ('I-title', 'Hat'), ('B-author', 'Dr.'), ('I-author', 'Seuss')]
    """
    for (tag, tokens) in sgml2segmentation(x):
        tokens = iter(tokens)
        yield ('B-' + tag, next(tokens))
        for w in tokens:
            yield ('I-' + tag, w)

@force
def sgml2seq(x):
    """
    >>> sgml2seq('<title>Cat in the Hat</title><author>Dr. Seuss</author>')
    [('title', 'Cat'), ('title', 'in'), ('title', 'the'), ('title', 'Hat'), ('author', 'Dr.'), ('author', 'Seuss')]
    """
    for (tag, tokens) in sgml2segmentation(x):
        for w in tokens:
            yield (tag, w)

def bracket2bio(x):
    """
    generate BIO-token pairs from bracket-style annotation.
    Note: splits text of spaces, so wordsplitting should already be done.

    >>> x = bracket2bio("[TITLE Cat in the Hat][AUTHOR Dr. Seuss]")
    >>> list(x)                                  #doctest:+NORMALIZE_WHITESPACE
    [('B-TITLE', 'Cat'), ('I-TITLE', 'in'), ('I-TITLE', 'the'),
     ('I-TITLE', 'Hat'), ('B-AUTHOR', 'Dr.'), ('I-AUTHOR', 'Seuss')]
    """
    if '\n' in x:
        raise ParseError('No newlines allowed in brack2bio annotation.')
    for label, tagged, word in re.findall('(?:(?:\[([A-Z0-9]+)\s+(.+?)\s*\]\s*)|(.+?)(?:\s+|$))', x):
        if word:
            yield ('O', word)
        else:
            words = iter(tagged.split())
            yield ('B-%s' % label, next(words))
            for w in words:
                if '[' in w or ']' in w:
                    raise ParseError('brackets can not appear within a word.')
                yield ('I-%s' % label, w)

# TIMV: we want something like a LineGroupIterator
def line_groups(text, pattern):
    """
    Very simple function for breaking up text into groups based on a
    single pattern.

    >>> list(line_groups("a BB c d BB", "BB"))
    ['a', 'c d']
    """
    for group in re.split(pattern, text):    # TODO: make this "lazier"
        group = group.strip()
        if group:
            yield group

def extract_contiguous(s, labeler=None):
    """
    >>> list(extract_contiguous(""))
    []

    >>> list(extract_contiguous("AAAA"))
    [Span(label='A', begins=0, ends=4)]

    >>> list(extract_contiguous("AABBC"))
    [Span(label='A', begins=0, ends=2), Span(label='B', begins=2, ends=4), Span(label='C', begins=4, ends=5)]

    >>> list(extract_contiguous("AABBB"))
    [Span(label='A', begins=0, ends=2), Span(label='B', begins=2, ends=5)]
    """
    if labeler is not None:
        s = map(labeler, s)
    prev = None
    b = e = 0
    for e, token in enumerate(s):
        if token != prev:
            if prev is not None:
                yield Span(prev, b, e)
            b = e
        prev = token
    # emit lingering bits
    if prev is not None:
        yield Span(prev, b, e + 1)

@force
def bio2span(seq, tagger=None, include_O=True):
    if tagger is not None:
        seq = map(tagger, seq)
    phrase = None
    intag = None
    for i, lbl in enumerate(seq):
        if lbl is None:
            lbl = 'O'
        label = lbl[2:]
        if lbl.startswith('B-'):
            if intag and phrase:
                yield phrase
            phrase = Span(label, i, i + 1)
        elif lbl.startswith('I-'):
            if intag == label:             # and youre still in the same span
                phrase.ends = i + 1
            else:                          # you're in a new span (hueristic correction)
                if phrase:
                    yield phrase
                phrase = Span(label, i, i + 1)
        else:
            if intag:                      # was in tag, now outiside ("O")
                if phrase:
                    yield phrase
                phrase = None
            if include_O:
                yield Span(lbl, i, i + 1)
        if lbl == 'O':
            intag = None
        else:
            intag = label
    if intag and phrase:                   # close any lingering spans
        yield phrase

if __name__ == '__main__':
    from arsenal.misc import piped
    def main():
        for line in piped() or []:
            for (label, w) in sgml2bio(line):
                print('%s\t%s' % (label, w))
            print()
    main()


#!/usr/bin/env python3
# - ids.py - #
# - Snort IDS for offline pcap scanning on Ubuntu 18.04
# - Author: mcb2Eexe
# - Last modified: 16 Oct 19
# - Dependencies: python3, pip3, pip install requests, curl, snort, rules dir, pcap dir, logs dir,
# - ToDo: add alert output, check sig errors, check snort version, check how to use rulepacks best, bash setup script
# - Issues: Rule packs have errors when updated (possibly ubuntu repo snort version), can't delete tar files

import sys
import time
import tarfile
import requests
from os import path
import subprocess as sp

SNORT_PATH = "/etc/snort/"
RULE_PATH = "/etc/snort/community_rules"
SNORT_TAR = "/etc/snort/community_rules/SC.tar.gz"
ET_TAR = "/etc/snort/community_rules/ET.tar.gz"
ET_URL = "https://rules.emergingthreats.net/open/snort-2.9.7.0/emerging.rules.tar.gz"
SNORT_URL = "https://snort.org/downloads/community/community-rules.tar.gz"

# - Clear Terminal - #
def clear():
    sp.call('clear', shell=True)
# --- #

# - Help menu - #
def help(error):
    clear()
    print(error)
    print("\n Usage: $ids.py [pcap] [-h]")
    print("\n     [pcap]  Input pcap file to be scanned")
    print("\n Example: ids file.pcap\n")
# --- #

# - Snort Function - #
def snort(pcap):
    sp.call('snort -A full -c {}snort.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - Emerging Threats Function - #
def et(pcap):
    sp.call('snort -A full -c {}snort_et.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - All Rules Function - #
def combined(pcap):
    sp.call('snort -A full -c {}snort_all.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - Update Rules - #
def update(rules, vendor, tar):
    clear()
    print("Searching for {} updates...".format(vendor))
    response = requests.get(rules)
    if response.status_code == 200:
        print("{} update found! Updating...".format(vendor))
        r = requests.get(rules)
        open('{}'.format(tar), 'wb').write(r.content)
        tar = tarfile.open("{}".format(tar), 'r:*')
        tar.extractall('{}'.format(RULE_PATH))
        tar.close()
        print("Update complete!")
        time.sleep(2)
    else:
        print("{} rule update not found!".format(vendor))
# --- #

# - Check Connections - #
def connection_status(url):
    response = requests.get('{}'.format(url))
    return response.status_code
# --- #

# - Start - #
while True:
    if len(sys.argv) != 2:
        help(" Invalid input! Check and try again...")
        exit(0)
    elif not path.exists(sys.argv[1]):
        help(" Pcap not found! Check and try again...")
        exit(0)
    else:
        argv = sys.argv[1]
    clear()
    sp.call('snort -V', shell=True)
    print(" Menu:\n")
    print("   [1] Snort Community Open Rules")
    print("   [2] Emerging Threats Rules")
    print("   [3] All Rules")
    print("   [4] Update Rules")
    arg = input("\n Select an option: ")
    if arg == "1":
        snort(argv)
        break
    if arg == "2":
        et(argv)
        break
    if arg == "3":
        combined(argv)
        break
    if arg == "4":
        update(ET_URL, "ET", ET_TAR)
        update(SNORT_URL, "SC", SNORT_TAR)
        clear()
        continue
    else:
        clear()
        print("\nUnknown input! Expecting integer...\n")
        continue
# - End - #


import numpy as np
import pandas as pd
from regressionML import RegressionML
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from statistics import stdev
from statistics import mean
import winsound
from sklearn.feature_selection import f_regression
import matplotlib.pyplot as plt
import seaborn as sns
from model_setting import Model_Setting

class SensitivityAnalysisExperiment(Model_Setting):
    def __init__(self, regression=True):
        super().__init__()

        self.cf['targets'] = ['OV (Oily value)']
        # self.cf['targets'] = ['OS (Oil separation)']
        # self.cf['targets'] = ['Turbidity']

        self.size_experiments = 10
        self.ml = RegressionML()
        for clf in self.ml.regressors:
            name = type(clf).__name__
            self.excel[name + '-MAE'] = []

        for clf in self.ml.regressors:
            name = type(clf).__name__
            self.excel[name + '-MAE-STD'] = []

    def show_heatmap_matrix(self, df, columns):
        cm = np.corrcoef(df[columns].values.T)
        sns.set(font_scale=0.8)
        # hm = sns.heatmap(cm, char=True, annot=True, square=True, fmt='.2f', annot_kws={'size':15}, yticklabels=columns, xticklabels=columns)
        hm = sns.heatmap(cm, annot=True, square=True, fmt='.2f', annot_kws={'size': 15}, yticklabels=columns,
                         xticklabels=columns)
        plt.show()
        plt.close()

    def get_f_regression(self, X_train, y_train):
        f_test, _ = f_regression(X_train, y_train)

        results_f_dict = {}
        for i in range(X_train.shape[1]):
            results_f_dict[X_train.columns.values[i]] = f_test[i]

        results_f_dict = {k: v for k, v in sorted(results_f_dict.items(), key=lambda item: item[1], reverse=True)}

        print("======================")
        print("=       F-test       =")
        f_results = ''
        for k, v in results_f_dict.items():
            f_results += f'{k}\t{v}\r'
            print(f'{k}\t{v}')

        return f_results

    def run_ml_removing_sensitivity_analysis(self):
        # 1. Load data from xlsx
        excel_data = pd.read_excel(self.cf['file'], self.cf['sheet'])

        for X in self.cf['selected_Xs']:
            selected_Xs = self.cf['selected_Xs'].copy()
            selected_Xs.remove(X)
            print(f'******************** removed {X} ********************')
            for target in self.cf['targets']:
                print('target: ', target)
                self.cf['target'] = target

                # 2. Select variables
                data = excel_data[selected_Xs + [self.cf['target']]]

                # 3. Remove data if it contains 'nan'
                data = data[~data[self.cf['target']].isna()]
                if len(data) == 0:
                    continue

                # 4. Surfactant is converted to a numeric variable
                if 'Surfactant name' in data.columns.values:
                    data['Surfactant name'] = LabelEncoder().fit_transform(data['Surfactant name'])

                # 5. Normalization is performed
                scaled_df = MinMaxScaler(feature_range=(0, 1)).fit_transform(data)
                data = pd.DataFrame(scaled_df, index=data.index, columns=data.columns)

                # 6. Separate Xs and y
                df_X, df_y = data[selected_Xs], data[self.cf['target']]

                # 7. Convert y to a categorical variable for classification
                # df_y = self.convert_to_categorical_variable(df_y)

                # 9. Perform several ML experiments
                sum_results = None
                all_results = {}
                for i in range(self.size_experiments):
                    # 9. Split into training and test part
                    X_train, X_test, y_train, y_test = train_test_split(df_X, df_y, test_size=.2)

                    # 13. Set data X and y for ML
                    self.ml.set_train_test_data(X_train, X_test, y_train, y_test)

                    # 14. Perform ML
                    results = self.ml.perform_ML()

                    if len(all_results) == 0:
                        all_results = {x: [v] for x, v in results.items()}
                    else:
                        for x, v in all_results.items():
                            for x2, v2 in results.items():
                                if x2 == x:
                                    v.append(v2)

                    if sum_results is None:
                        sum_results = results
                    else:
                        sum_results = {x: v + v2 for x, v in sum_results.items() for x2, v2 in results.items() if x2 == x}

                # 15. Set all results for the excel output
                for clf in self.ml.regressors:
                    name = type(clf).__name__
                    # self.excel[name + '-MAE'].append(avg_results[name])
                    self.excel[name + '-MAE'].append(round(mean(all_results[name]), 4))
                    self.excel[name + '-MAE-STD'].append(round(stdev(all_results[name]), 4))

                self.excel['Train Size'].append(len(X_train))
                self.excel['Test Size'].append(len(X_test))
                self.excel['Target'].append(target)
                self.excel['X Removed'].append(X)
                self.excel['F Results'].append('')

        self.save_excel_file()

    def run_experiment(self, excel_data):
        for target in self.cf['targets']:
            print('target: ', target)
            self.cf['target'] = target

            # 2. Select variables
            data = excel_data[self.cf['selected_Xs'] + [self.cf['target']]]

            # 3. Remove data if it contains 'nan'
            data = data[~data[self.cf['target']].isna()]

            # 5. Normalization is performed
            scaled_df = MinMaxScaler(feature_range=(0, 1)).fit_transform(data)
            data = pd.DataFrame(scaled_df, index=data.index, columns=data.columns)

            # 6. Separate Xs and y
            df_X, df_y = data[self.cf['selected_Xs']], data[self.cf['target']]

            # 8. Perform Sensitivity Analysis
            f_results = ''
            # columns = df_X.columns.copy()
            # columns.append(self.cf['target'])
            # self.show_heatmap_matrix(data, columns)
            f_results = self.get_f_regression(df_X, df_y)

    def run(self):
        # 1. Load data from xlsx
        excel_data = pd.read_excel(self.cf['file'], self.cf['sheet'])

        self.run_experiment(excel_data)

        self.save_excel_file()

# 1. looks at F-score
# ex = SensitivityAnalysisExperiment().run()

# 2. performs one variable removal test
SensitivityAnalysisExperiment().run_ml_removing_sensitivity_analysis()

winsound.Beep(1000, 440)


"""
Write a function to find the longest common prefix string amongst an array of strings.
If there is no common prefix, return an empty string "".
"""
from typing import List

class Solution:
    def longestCommonPrefix(self, strs: List[str]) -> str:
        strs.sort()
        first = strs[0]
        last = strs[-1]
        prefix = ""
        for i in range(len(first)):
            if i < len(last) and first[i] == last[i]:
                prefix += first[i]
            else:
                break
        return prefix

if __name__ == '__main__':
    s = Solution()

    # Test case 1
    strs1 = ["flower","flow","flight"]
    expected1 = "fl"
    r1 = s.longestCommonPrefix(strs1)
    assert r1 == expected1, f"Test Case 1 Failed: expected output {expected1}, but got {r1}"

    # Test case 2
    strs2 = ["dog","racecar","car"]
    expected2 = ""
    r2 = s.longestCommonPrefix(strs2)
    assert r2 == expected2, f"Test Case 2 Failed: expected output {expected2}, but got {r2}"

    # Test case 3
    strs3 = [""]
    expected3 = ""
    r3 = s.longestCommonPrefix(strs3)
    assert r3 == expected3, f"Test Case 3 Failed: expected output {expected3}, but got {r3}"


# -*- coding: utf-8 -*-

from tastypie.resources import ModelResource
from exhibition.models import Exhibition, ExhibitionComment, ExhibitionRate
from work.models import Work
from buildengine.models import PrefWebsite
from django.contrib.auth.models import User
from tastypie.serializers import Serializer
import time
from tastypie import fields
from django.utils import simplejson
from django.core.serializers import json
from tastypie.authorization import DjangoAuthorization
from tastypie.resources import ModelResource, ALL, ALL_WITH_RELATIONS

class UserResource(ModelResource):
    class Meta:
        queryset = User.objects.filter(is_active=True)
        resource_name = 'user'
        excludes = ['email', 'password', 'is_superuser', 'date_joined', 'is_staff', 'is_active', 'last_login', 'resource_uri',
                    'first_name', 'last_name']

class ExhibitionResource(ModelResource):
    user = fields.ForeignKey(UserResource, 'user', full=True)
    class Meta:
        queryset = Exhibition.objects.filter(user_id__groups__name="Artist")
        resource_name = 'exhibition'
        excludes = ['rate']
        authorization = DjangoAuthorization()
        filtering = {
            'comment': ALL_WITH_RELATIONS,
            'date_pub': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'date_begin': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'date_end': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'mapLongitude': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'mapLatitude': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'zipcode': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
        }

    def dehydrate(self, bundle):
        bundle.data['work_preview'] = [str(x.image) for x in Work.objects.filter(user_id=bundle.obj.user.id)[:3]]
        bundle.data['anonymity'] = ''.join([str(x.anonymity) for x in PrefWebsite.objects.filter(user_id=bundle.obj.user.id)])
        arrayRate = [x.rate for x in ExhibitionRate.objects.filter(exhibition = bundle.obj.id)]
        if(len(arrayRate) == 0):
            bundle.data['rateMedian'] = -1
        else:
            bundle.data['rateMedian'] = sum(arrayRate) / len(arrayRate)
        if not bundle.data['anonymity']:
            bundle.data['last_name'] = ''.join([str(x.last_name) for x in User.objects.filter(id=bundle.obj.user.id)])
            bundle.data['first_name'] = ''.join([str(x.first_name) for x in User.objects.filter(id=bundle.obj.user.id)])
        return bundle

class CommentResource(ModelResource):
    exhibition = fields.ForeignKey(ExhibitionResource, 'exhibition', full=True)
    class Meta:
        queryset = ExhibitionComment.objects.all()
        resource_name = 'comment'
        authorization = DjangoAuthorization()
        filtering = {
            'exhibition': ALL_WITH_RELATIONS
        }

    def dehydrate(self, bundle):
        bundle.data['username'] = bundle.obj.user.username
        bundle.data['rate'] = ''.join([str(x.rate) for x in ExhibitionRate.objects.filter(exhibition=bundle.obj.exhibition.id, user=bundle.obj.user.id)])
        return bundle


import time
from copy import deepcopy

import matplotlib.pyplot as plt
from aocd import submit

from pyutils import parsing, utils

def parse(data):
    items = parsing.recursively_split(data, [("\n\n", None), ("\n", lambda x: list(map(int, x)))])
    totals = [sum(cals) for cals in items]
    return totals

def part_a(data):
    # Visualization.
    # max_idx = max(range(len(data)), key=lambda idx: data[idx])
    # colors = ["#1f77b4"] * len(data)
    # colors[max_idx] = "r"
    # fig, ax = plt.subplots(1, figsize=(15, 6))
    # ax.bar(range(len(data)), data, color=colors)
    # rect = ax.patches[max_idx]
    # height = rect.get_height()
    # ax.text(
    #     rect.get_x() + rect.get_width() / 2, height + 5, data[max_idx], ha="center", va="bottom"
    # )
    # ax.set_xlabel("Elf ID")
    # ax.set_ylabel("Calories")
    # fig.savefig("viz/2022/2022-01-a.png")
    # End Visualization.
    return max(data)

def part_b(data):
    idxs = sorted(range(len(data)), key=lambda idx: data[idx])
    top = idxs[-3:]
    cals = sum(data[idx] for idx in top)

    # Visualization.
    # colors = ["#1f77b4"] * len(data)
    # fig, ax = plt.subplots(1, figsize=(15, 6))
    # ax.set_xlabel("Elf ID")
    # ax.set_ylabel("Calories")
    # for idx in top:
    #     colors[idx] = "r"

    # ax.bar(range(len(data)), data, color=colors)
    # rects = ax.patches
    # for idx in top:
    #     rect = rects[idx]
    #     height = rect.get_height()
    #     ax.text(
    #         rect.get_x() + rect.get_width() / 2, height + 5, data[idx], ha="center", va="bottom"
    #     )

    # fig.savefig("viz/2022/2022-01-b.png")
    # End visualization.

    return cals

if __name__ == "__main__":
    with open("/home/ryan/Desktop/repos/aoc/aoc-all/inputs/2022/day01.txt", "r") as fp:
        data = fp.read().strip()

    data = parse(data)

    data_a = deepcopy(data)
    print("Running day 1 part A")
    start_a = time.perf_counter()

    solution_a = part_a(data_a)

    stop_a = time.perf_counter()
    elapsed_a = stop_a - start_a
    print(f"Part A finished in {utils.format_time(elapsed_a)} with solution: {solution_a}, submitting...")
    submit(solution_a, part="a", day=1, year=2022)

    print("Running day 1 part B")
    start_b = time.perf_counter()

    solution_b = part_b(data)

    stop_b = time.perf_counter()
    elapsed_b = stop_b - start_b
    print(f"Part B finished in {utils.format_time(elapsed_b)} with solution: {solution_b}, submitting...")
    submit(solution_b, part="b", day=1, year=2022)


import time

import openpyxl
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from PageObjects.Elements import Elements
from Utils import XLUtils
from Utils.BaseClass import BaseClass
from Utils.readProperties import ReadConfig

class TestOne(BaseClass):
    log = BaseClass.getLogger()
    def test_datadriven(self):
        readconfig = ReadConfig()  # create object without any argument
        self.baseURL = readconfig.getApplicationURL()
        self.path= readconfig.getinputPath()
        self.driver.get(self.baseURL)
        self.log.info("Opened the URL")
        self.log.info(self.driver.title)
        elements_page=Elements(self.driver)
        # Scroll down to the "Elements" link
        self.driver.execute_script("window.scrollBy(0,300)")
        # elements_link = self.driver.find_element(By.XPATH, "//h5[normalize-space()='Elements']")
        elements_link = elements_page.click_elements_link()
        #To hover the cursor over the elements link and click
        actions = ActionChains(self.driver)
        actions.move_to_element(elements_link).click().perform()
        self.log.info("Clicked on Elements")
        # elements_heading=self.driver.find_element(By.XPATH,"//div[@class ='main-header']").text
        elements_heading = elements_page.verify_elements_heading().text
        self.log.info(elements_heading)
        assert "Elements" in elements_heading

        # self.driver.find_element(By.XPATH, "//span[normalize-space()='Text Box']").click()
        elements_page.click_elements_textbox().click()
        self.log.info("Clicked on Text Box")
        workbook = openpyxl.load_workbook(
            "C:/Users/amuly/PycharmProjects1/WebUIAutomation/Input/Input_TestBox.xlsx")
        worksheet = workbook.active
        self.rows = XLUtils.getRowCount(self.path, 'Sheet1')
        for r in range(2, self.rows+1):
            self.Fullname_value = XLUtils.readData(self.path, 'Sheet1', r, 1)
            self.Email_value = XLUtils.readData(self.path, 'Sheet1', r, 2)
            self.CurrentAddress_value = XLUtils.readData(self.path, 'Sheet1', r, 3)
            self.PermenantAdd_value = XLUtils.readData(self.path, 'Sheet1', r, 4)
            # Fullname_input=self.driver.find_element(By.XPATH,"//input[@id='userName']")
            Fullname_input = elements_page.input_textbox_fullname()
            Fullname_input.clear()
            Fullname_input.send_keys(self.Fullname_value)
            self.log.info("Given Full Name")
            # Email_input = self.driver.find_element(By.XPATH, "//input[@id='userEmail']")
            Email_input = elements_page.input_textbox_email()
            Email_input.clear()
            Email_input.send_keys(self.Email_value)
            self.log.info("Given Email")
            # CurrentAddr_input = self.driver.find_element(By.XPATH, "//textarea[@id='currentAddress']")
            CurrentAddr_input = elements_page.input_textbox_currentaddress()
            CurrentAddr_input.clear()
            CurrentAddr_input.send_keys(self.CurrentAddress_value)
            self.log.info("Given Current Address")
            # PermanantAdd_input = self.driver.find_element(By.XPATH, "//textarea[@id='permanentAddress']")
            PermanantAdd_input = elements_page.input_textbox_permanentaddress()
            PermanantAdd_input.clear()
            PermanantAdd_input.send_keys(self.PermenantAdd_value)
            self.log.info("Given Permenant Address")
            self.driver.execute_script("window.scrollBy(0,300)")
            # self.driver.find_element(By.XPATH, "//button[@id = 'submit']").click()
            elements_page.click_submit_button().click()
            self.log.info("Clicked on Submit button")
            time.sleep(5)
            FullName=self.driver.find_element(By.XPATH, "//p[@id='name']").text
            Email=self.driver.find_element(By.XPATH, "//p[@id='email']").text
            Current_Address=self.driver.find_element(By.XPATH, "//p[@id='currentAddress']").text
            Permanent_Address=self.driver.find_element(By.XPATH, "//p[@id='permanentAddress']").text
            self.log.info(FullName)
            self.log.info(Email)
            self.log.info(Current_Address)
            self.log.info(Permanent_Address)
            self.log.info("Form details displayed on UI as given in text boxes")
            split_name=FullName.split(':')
            name_split=split_name[1]
            if name_split == self.Fullname_value:
                assert True
                self.log.info("Pass")
            else:
                assert False
                self.log.info("Fail")


from datetime import datetime, date

def formatNumber(numbers):
    value = '('
    for acode in numbers[0:3]:
        value += str(acode)
    value += ') '
    for f3 in numbers[3:6]:
        value += str(f3)
    value += '-'
    for l4 in numbers[6:11]:
        value += l4
    return value

def indexOfLastDigit(number):
    index = 0
    for x in range(0, len(number)):
        if (number[x].isdigit()):
            index = x
    return index

def parseDateTime(fileName):
    for i in [i for i, ltr in enumerate(fileName) if ltr == '_']:
        fileName = fileName[:i] + ':' + fileName[i + 1:]
    try:
        return datetime.fromisoformat(fileName[fileName.rindex(' ') + 1:fileName.rindex('Z')])
    except ValueError as err:
        try:
            return date.fromisoformat(fileName[fileName.rindex(' ') + 1:fileName.rindex('T')])
        except Exception as err2:
            raise Exception

def sortDictionaryByDate(history):
    return sorted(history)


"""TOJA command-line application."""
import asyncio
import click
import logging
import logging.config
import yaml
import os

from cerberus import Validator
from typing import Union

from .actions import actions
from ..background import run_background_app
from ..server import run_application_server
from ..utils import set_config
from ..setup import setup_backend

logger = logging.getLogger(__name__)

CONFIG_SCHEMA = {
    'server': {
        'type': 'dict',
        'schema': {
            'host': {
                'type': 'string',
                'default': '127.0.0.1'
            },
            'port': {
                'type': 'integer',
                'default': 6543
            },
            'base': {
                'type': 'string',
                'default': 'http://127.0.0.1:6543'
            }
        },
        'default': {
            'host': '127.0.0.1',
            'port': 6543,
            'base': 'http://127.0.0.1:6543'
        }
    },
    'database': {
        'type': 'dict',
        'required': True,
        'schema': {
            'server': {
                'type': 'string',
                'default': 'http://localhost:5984'
            },
            'user': {
                'type': 'string'
            },
            'password': {
                'type': 'string'
            }
        }
    },
    'mosquitto': {
        'type': 'dict',
        'required': True,
        'schema': {
            'host': {
                'type': 'string',
                'default': 'localhost'
            },
            'username': {
                'type': 'string',
                'nullable': True,
                'default': None
            },
            'password': {
                'type': 'string',
                'nullable': True,
                'default': None
            }
        }
    },
    'meilisearch': {
        'type': 'dict',
        'required': True,
        'schema': {
            'server': {
                'type': 'string',
                'required': True
            },
            'auth_token': {
                'type': 'string',
                'empty': True,
                'default': None
            }
        }
    },
    'email': {
        'type': 'dict',
        'schema': {
            'server': {
                'type': 'string',
                'required': True,
                'empty': False,
            },
            'secure': {
                'type': 'boolean',
                'default': True,
            },
            'auth': {
                'type': 'dict',
                'schema': {
                    'user': {
                        'type': 'string',
                        'required': True,
                        'empty': False,
                    },
                    'password': {
                        'type': 'string',
                        'required': True,
                        'empty': False,
                    }
                }
            },
            'sender': {
                'type': 'string',
                'default': 'automaton@oldjokearchive.com'
            }
        }
    },
    'debug': {
        'type': 'boolean',
        'default': False,
    },
    'test': {
        'type': 'boolean',
        'default': False,
    },
    'logging': {
        'type': 'dict'
    }
}

def validate_config(config: dict) -> dict:
    """Validate the configuration.

    :param config: The configuration to validate
    :type config: dict
    :return: The validated and normalised configuration
    :rtype: dict
    """
    validator = Validator(CONFIG_SCHEMA)
    if validator.validate(config):
        return validator.normalized(config)
    else:
        error_list = []

        def walk_error_tree(err: Union[dict, list], path: str) -> None:
            if isinstance(err, dict):
                for key, value in err.items():
                    walk_error_tree(value, path + (str(key), ))
            elif isinstance(err, list):
                for sub_err in err:
                    walk_error_tree(sub_err, path)
            else:
                error_list.append(f'{".".join(path)}: {err}')

        walk_error_tree(validator.errors, ())
        error_str = '\n'.join(error_list)
        raise click.ClickException(f'Configuration errors:\n\n{error_str}')

@click.group()
def main() -> None:
    """Run the TOJA applications."""
    config = None
    if os.path.exists('config.yml'):
        with open('config.yml') as in_f:
            config = yaml.load(in_f, Loader=yaml.FullLoader)
    elif os.path.exists('/etc/toja/config.yml'):
        with open('/etc/toja/config.yml') as in_f:
            config = yaml.load(in_f, Loader=yaml.FullLoader)
    if not config:
        raise click.ClickException('No configuration found (./config.yml, /etc/toja/config.yml)')
    normalised = validate_config(config)
    set_config(normalised)
    if 'logging' in normalised:
        logging.config.dictConfig(normalised['logging'])

@click.command()
def server() -> None:
    """Run the TOJA server."""
    run_application_server()

main.add_command(server)

@click.command()
def background() -> None:
    """Run the TOJA background processing application."""
    run_background_app()

main.add_command(background)
main.add_command(actions)

@click.command()
def setup() -> None:
    """Set up the TOJA system."""
    asyncio.run(setup_backend())

main.add_command(setup)


import ctypes
import numpy as np
import os
import sys
import torch
import pdb

class _gnn_lib(object):

    def __init__(self, args):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        self.lib = ctypes.CDLL('%s/build/dll/libgnn.so' % dir_path)

        self.lib.GetGraphStruct.restype = ctypes.c_void_p
        self.lib.PrepareBatchGraph.restype = ctypes.c_int
        self.lib.PrepareSparseMatrices.restype = ctypes.c_int
        self.lib.NumEdgePairs.restype = ctypes.c_int

        if sys.version_info[0] > 2:
            args = [arg.encode() for arg in args]  # str -> bytes for each element in args
        arr = (ctypes.c_char_p * len(args))()
        arr[:] = args
        self.lib.Init(len(args), arr)

        self.batch_graph_handle = ctypes.c_void_p(self.lib.GetGraphStruct())

    def _prepare_graph(self, graph_list, is_directed=0):
        edgepair_list = (ctypes.c_void_p * len(graph_list))()
        list_num_nodes = np.zeros((len(graph_list), ), dtype=np.int32)
        list_num_edges = np.zeros((len(graph_list), ), dtype=np.int32)
        for i in range(len(graph_list)):
            if type(graph_list[i].edge_pairs) is ctypes.c_void_p:
                edgepair_list[i] = graph_list[i].edge_pairs
            elif type(graph_list[i].edge_pairs) is np.ndarray:
                edgepair_list[i] = ctypes.c_void_p(graph_list[i].edge_pairs.ctypes.data)
            else:
                raise NotImplementedError

            list_num_nodes[i] = graph_list[i].num_nodes
            list_num_edges[i] = graph_list[i].num_edges
        total_num_nodes = np.sum(list_num_nodes)
        total_num_edges = np.sum(list_num_edges)

        self.lib.PrepareBatchGraph(self.batch_graph_handle,
                                len(graph_list),
                                ctypes.c_void_p(list_num_nodes.ctypes.data),
                                ctypes.c_void_p(list_num_edges.ctypes.data),
                                ctypes.cast(edgepair_list, ctypes.c_void_p),
                                is_directed)

        return total_num_nodes, total_num_edges

    def PrepareSparseMatrices(self, graph_list, is_directed=0):
        assert not is_directed
        total_num_nodes, total_num_edges = self._prepare_graph(graph_list, is_directed)

        n2n_idxes = torch.LongTensor(2, total_num_edges * 2)
        n2n_vals = torch.FloatTensor(total_num_edges * 2)

        e2n_idxes = torch.LongTensor(2, total_num_edges * 2)
        e2n_vals = torch.FloatTensor(total_num_edges * 2)

        subg_idxes = torch.LongTensor(2, total_num_nodes)
        subg_vals = torch.FloatTensor(total_num_nodes)

        idx_list = (ctypes.c_void_p * 3)()
        idx_list[0] = n2n_idxes.numpy().ctypes.data
        idx_list[1] = e2n_idxes.numpy().ctypes.data
        idx_list[2] = subg_idxes.numpy().ctypes.data

        val_list = (ctypes.c_void_p * 3)()
        val_list[0] = n2n_vals.numpy().ctypes.data
        val_list[1] = e2n_vals.numpy().ctypes.data
        val_list[2] = subg_vals.numpy().ctypes.data

        self.lib.PrepareSparseMatrices(self.batch_graph_handle,
                                ctypes.cast(idx_list, ctypes.c_void_p),
                                ctypes.cast(val_list, ctypes.c_void_p))

        n2n_sp = torch.sparse.FloatTensor(n2n_idxes, n2n_vals, torch.Size([total_num_nodes, total_num_nodes]))
        e2n_sp = torch.sparse.FloatTensor(e2n_idxes, e2n_vals, torch.Size([total_num_nodes, total_num_edges * 2]))
        subg_sp = torch.sparse.FloatTensor(subg_idxes, subg_vals, torch.Size([len(graph_list), total_num_nodes]))
        return n2n_sp, e2n_sp, subg_sp

dll_path = '%s/build/dll/libgnn.so' % os.path.dirname(os.path.realpath(__file__))
if os.path.exists(dll_path):
    GNNLIB = _gnn_lib(sys.argv)
else:
    GNNLIB = None


"""
- Worst Case:   O(n^2)
- Best Case:    O(n + k)
- Average Case: O(n + k)

where n is the number of elements in the array to be sorted, and k is the number
of buckets

Bucket sort is a sorting algorithm that works by distributing the elements of an
array into a number of buckets. Each bucket is then sorted individually, either
using a different sorting algorithm or by recursively applying the bucket
sorting algorithm.

This implementation will work only on a list of integers
"""

from quicksort import quicksort

def bucketsort(input_list):
    biggest = max(input_list)
    n_buckets = biggest / 10 + 1
    buckets = [list() for _ in range(n_buckets)]
    for idx, x in enumerate(input_list):
        buckets[x / 10].append(x)
    for idx, bucket in enumerate(buckets):
        buckets[idx] = quicksort(bucket)
    result = []
    for bucket in buckets:
        result += bucket
    return result


################################################
# Name: Jacob Pace
# Date: 2/8/2023
# Description: The Reckoner
################################################

from tkinter import *

####### MAY HAVE TO CHANGE THIS FOR IT TO WORK FOR GRADING
####### THIS IS HOW WINDOWS IS
folder = "images\\"
ButtonDisplays = [
    ["lpr", "rpr", "clr", "bak"],
    ["7", "8", "9", "div"],
    ["4", "5", "6", "mul"],
    ["1", "2", "3", "sub"],
    ["0", "dot", "", "add"],
    ["eql-wide", "pow", "mod"]
]

buttonDef = {
    "lpr" : "(",
    "rpr" : ")",
    "clr" : "clr",
    "pow" : "**",
    "mul" : "*",
    "div" : "/",
    "sub" : "-",
    "dot" : ".",
    "eql-wide" : "=",
    "add" : "+",
    "mod" : "%",
}

class MainGUI(Frame):
    def __init__(self, parent):
        Frame.__init__(self, parent, bg="white")
        self.setupGUI()

    def setupGUI(self):
        self.display = Label(self, text="", anchor=E, bg="white", height=2, font=("TextGyreAdventor", 45))
        self.display.grid(row=0, column=0, columnspan=4, sticky=E+W+N+S)

        # creating buttons
        for row in range(len(ButtonDisplays)):
            for column in range(len(ButtonDisplays[row])):
                if (ButtonDisplays[row][column] != "" and ButtonDisplays[row][column] != "eql-wide" and ButtonDisplays[row][column] != "pow" and ButtonDisplays[row][column] !="mod"):
                    img = PhotoImage(file=folder+ButtonDisplays[row][column]+".gif")
                    button = Button(self, bg="white", image=img, borderwidth=0, highlightthickness=0, activebackground="white", command=lambda button=ButtonDisplays[row][column]: self.process(button))
                    button.image = img
                    button.grid(row=row+1, column=column, stick=N+S+E+W)
                elif (ButtonDisplays[row][column] == "eql-wide"):
                    img = PhotoImage(file=folder+ButtonDisplays[row][column]+".gif")
                    button = Button(self, bg="white", image=img, borderwidth=0, highlightthickness=0, activebackground="white", command=lambda button=ButtonDisplays[row][column]: self.process(button))
                    button.image = img
                    button.grid(row=row+1, column=0, columnspan=2, stick=N+S+E+W)
                elif (ButtonDisplays[row][column] == "pow"):
                    img = PhotoImage(file=folder+ButtonDisplays[row][column]+".gif")
                    button = Button(self, bg="white", image=img, borderwidth=0, highlightthickness=0, activebackground="white", command=lambda button=ButtonDisplays[row][column]: self.process(button))
                    button.image = img
                    button.grid(row=row+1, column=2, stick=N+S+E+W)
                elif (ButtonDisplays[row][column] == "mod"):
                    img = PhotoImage(file=folder+ButtonDisplays[row][column]+".gif")
                    button = Button(self, bg="white", image=img, borderwidth=0, highlightthickness=0, activebackground="white", command=lambda button=ButtonDisplays[row][column]: self.process(button))
                    button.image = img
                    button.grid(row=row+1, column=3, stick=N+S+E+W)

        self.pack(fill=BOTH, expand=1)

    def process(self, button):
        if (button in buttonDef):
            button = buttonDef[button]
        if (button == "clr"):
            self.display["text"] = ""
        elif (button == "bak"):
            self.display["text"] = self.display["text"][:-1]
        elif (button == "="):
            try:
                result = eval(self.display["text"])
                if (len(str(result)) > 14):
                    result = str(result)
                    result = str(result[0:11]+"...")
                    self.display["text"] = result
                elif (len(str(result)) < 14):
                    self.display["text"] = result
            except:
                self.display["text"] = "ERROR"
        elif (len(self.display["text"]) < 14):
                self.display["text"] += button

### MAIN ###

# create window
window = Tk()

# set window title
window.title("The Reckoner")

# generate the GUI
calc = MainGUI(window)

# display interface
window.mainloop()