# coding: utf-8

"""
    CryptoAPIs

    Crypto APIs is a complex and innovative infrastructure layer that radically simplifies the development of any Blockchain and Crypto related applications. Organized around REST, Crypto APIs can assist both novice Bitcoin/Ethereum enthusiasts and crypto experts with the development of their blockchain applications. Crypto APIs provides unified endpoints and data, raw data, automatic tokens and coins forwardings, callback functionalities, and much more.  # noqa: E501

    The version of the OpenAPI document: 2021-03-20
    Contact: developers@cryptoapis.io
    Generated by OpenAPI Generator (https://openapi-generator.tech)

    Do not edit the class manually.
"""

from __future__ import annotations
from inspect import getfullargspec
import json
import pprint
import re  # noqa: F401

from typing import Any, List, Optional
from pydantic import BaseModel, Field, StrictStr, ValidationError, validator
from cryptoapis.models.list_transactions_by_block_hash_ribsb import ListTransactionsByBlockHashRIBSB
from cryptoapis.models.list_transactions_by_block_hash_ribsbc import ListTransactionsByBlockHashRIBSBC
from cryptoapis.models.list_transactions_by_block_hash_ribsbsc import ListTransactionsByBlockHashRIBSBSC
from cryptoapis.models.list_transactions_by_block_hash_ribsd import ListTransactionsByBlockHashRIBSD
from cryptoapis.models.list_transactions_by_block_hash_ribsd2 import ListTransactionsByBlockHashRIBSD2
from cryptoapis.models.list_transactions_by_block_hash_ribse import ListTransactionsByBlockHashRIBSE
from cryptoapis.models.list_transactions_by_block_hash_ribsec import ListTransactionsByBlockHashRIBSEC
from cryptoapis.models.list_transactions_by_block_hash_ribsl import ListTransactionsByBlockHashRIBSL
from cryptoapis.models.list_transactions_by_block_hash_ribsz import ListTransactionsByBlockHashRIBSZ
from typing import Any, List
from pydantic import StrictStr, Field

LISTTRANSACTIONSBYBLOCKHASHRIBS_ONE_OF_SCHEMAS = ["ListTransactionsByBlockHashRIBSB", "ListTransactionsByBlockHashRIBSBC", "ListTransactionsByBlockHashRIBSBSC", "ListTransactionsByBlockHashRIBSD", "ListTransactionsByBlockHashRIBSD2", "ListTransactionsByBlockHashRIBSE", "ListTransactionsByBlockHashRIBSEC", "ListTransactionsByBlockHashRIBSL", "ListTransactionsByBlockHashRIBSZ"]

class ListTransactionsByBlockHashRIBS(BaseModel):
    """
    ListTransactionsByBlockHashRIBS
    """
    # data type: ListTransactionsByBlockHashRIBSB
    oneof_schema_1_validator: Optional[ListTransactionsByBlockHashRIBSB] = None
    # data type: ListTransactionsByBlockHashRIBSBC
    oneof_schema_2_validator: Optional[ListTransactionsByBlockHashRIBSBC] = None
    # data type: ListTransactionsByBlockHashRIBSL
    oneof_schema_3_validator: Optional[ListTransactionsByBlockHashRIBSL] = None
    # data type: ListTransactionsByBlockHashRIBSD
    oneof_schema_4_validator: Optional[ListTransactionsByBlockHashRIBSD] = None
    # data type: ListTransactionsByBlockHashRIBSD2
    oneof_schema_5_validator: Optional[ListTransactionsByBlockHashRIBSD2] = None
    # data type: ListTransactionsByBlockHashRIBSE
    oneof_schema_6_validator: Optional[ListTransactionsByBlockHashRIBSE] = None
    # data type: ListTransactionsByBlockHashRIBSEC
    oneof_schema_7_validator: Optional[ListTransactionsByBlockHashRIBSEC] = None
    # data type: ListTransactionsByBlockHashRIBSBSC
    oneof_schema_8_validator: Optional[ListTransactionsByBlockHashRIBSBSC] = None
    # data type: ListTransactionsByBlockHashRIBSZ
    oneof_schema_9_validator: Optional[ListTransactionsByBlockHashRIBSZ] = None
    actual_instance: Any
    one_of_schemas: List[str] = Field(LISTTRANSACTIONSBYBLOCKHASHRIBS_ONE_OF_SCHEMAS, const=True)

    class Config:
        validate_assignment = True

    def __init__(self, *args, **kwargs):
        if args:
            if len(args) > 1:
                raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`")
            if kwargs:
                raise ValueError("If a position argument is used, keyword arguments cannot be used.")
            super().__init__(actual_instance=args[0])
        else:
            super().__init__(**kwargs)

    @validator('actual_instance')
    def actual_instance_must_validate_oneof(cls, v):
        instance = ListTransactionsByBlockHashRIBS.construct()
        error_messages = []
        match = 0
        # validate data type: ListTransactionsByBlockHashRIBSB
        if not isinstance(v, ListTransactionsByBlockHashRIBSB):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSB`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSBC
        if not isinstance(v, ListTransactionsByBlockHashRIBSBC):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSBC`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSL
        if not isinstance(v, ListTransactionsByBlockHashRIBSL):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSL`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSD
        if not isinstance(v, ListTransactionsByBlockHashRIBSD):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSD`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSD2
        if not isinstance(v, ListTransactionsByBlockHashRIBSD2):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSD2`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSE
        if not isinstance(v, ListTransactionsByBlockHashRIBSE):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSE`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSEC
        if not isinstance(v, ListTransactionsByBlockHashRIBSEC):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSEC`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSBSC
        if not isinstance(v, ListTransactionsByBlockHashRIBSBSC):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSBSC`")
        else:
            match += 1
        # validate data type: ListTransactionsByBlockHashRIBSZ
        if not isinstance(v, ListTransactionsByBlockHashRIBSZ):
            error_messages.append(f"Error! Input type `{type(v)}` is not `ListTransactionsByBlockHashRIBSZ`")
        else:
            match += 1
        if match > 1:
            # more than 1 match
            raise ValueError("Multiple matches found when setting `actual_instance` in ListTransactionsByBlockHashRIBS with oneOf schemas: ListTransactionsByBlockHashRIBSB, ListTransactionsByBlockHashRIBSBC, ListTransactionsByBlockHashRIBSBSC, ListTransactionsByBlockHashRIBSD, ListTransactionsByBlockHashRIBSD2, ListTransactionsByBlockHashRIBSE, ListTransactionsByBlockHashRIBSEC, ListTransactionsByBlockHashRIBSL, ListTransactionsByBlockHashRIBSZ. Details: " + ", ".join(error_messages))
        elif match == 0:
            # no match
            raise ValueError("No match found when setting `actual_instance` in ListTransactionsByBlockHashRIBS with oneOf schemas: ListTransactionsByBlockHashRIBSB, ListTransactionsByBlockHashRIBSBC, ListTransactionsByBlockHashRIBSBSC, ListTransactionsByBlockHashRIBSD, ListTransactionsByBlockHashRIBSD2, ListTransactionsByBlockHashRIBSE, ListTransactionsByBlockHashRIBSEC, ListTransactionsByBlockHashRIBSL, ListTransactionsByBlockHashRIBSZ. Details: " + ", ".join(error_messages))
        else:
            return v

    @classmethod
    def from_dict(cls, obj: dict) -> ListTransactionsByBlockHashRIBS:
        return cls.from_json(json.dumps(obj))

    @classmethod
    def from_json(cls, json_str: str) -> ListTransactionsByBlockHashRIBS:
        """Returns the object represented by the json string"""
        instance = ListTransactionsByBlockHashRIBS.construct()
        error_messages = []
        match = 0

        # deserialize data into ListTransactionsByBlockHashRIBSB
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSB.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSBC
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSBC.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSL
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSL.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSD
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSD.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSD2
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSD2.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSE
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSE.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSEC
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSEC.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSBSC
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSBSC.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))
        # deserialize data into ListTransactionsByBlockHashRIBSZ
        try:
            instance.actual_instance = ListTransactionsByBlockHashRIBSZ.from_json(json_str)
            match += 1
        except (ValidationError, ValueError) as e:
            error_messages.append(str(e))

        if match > 1:
            # more than 1 match
            raise ValueError("Multiple matches found when deserializing the JSON string into ListTransactionsByBlockHashRIBS with oneOf schemas: ListTransactionsByBlockHashRIBSB, ListTransactionsByBlockHashRIBSBC, ListTransactionsByBlockHashRIBSBSC, ListTransactionsByBlockHashRIBSD, ListTransactionsByBlockHashRIBSD2, ListTransactionsByBlockHashRIBSE, ListTransactionsByBlockHashRIBSEC, ListTransactionsByBlockHashRIBSL, ListTransactionsByBlockHashRIBSZ. Details: " + ", ".join(error_messages))
        elif match == 0:
            # no match
            raise ValueError("No match found when deserializing the JSON string into ListTransactionsByBlockHashRIBS with oneOf schemas: ListTransactionsByBlockHashRIBSB, ListTransactionsByBlockHashRIBSBC, ListTransactionsByBlockHashRIBSBSC, ListTransactionsByBlockHashRIBSD, ListTransactionsByBlockHashRIBSD2, ListTransactionsByBlockHashRIBSE, ListTransactionsByBlockHashRIBSEC, ListTransactionsByBlockHashRIBSL, ListTransactionsByBlockHashRIBSZ. Details: " + ", ".join(error_messages))
        else:
            return instance

    def to_json(self) -> str:
        """Returns the JSON representation of the actual instance"""
        if self.actual_instance is None:
            return "null"

        to_json = getattr(self.actual_instance, "to_json", None)
        if callable(to_json):
            return self.actual_instance.to_json()
        else:
            return json.dumps(self.actual_instance)

    def to_dict(self) -> dict:
        """Returns the dict representation of the actual instance"""
        if self.actual_instance is None:
            return None

        to_dict = getattr(self.actual_instance, "to_dict", None)
        if callable(to_dict):
            return self.actual_instance.to_dict()
        else:
            # primitive type
            return self.actual_instance

    def to_str(self) -> str:
        """Returns the string representation of the actual instance"""
        return pprint.pformat(self.dict())


print("import modules...")
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD

print("load dataset...")

def load_data(filename):
    dat = np.loadtxt(filename, delimiter=',')
    if len(dat[0])==206:
        return (dat[:,0:196],dat[:,196:])
    if len(dat[0])==794:
        return (dat[:,0:784],dat[:,784:])

X, Y = load_data("mnist/mnist_test_2.csv")
INPUT_SIZE = len(X[0])
OUTPUT_SIZE = 10
print(X)
print(Y)

print("define and compile keras model...")
model = Sequential()
model.add(Dense(10, input_dim=INPUT_SIZE, activation='softmax'))
opt = SGD(lr=0.01, momentum=0.9)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])

print("fit keras model...")
old_accuracy = 0
for i in range(20):
    model.fit(X, Y, epochs=10, batch_size=100)
    _, accuracy = model.evaluate(X, Y)
    if abs(accuracy-old_accuracy) < 0.001:
        break
    old_accuracy = accuracy

def output_to_class(output):
    res = []
    for item in output:
        maxi = -1
        maxval = float("-inf")
        for i in range(len(item)):
            if item[i]>maxval:
                maxi, maxval = i, item[i]
        res.append(maxi)
    return res

expected = output_to_class(Y)
predicted = output_to_class(model.predict(X))
correct_count = 0
for i in range(len(expected)):
    if expected[i]==predicted[i]:
        correct_count += 1
print(f"Accuracy: {correct_count/len(expected)}")

# get weights and biases
weights, biases = model.layers[0].get_weights()
print("weights =", weights.tolist())
print("biases =", biases.tolist())


import sys

from swagger_ui import core
from swagger_ui.handlers import supported_list

def api_doc(app, **kwargs):
    doc = core.ApplicationDocument(app, **kwargs)

    handler = doc.match_handler()

    if not handler:
        raise Exception('No match application isinstance type!')

    if not callable(handler):
        raise Exception('handler is callable required!')

    return handler(doc)

def create_api_doc(app_type):
    def _api_doc(app, **kwargs):
        kwargs['app_type'] = app_type
        return api_doc(app, **kwargs)
    return _api_doc

for name in supported_list:
    setattr(sys.modules[__name__], '{}_api_doc'.format(name), create_api_doc(name))


import re
from random import choice
from time import time

from Cb_constants import CbServer
from FtsLib.FtsOperations import FtsHelper
from bucket_collections.collections_base import CollectionBase
from couchbase_helper.tuq_helper import N1QLHelper
from error_simulation.cb_error import CouchbaseError
from membase.api.rest_client import RestConnection
from platform_constants.os_constants import Windows
from remote.remote_util import RemoteMachineShellConnection
from table_view import TableView

from java.lang import Exception as Java_base_exception

class ConfigPurging(CollectionBase):
    def setUp(self):
        super(ConfigPurging, self).setUp()
        is_windows = False

        for node in self.cluster.servers:
            shell = RemoteMachineShellConnection(node)
            if shell.info.type.lower() == Windows.NAME:
                is_windows = True
            shell.enable_diag_eval_on_non_local_hosts()
            shell.disconnect()

        self.cluster_util.update_cluster_nodes_service_list(self.cluster)

        # Default purger values
        self.default_run_interval = 60
        self.default_purge_age = 300

        self.time_stamp = time()
        self.num_index = self.input.param("num_index", 0)
        self.index_type = self.input.param("index_type", CbServer.Services.FTS)
        self.index_name_len = self.input.param("index_name_len", 10)
        self.fts_index_partition = self.input.param("fts_index_partition", 1)
        self.index_replicas = self.input.param("gsi_index_replicas", 1)
        self.fts_helper = FtsHelper(self.cluster.fts_nodes[0]) \
            if self.cluster.fts_nodes else None
        self.n1ql_helper = N1QLHelper(server=self.cluster.query_nodes[0],
                                      use_rest=True, log=self.log) \
            if self.cluster.query_nodes else None
        self.spare_node = self.servers[-1]
        self.couchbase_base_dir = "/opt/couchbase"
        if is_windows:
            self.couchbase_base_dir = \
                "/cygdrive/c/Program\\ Files/Couchbase/Server"

        # Param order:
        # fts_name, bucket_name, index_partitions, scope_name, collection_name
        self.fts_param_template = '{ \
          "type": "fulltext-index", \
          "name": "%s", \
          "sourceType": "gocbcore", \
          "sourceName": "%s", \
          "sourceUUID": "%s", \
          "planParams": { \
            "maxPartitionsPerPIndex": 1024, \
            "indexPartitions": %d \
          }, \
          "params": { \
            "doc_config": { \
              "docid_prefix_delim": "", \
              "docid_regexp": "", \
              "mode": "scope.collection.type_field", \
              "type_field": "type" \
            }, \
            "mapping": { \
              "analysis": {}, \
              "default_analyzer": "standard", \
              "default_datetime_parser": "dateTimeOptional", \
              "default_field": "_all", \
              "default_mapping": { \
                "dynamic": true, \
                "enabled": false \
              }, \
              "default_type": "_default", \
              "docvalues_dynamic": false, \
              "index_dynamic": true, \
              "store_dynamic": false, \
              "type_field": "_type", \
              "types": { \
                "%s.%s": { \
                  "dynamic": true, \
                  "enabled": true \
                } \
              } \
            }, \
            "store": { \
              "indexType": "scorch", \
              "segmentVersion": 15 \
            } \
          }, \
          "sourceParams": {} \
        }'

        self.gsi_index_name_template = "%s_%s_%s_%d"
        self.gsi_create_template = "CREATE PRIMARY INDEX `%s` " \
                                   "ON `%s`.`%s`.`%s` USING GSI " \
                                   "WITH {\"num_replica\": %d}"
        self.gsi_drop_template = "DROP INDEX `%s`.`%s` USING GSI"

        self.op_create = "key_create"
        self.op_remove = "key_delete"

        self.ts_during_start = self.__get_current_timestamps_from_debug_log()
        self.initial_tombstones = \
            self.cluster_util.get_metakv_dicts(self.cluster.master)
        self.log.info(self.ts_during_start)

    def tearDown(self):
        self.log_setup_status("ConfigPurging", "start", "tearDown")
        self.log.info("Resetting purger age value to default")
        rest = RestConnection(self.cluster.master)
        rest.run_tombstone_purger(self.default_purge_age)
        rest.enable_tombstone_purger()
        self.log_setup_status("ConfigPurging", "complete", "tearDown")
        super(ConfigPurging, self).tearDown()

    def __perform_meta_kv_op_on_rand_key(self, op_type, key_name, value=None):
        self.log.info("%s meta_kv key '%s'" % (op_type, key_name))
        if op_type == self.op_create:
            self.cluster_util.create_metakv_key(self.cluster.master,
                                                key_name, value)
        elif op_type == self.op_remove:
            self.cluster_util.delete_metakv_key(self.cluster.master, key_name)
        else:
            self.fail("Invalid operation: %s" % op_type)

    def __get_deleted_key_count(self, check_if_zero=False):
        deleted_keys = self.cluster_util.get_ns_config_deleted_keys_count(self.cluster)
        tbl = TableView(self.log.info)
        tbl.set_headers(["Node", "Deleted_key_count"])
        for t_ip, k_count in deleted_keys.items():
            tbl.add_row(["%s" % t_ip, "%s" % k_count])
        tbl.display("Tombstone count on cluster nodes:")

        if not check_if_zero:
            return
        for t_ip, k_count in deleted_keys.items():
            if k_count != 0:
                self.fail("%s Deleted key count %s != 0" % (t_ip, k_count))

    def __fts_index(self, op_type, index_name, bucket, scope, collection):
        self.log.info("%s fts index %s" % (op_type, index_name))
        if op_type == self.op_create:
            template = self.fts_param_template % (
                index_name, bucket.name, bucket.uuid,
                self.fts_index_partition, scope, collection)
            self.fts_helper.create_fts_index_from_json(index_name, template)
        elif op_type == self.op_remove:
            self.fts_helper.delete_fts_index(index_name)

    def __gsi_index(self, op_type, b_name, s_name, c_name, index_counter):
        query = None
        index_name = "%s_%s_%s_%d" % (b_name, s_name, c_name, index_counter)
        self.log.info("GSI %s: %s" % (op_type, index_name))
        if op_type == self.op_create:
            query = self.gsi_create_template % (index_name,
                                                b_name, s_name, c_name,
                                                self.index_replicas)
        elif op_type == self.op_remove:
            query = self.gsi_drop_template % (b_name, index_name)

        try:
            self.n1ql_helper.run_cbq_query(query)
        except Exception as e:
            self.log.critical(e)
        except Java_base_exception as e:
            self.log.critical(e)

    def __get_purged_tombstone_from_last_run(self, nodes=None):
        """
        :return last_purged_tombstones: Dict of format,
            { node_ip: {'count': N, 'keys': [k1, k2, ..] }, ...}
        """
        tail_cmd = "cat %s/var/lib/couchbase/logs/debug.log " \
                   % self.couchbase_base_dir \
                   + "| sed -n '/%s/,$p'"
        purged_ts_count_pattern = ".*Purged ([0-9]+) ns_config tombstone"
        meta_kv_keys_pattern = ".*{metakv,[ ]*<<\"/([0-9a-zA-Z_\-\.]+)\">>"
        start_of_line = "^\[ns_server:"

        start_of_line = re.compile(start_of_line)
        meta_kv_keys_pattern = re.compile(meta_kv_keys_pattern)
        purged_ts_count_pattern = re.compile(purged_ts_count_pattern)
        tbl_view = TableView(self.log.info)
        tbl_view.set_headers(["Node", "Purged Keys"])

        last_purged_tombstones = dict()
        if nodes is None:
            nodes = self.cluster_util.get_nodes_in_cluster(self.cluster)
        for node in nodes:
            self.log.info("Processing debug logs from %s" % node.ip)
            shell = RemoteMachineShellConnection(node)
            output, _ = shell.execute_command(
                tail_cmd % self.ts_during_start[node.ip])
            if not output:
                output, _ = shell.execute_command(
                    " ".join(tail_cmd.split(' ')[:2]))
            self.log.debug("Tail stdout:\n%s" % output)
            o_len = len(output)
            target_buffer = ""
            total_ts_purged = 0
            for index in range(o_len-1, -1, -1):
                line = output[index]
                if not start_of_line.match(line):
                    target_buffer = line + target_buffer
                elif "tombstone_agent:purge:" in line:
                    total_ts_purged = \
                        purged_ts_count_pattern.match(line).group(1)
                    break
                else:
                    target_buffer = ""

            last_purged_tombstones[node.ip] = dict()
            last_purged_tombstones[node.ip]["count"] = int(total_ts_purged)
            last_purged_tombstones[node.ip]["keys"] = \
                meta_kv_keys_pattern.findall(target_buffer)
            tbl_view.add_row([node.ip, total_ts_purged])
            shell.disconnect()

        tbl_view.display("Purged_keys:")
        self.log.debug("Purged keys :: %s" % last_purged_tombstones)
        return last_purged_tombstones

    def __get_current_timestamps_from_debug_log(self):
        cmd = "cat %s/var/lib/couchbase/logs/debug.log " \
              "| grep '\[ns_server:' | tail -1" % self.couchbase_base_dir
        timestamp_pattern = "\[ns_server:[a-zA-Z]+,([T0-9-:.]+),"
        timestamp_pattern = re.compile(timestamp_pattern)
        ts_result = dict()
        for server in self.cluster.servers:
            shell = RemoteMachineShellConnection(server)
            output, _ = shell.execute_command(cmd)
            if type(output) is list:
                output = output[0]
            ts_result[server.ip] = timestamp_pattern.match(output).group(1)
            shell.disconnect()
        return ts_result

    def test_meta_kv_resource_usage(self):
        """
        1. Records initial memory & disk usage stats
        2. Create huge number of meta_kv keys
        3. Records memory disk usage post creation
        4. Remove created keys to create tombstones
        5. Record memory & disk stats (with tombstones present)
        6. Wait for purger to cleanup the tombstones
        7. Validate the memory and disk space to reclaimed properly
        """
        def get_node_stats():
            for t_ip, t_conn in shell_conn.items():
                self.log.info("Node: %s" % t_ip)
                for t_cmd in commands_to_run:
                    self.log.info(
                        "".join(t_conn.execute_command(t_cmd)[0]))

        fts_generic_name = "fts_%s" % int(self.time_stamp) + "-%d"
        commands_to_run = [
            "free -h",
            "ls -lh %s/var/lib/couchbase/config/config.dat "
            "| awk '{print $5,$9}'" % self.couchbase_base_dir]

        rest = RestConnection(self.cluster.master)

        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()
        rest.run_tombstone_purger(1)

        shell_conn = dict()
        # Open required ssh connections
        for node in self.cluster.nodes_in_cluster:
            shell_conn[node.ip] = RemoteMachineShellConnection(node)

        self.log.info("Collecting initial cluster stats")
        self.__get_deleted_key_count(check_if_zero=True)
        get_node_stats()

        # Create required indexes
        self.log.info("Creating %s indexes" % self.num_index)
        for index in range(self.num_index):
            key = fts_generic_name % index
            if self.fts_helper is None:
                self.__perform_meta_kv_op_on_rand_key(self.op_create, key,
                                                      "dummy_val")
            else:
                self.__fts_index(self.op_create, key,
                                 self.cluster.buckets[0],
                                 CbServer.default_scope,
                                 CbServer.default_collection)
        self.log.info("Done creating indexes")

        self.log.info("Stats after index creation")
        get_node_stats()

        # Remove indexes to create tombstones
        self.log.info("Removing keys to create tombstones")
        for index in range(self.num_index):
            key = fts_generic_name % index
            if self.fts_helper is None:
                self.__perform_meta_kv_op_on_rand_key(self.op_create, key)
            else:
                self.__fts_index(self.op_remove, key,
                                 self.cluster.buckets[0],
                                 CbServer.default_scope,
                                 CbServer.default_collection)
        self.log.info("Done creating tombstones")

        if self.fts_helper is not None:
            self.sleep(30, "Wait for FTS delete to complete")

        self.log.info("Stats with active tombstones")
        self.__get_deleted_key_count()
        get_node_stats()

        self.log.info("Triggering tombstone purger")
        rest.run_tombstone_purger(1)
        self.sleep(10, "Wait for purger run to complete")

        self.log.info("Purged tombstone info:")
        self.log.info(self.__get_purged_tombstone_from_last_run())

        self.log.info("Post purge cluster stats")
        self.__get_deleted_key_count(check_if_zero=True)
        get_node_stats()

        # Close all ssh connections
        for _, conn in shell_conn.items():
            conn.disconnect()

    def test_meta_kv_key_purging(self):
        """
        1. Create meta_kv key entries for 'self.default_purge_age' seconds
        2. Wait for 'purge_time_interval' time and check the keys have been
           purged from the meta_kv or not
        """
        # Can take values custom / fts/ 2i
        i_count = 0
        created_keys = list()
        generic_name = "meta_kv_key-%s-%s"
        start_time = int(time())
        end_time = start_time + self.default_run_interval
        key_type = self.input.param("index_type", "custom")

        self.log.info("Creating meta_kv tombstones for %d seconds"
                      % self.default_run_interval)
        while int(time()) < end_time:
            index_name = generic_name \
                % (time(), str(i_count).zfill(self.index_name_len))
            if key_type == "custom":
                self.__perform_meta_kv_op_on_rand_key(
                    self.op_create, index_name, "value_dummy")
                self.__perform_meta_kv_op_on_rand_key(
                    self.op_remove, index_name)
            elif key_type == CbServer.Services.FTS:
                self.__fts_index(self.op_create, index_name,
                                 self.cluster.buckets[0],
                                 CbServer.default_scope,
                                 CbServer.default_collection)
                self.__fts_index(self.op_remove, index_name,
                                 self.cluster.buckets[0],
                                 CbServer.default_scope,
                                 CbServer.default_collection)
            created_keys.append(index_name)
            i_count += 1
        self.log.info("Created %d meta_kv_tombstones" % i_count)
        self.sleep(self.default_purge_age - self.default_run_interval,
                   "Wait for purging to run")

        # Validating purged keys with default settings
        total_keys_purged = dict()
        for iter in range(3):
            if iter == 0:
                #ignoring the first internal purge
                self.sleep(self.default_run_interval + 15,
                           "Wait for next purger run")
                continue
            ts_data = self.__get_purged_tombstone_from_last_run()
            for node_ip, purge_data in ts_data.items():
                if node_ip not in total_keys_purged:
                    total_keys_purged[node_ip] = 0
                total_keys_purged[node_ip] += purge_data["count"]
            self.sleep(self.default_run_interval + 15,
                       "Wait for next purger run")

        for node_ip, purged_count in total_keys_purged.items():
            if purged_count != i_count:
                self.fail("%s - Expected %s to be purged. Actual %s purged"
                          % (node_ip, i_count, purged_count))

    def test_create_meta_kv_keys_max_len(self):
        """
        Create meta_kv tombstones with huge key_size and validate purging
        """
        rest = RestConnection(self.cluster.master)
        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()

        self.log.info("Creating %s indexes with length %s"
                      % (self.num_index, self.index_name_len))
        t_fts_name = "fts_%s" % int(self.time_stamp) \
                     + "-%0" + str(self.index_name_len) + "d"
        # Create req. number of index
        for index in range(self.num_index):
            fts_name = t_fts_name % index
            self.__fts_index(self.op_create, fts_name,
                             self.cluster.buckets[0],
                             CbServer.default_scope,
                             CbServer.default_collection)

        self.log.info("Deleting indexes to create tombstones")
        # Delete created indexes
        for index in range(self.num_index):
            fts_name = t_fts_name % index
            self.__fts_index(self.op_remove, fts_name,
                             self.cluster.buckets[0],
                             CbServer.default_scope,
                             CbServer.default_collection)

        deleted_keys = self.cluster_util.get_ns_config_deleted_keys_count(self.cluster)
        self.sleep(60, "Wait before triggering purger")
        rest.run_tombstone_purger(50)
        self.sleep(10, "Wait for purger to cleanup tombstones")

        self.log.info("Validating purged keys on each node")
        purged_ts_dict = self.__get_purged_tombstone_from_last_run()
        for node_ip, purged_data in purged_ts_dict.items():
            if purged_data["count"] != deleted_keys[node_ip]:
                self.log_failure("%s - keys deleted %s != %s purged count"
                                 % (node_ip, deleted_keys[node_ip],
                                    purged_data["count"]))

        self.validate_test_failure()

    def test_create_similar_tombstones_across_each_node(self):
        """
        1. Create meta_kv tombstone independently on each cluster node
        2. Make sure purger cleans up the proper key without any conflict
           across the nodes
        """

        rest = RestConnection(self.cluster.master)
        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()

        rest_objects = dict()
        for node in self.cluster.nodes_in_cluster:
            rest_objects[node.ip] = RestConnection(node)

        t_key = "meta_kv_key-%d"
        value = "test_create_similar_tombstones_across_each_node"

        # Create tombstones with custom key on each cluster node
        self.log.info("Creating %s tombstones" % self.num_index)
        for index in range(self.num_index):
            meta_kv_key = t_key % index
            for _, rest_obj in rest_objects.items():
                rest_obj.create_metakv_key(meta_kv_key, value)
                rest_obj.delete_metakv_key(meta_kv_key)

        self.sleep(15, "Waiting to trigger purger with tombstone age > 15secs")
        rest.run_tombstone_purger(15)
        self.sleep(30, "Waiting for purger to run")

        # Validate tombstone purging
        self.log.info("Validating purged keys on each node")
        purged_keys_dict = self.__get_purged_tombstone_from_last_run()
        for node_ip, purged_data in purged_keys_dict.items():
            for index in range(self.num_index):
                meta_kv_key = t_key % index
                if meta_kv_key not in purged_data['keys']:
                    self.log_failure(
                        "%s :: tombstone %s missing in purged keys. "
                        "Purged keys -> %s"
                        % (node_ip, meta_kv_key, purged_data['keys']))

        self.validate_test_failure()

    def test_meta_kv_purger_timer(self):
        """
        1. Create a meta_kv key entry and delete the key
        2. Wait for few minutes
        3. Create new tombstone entry with new_key
        4. Wait till purger run
        5. Validate whether purger has removed only key
           from step#1 and key from step#3 is retained
        """

        t_key = "meta_kv_key-%s" % self.time_stamp + "-%s"
        value = "dummy_value"
        max_ts_to_create = \
            int(self.default_purge_age/self.default_run_interval)
        last_key_index = max_ts_to_create - 1

        rest = RestConnection(self.cluster.master)
        # Remove any old tombstones prior to the test
        rest.run_tombstone_purger(1)
        # Reset default purger run values
        rest.update_tombstone_purger_run_interval(self.default_run_interval)
        rest.update_tombstone_purge_age_for_removal(self.default_purge_age)
        rest.disable_tombstone_purger()

        # Create tombstones
        for index in range(max_ts_to_create):
            key = t_key % index
            self.log.info("Creating tombstone with key '%s'" % key)
            self.cluster_util.create_metakv_key(self.cluster.master,
                                                key, value)
            self.cluster_util.delete_metakv_key(self.cluster.master, key)
            if index != last_key_index:
                self.sleep(self.default_run_interval,
                           "Wait before creating next tombstone")

        self.log.info("Enabling ts-purger with default settings")
        rest.enable_tombstone_purger()
        rest.update_tombstone_purger_run_interval(self.default_run_interval)
        rest.update_tombstone_purge_age_for_removal(self.default_purge_age)

        self.sleep(40, "Wait for purger to start running")

        # Trigger purger with default values to validate the purging keys
        for index in range(max_ts_to_create):
            # Reset timestamp to track since the purger ran now
            self.ts_during_start = \
                self.__get_current_timestamps_from_debug_log()
            self.log.info("New timestamp reference: %s" % self.ts_during_start)

            key = t_key % index
            self.sleep(self.default_run_interval, "Wait for purger to run")
            self.log.info("Validating purger for key '%s'" % key)
            purged_ts = self.__get_purged_tombstone_from_last_run()
            for node_ip, purged_data in purged_ts.items():
                if not purged_data['keys']:
                    self.fail("%s No tombstone purged" % node_ip)
                elif purged_data['count'] > 1:
                    self.fail("%s - multiple meta_kv tombstones purged: %s"
                              % (node_ip, purged_data['keys']))
                elif purged_data['keys'][0] != key:
                    self.fail("%s - Tombstone '%s' missing in purger log"
                              % (node_ip, key))

    def test_create_remove_same_meta_kv_key(self):
        """
        1. Create and remove same key across the entire purger cycle
        2. Make sure no key is removed during the purger run
        """
        key = "meta_kv_key-%d" % self.time_stamp
        value = "dummy_value"
        max_ts_to_create = \
            int(self.default_purge_age/self.default_run_interval)
        last_key_index = max_ts_to_create - 1

        rest = RestConnection(self.cluster.master)
        # Remove any old tombstones prior to the test
        rest.run_tombstone_purger(1)
        # Reset default purger run values
        rest.update_tombstone_purger_run_interval(self.default_run_interval)
        rest.update_tombstone_purge_age_for_removal(self.default_purge_age)
        rest.disable_tombstone_purger()

        # Create tombstones
        for index in range(max_ts_to_create):
            self.log.info("Creating tombstone with key '%s'" % key)
            self.cluster_util.create_metakv_key(self.cluster.master,
                                                key, value)
            self.cluster_util.delete_metakv_key(self.cluster.master, key)
            if index != last_key_index:
                self.sleep(self.default_run_interval,
                           "Wait before creating next tombstone")

        self.log.info("Enabling ts-purger with default settings")
        rest.enable_tombstone_purger()
        rest.update_tombstone_purger_run_interval(self.default_run_interval)
        rest.update_tombstone_purge_age_for_removal(self.default_purge_age)
        self.sleep(20, "Wait for purger to start running")

        # Trigger purger with default values to validate the purging keys
        for index in range(max_ts_to_create):
            self.sleep(self.default_run_interval,
                       "Wait before validating for key %s" % key)
            purged_ts = self.__get_purged_tombstone_from_last_run()
            for node_ip, purged_data in purged_ts.items():
                if index == last_key_index:
                    if purged_data['count'] == 0:
                        self.fail("%s - No tombstones purged" % node_ip)
                    elif purged_data['count'] != 1:
                        self.fail("%s - multiple meta_kv tombstones purged: %s"
                                  % (node_ip, purged_data['keys']))
                    elif purged_data['keys'][0] != key:
                        self.fail("%s - Tombstone '%s' missing in purger log"
                                  % (node_ip, key))
                else:
                    if len(purged_data['keys']) != 0:
                        self.fail("%s - Some meta_kv key(s) purged: %s"
                                  % (node_ip, purged_data['keys']))

    def test_fail_node_during_purge_run(self):
        """
        1. Create and remove meta_kv key(s)
        2. Hard failover a random node (non-orchestrator)
           such that the purger gets triggered as expected
        3. Let the purger run complete
        """
        t_key = "fts_index-%s" % int(self.time_stamp) + "-%s"
        rest = RestConnection(self.cluster.master)
        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()

        self.log.info("Creating fts_index tombstones")
        for index in range(self.num_index):
            key = t_key % index
            self.__fts_index(self.op_create, key,
                             self.cluster.buckets[0],
                             CbServer.default_scope,
                             CbServer.default_collection)
            self.__fts_index(self.op_remove, key,
                             self.cluster.buckets[0],
                             CbServer.default_scope,
                             CbServer.default_collection)

        deleted_keys = self.cluster_util.get_ns_config_deleted_keys_count(self.cluster)
        del_key_count = None
        for node_ip, curr_count in deleted_keys.items():
            if del_key_count is None:
                del_key_count = curr_count
            elif del_key_count != curr_count:
                self.fail(
                    "%s - Deleted keys count mismatch. Expected %s, got %s"
                    % (node_ip, del_key_count, curr_count))

        random_node = choice(self.cluster.servers[1:])
        shell = RemoteMachineShellConnection(random_node)
        cb_err = CouchbaseError(self.log,
                                shell,
                                node=random_node)

        if choice([True, False]):
            cb_err.create(cb_err.KILL_BEAMSMP)
        else:
            self.log.info("Stopping couchbase server gracefully on %s"
                          % random_node.ip)
            shell.stop_couchbase()

        self.sleep(15, "Wait before triggering purger")
        self.log.info("Triggering purger when a node is in failed state")
        rest.run_tombstone_purger(10)
        purged_keys_dict = self.__get_purged_tombstone_from_last_run()
        for node_ip, purged_data in purged_keys_dict.items():
            if purged_data['count'] or purged_data['count'] != 0:
                shell.disconnect()
                self.fail("%s - Some keys got purged when node is failed: %s"
                          % (node_ip, purged_data))

        self.log.info("Healing node with server restart")
        shell.restart_couchbase()
        shell.disconnect()
        self.sleep(60, "Wait for node to come online")

        self.log.info("Triggering purger when a node is in failed state")
        rest.run_tombstone_purger(10)
        self.sleep(10, "Wait for purger to run")
        purged_keys_dict = self.__get_purged_tombstone_from_last_run()
        for node_ip, purged_data in purged_keys_dict.items():
            if purged_data['count'] == 0:
                self.fail("%s - No keys purged: %s" % (node_ip, purged_data))
            if purged_data['count'] != del_key_count:
                self.fail("%s - Purged key count mismatch. Expected %s, got %s"
                          % (node_ip, del_key_count, purged_data['count']))

    def test_fail_node_during_meta_key_delete(self):
        """
        1. Create a meta_kv key entry
        2. Stop any random node and trigger a meta_kv key delete operation
        3. Recover back the node and make sure the purging goes through fine
        """
        def heal_the_node(shell):
            self.log.info("Healing the cluster node")
            shell.restart_couchbase()
            shell.disconnect()
            self.sleep(60, "Wait for node to come online")

        custom_meta_kv_key = "key_01_%s" % self.time_stamp
        fts_key = "fts_index_%s" % int(self.time_stamp)
        rest = RestConnection(self.cluster.master)

        random_node = choice(self.cluster.servers[1:self.nodes_init])
        if random_node.ip == self.cluster.fts_nodes[0].ip:
            self.fts_helper = FtsHelper(self.cluster.fts_nodes[1])

        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()

        # Creating meta_kv keys
        self.__fts_index(self.op_create, fts_key,
                         self.cluster.buckets[0],
                         CbServer.default_scope,
                         CbServer.default_collection)
        self.__perform_meta_kv_op_on_rand_key(self.op_create,
                                              custom_meta_kv_key, "value")

        shell = RemoteMachineShellConnection(random_node)
        cb_err = CouchbaseError(self.log,
                                shell,
                                node=random_node)

        self.sleep(5, "Waiting for meta_kv sync to happen")

        # Stopping ns_server on random node
        if choice([True, False]):
            self.log.info("Stopping couchbase server gracefully on %s"
                          % random_node.ip)
            shell.stop_couchbase()
        else:
            cb_err.create(cb_err.KILL_BEAMSMP)

        # Removing meta_kv keys when one of the node is down
        self.__fts_index(self.op_remove, fts_key,
                         self.cluster.buckets[0],
                         CbServer.default_scope,
                         CbServer.default_collection)
        self.__perform_meta_kv_op_on_rand_key(self.op_remove,
                                              custom_meta_kv_key)

        self.sleep(15, "Wait before triggering purger")
        rest.run_tombstone_purger(10)

        purged_keys_dict = self.__get_purged_tombstone_from_last_run()
        for node_ip, purged_data in purged_keys_dict.items():
            if purged_data['count'] or purged_data['count'] != 0:
                heal_the_node(shell)
                self.fail("%s - Some keys got purged when node is failed: %s"
                          % (node_ip, purged_data))

        # Recover from the error condition
        heal_the_node(shell)

        deleted_keys = self.cluster_util.get_ns_config_deleted_keys_count(self.cluster)
        self.log.info(deleted_keys)
        del_key_count = None
        for node_ip, curr_count in deleted_keys.items():
            if del_key_count is None:
                del_key_count = curr_count
            elif del_key_count != curr_count:
                self.fail(
                    "%s - Deleted keys count mismatch. Expected %s, got %s"
                    % (node_ip, del_key_count, curr_count))

        rest.run_tombstone_purger(10)
        retry_count = 10
        # Validate the key has been deleted from meta_kv
        for i in range(retry_count):
            try:
                purged_keys_dict = self.__get_purged_tombstone_from_last_run()
                for node_ip, purged_data in purged_keys_dict.items():
                    if purged_data['count'] == 0:
                        self.fail(
                            "%s - No keys purged: %s" % (node_ip, purged_data))
                    if purged_data['count'] != del_key_count:
                        self.fail(
                            "%s - Purged key count mismatch. Expected %s, got %s"
                            % (node_ip, del_key_count, purged_data['count']))
                    if custom_meta_kv_key not in purged_data['keys']:
                        self.fail("%s - Key %s missing in purger: %s"
                                  % (node_ip, custom_meta_kv_key,
                                     purged_data['keys']))
                break
            except Exception as e:
                if i == retry_count-1:
                    raise e
                else:
                    self.sleep(1, "waiting 1 sec before retry")

    def test_node_add_back(self):
        """
        1. Create tombstones in meta_kv
        2. Rebalance random_node out the the cluster
        3. Add back the node within/after meta_kv purger interval
        4. Validate purging happening for prev. deleted keys
        """
        def run_purger_and_validate_purged_key(check_key="exists"):
            self.__get_deleted_key_count()
            self.log.info("Triggering meta_kv purger")
            rest.run_tombstone_purger(10)
            self.sleep(10, "Wait for purger to complete")

            # Validate purger runs on targeted node
            purged_keys_dict = self.__get_purged_tombstone_from_last_run()
            self.log.info(purged_keys_dict)
            for t_ip, purged_data in purged_keys_dict.items():
                if check_key == "exists" \
                        and custom_meta_kv_key not in purged_data['keys']:
                    self.fail("%s - keys not present as expected: %s"
                              % (t_ip, purged_data['keys']))
                elif check_key == "not_exists" \
                        and custom_meta_kv_key in purged_data['keys']:
                    self.fail("%s - keys purged again: %s"
                              % (t_ip, purged_data['keys']))

            self.__get_deleted_key_count(check_if_zero=True)

            # Reset timestamp to track since the purger ran now
            self.ts_during_start = \
                self.__get_current_timestamps_from_debug_log()
            self.log.info("New timestamp reference: %s" % self.ts_during_start)

        rest = RestConnection(self.cluster.master)
        custom_meta_kv_key = "metakv_key_%s" % int(self.time_stamp)
        fts_key = "fts_index_%s" % int(self.time_stamp)
        target_node_type = \
            self.input.param("target_node", CbServer.Services.KV)
        remove_node_method = \
            self.input.param("removal_method", "rebalance_out")
        add_back_node_timing = self.input.param("add_back_node",
                                                "within_purge_interval")

        self.log.info("Stopping meta_kv purger autorun")
        rest.disable_tombstone_purger()

        # Creating meta_kv keys
        self.__fts_index(self.op_create, fts_key,
                         self.cluster.buckets[0],
                         CbServer.default_scope,
                         CbServer.default_collection)
        self.__perform_meta_kv_op_on_rand_key(self.op_create,
                                              custom_meta_kv_key, "value")

        # Remove meta_kv keys for tombstone creation
        self.__fts_index(self.op_remove, fts_key,
                         self.cluster.buckets[0],
                         CbServer.default_scope,
                         CbServer.default_collection)
        self.__perform_meta_kv_op_on_rand_key(self.op_remove,
                                              custom_meta_kv_key)

        if target_node_type == CbServer.Services.KV:
            target_node = choice(self.cluster.kv_nodes)
        elif target_node_type == CbServer.Services.FTS:
            target_node = choice(self.cluster.fts_nodes)
        else:
            target_node = choice(self.cluster.nodes_in_cluster)

        self.__get_deleted_key_count()

        self.log.info("Removing node %s using %s"
                      % (target_node.ip, remove_node_method))

        # Remove the target node
        if remove_node_method == "rebalance_out":
            rebalance_task = self.task.async_rebalance(
                self.cluster, [], [target_node])
        elif remove_node_method == "swap_rebalance":
            rebalance_task = self.task.async_rebalance(
                self.cluster,
                [self.cluster.servers[self.nodes_init]], [target_node])
        elif remove_node_method == "failover":
            rebalance_task = self.task.async_failover(
                self.cluster.servers[0:self.nodes_init], [target_node],
                graceful=True)
        else:
            self.fail("Invalid remove method '%s'" % remove_node_method)

        self.task_manager.get_task_result(rebalance_task)
        self.assertTrue(rebalance_task.result, "Rebalance failed")
        self.sleep(15, "Wait after rebalance")

        self.__get_deleted_key_count()

        # Update new master node
        if target_node.ip == self.cluster.master.ip:
            self.cluster_util.find_orchestrator(self.cluster,
                                                self.cluster.servers[1])

        self.log.info("Current master node: %s" % self.cluster.master.ip)

        if add_back_node_timing == "before_purge_interval":
            run_purger_and_validate_purged_key(check_key="exists")

        # Add back the removed node
        self.log.info("Adding back node %s" % target_node.ip)
        if remove_node_method in ["rebalance_out", "swap_rebalance"]:
            self.cluster_util.add_node(self.cluster, target_node,
                                       [CbServer.Services.KV])
        elif remove_node_method == "failover":
            rest.set_recovery_type(otpNode="ns_1@"+target_node.ip,
                                   recoveryType=self.recovery_type)
            rebalance_result = self.task.rebalance(
                self.cluster_util.get_nodes_in_cluster(self.cluster), [], [])
            self.assertTrue(rebalance_result, "Add back rebalance failed")

        if add_back_node_timing == "within_purge_interval":
            run_purger_and_validate_purged_key(check_key="exists")
        elif add_back_node_timing == "before_purge_interval":
            run_purger_and_validate_purged_key(check_key="not_exists")

    def test_fts_2i_purging_with_cluster_ops(self):
        """
        MB-43291
        1. Cluster with 3 kv, 2 index+n1ql, 4 search nodes
        2. 6 buckets with 5000 docs each
        3. Built 200 GSI indexes with replica 1 (50 indexes on each bucket)
        4. Created 30 fts custom indexes (10 indexes on 3 buckets),
           just to add more entries to meta-kv
        5. Create and Drop 100 gsi indexes sequentially on 4 buckets
           (so this would be adding more entries of create/drop of 400 indexes)
        6. Create and drop 50 fts indexes on 3 buckets.
        7. Graceful Failover a node with kv service.
        8. Failover took 1hr 30 minutes

        Note: Test requires one spare node for performing swap/reb_in tests
        """

        self.num_gsi_index = self.input.param("num_gsi_index", 50)
        self.num_fts_index = self.input.param("num_fts_index", 10)
        self.fts_indexes_to_create_drop = \
            self.input.param("fts_indexes_to_create_drop", 50)
        self.gsi_indexes_to_create_drop = \
            self.input.param("gsi_index_to_create_drop", 200)

        self.num_nodes_to_run = self.input.param("num_nodes_to_run", 1)
        self.target_service_nodes = \
            self.input.param("target_nodes", "kv").split(";")
        self.cluster_actions = \
            self.input.param("cluster_action", "rebalance_in").split(";")
        self.recovery_type = self.input.param(
            "recovery_type", CbServer.Failover.RecoveryType.DELTA)

        fts_generic_name = "fts_%s" % int(self.time_stamp) + "_%s_%s_%s_%d"

        rest = RestConnection(self.cluster.master)
        # Set RAM quota and Index storage mode
        rest.set_service_mem_quota(
            {CbServer.Settings.KV_MEM_QUOTA: 1024,
             CbServer.Settings.INDEX_MEM_QUOTA: 4096,
             CbServer.Settings.FTS_MEM_QUOTA: 4096,
             CbServer.Settings.CBAS_MEM_QUOTA: 1024,
             CbServer.Settings.EVENTING_MEM_QUOTA: 256})
        rest.set_indexer_storage_mode(storageMode="plasma")

        # Open SDK for connection for running n1ql queries
        self.client = self.sdk_client_pool.get_client_for_bucket(
            self.cluster.buckets[0])

        # Create required GSI indexes
        for bucket in self.cluster.buckets[:2]:
            for _, scope in bucket.scopes.items():
                for c_name, _ in scope.collections.items():
                    for index in range(0, self.num_gsi_index):
                        self.__gsi_index(self.op_create, bucket.name,
                                         scope.name, c_name, index)
            self.log.info("Done creating GSI indexes for %s" % bucket.name)

        # Create required FTS indexes
        bucket = self.cluster.buckets[0]
        for _, scope in bucket.scopes.items():
            for c_name, _ in scope.collections.items():
                for index in range(0, self.num_fts_index):
                    fts_name = fts_generic_name % (bucket.name, scope.name,
                                                   c_name, index)
                    self.__fts_index(self.op_create, fts_name,
                                     bucket, scope.name, c_name)
        self.log.info("Done creating FTS indexes for %s" % bucket.name)

        # Create GSI tombstones
        for bucket in self.cluster.buckets[:2]:
            self.log.info("Create and drop %s GSI indexes on %s"
                          % (self.gsi_indexes_to_create_drop, bucket.name))
            for _, scope in bucket.scopes.items():
                for c_name, _ in scope.collections.items():
                    for index in range(self.num_gsi_index,
                                       self.num_gsi_index
                                       + self.gsi_indexes_to_create_drop):
                        self.__gsi_index(self.op_create, bucket.name,
                                         scope.name, c_name, index)
                        self.__gsi_index(self.op_remove, bucket.name,
                                         scope.name, c_name, index)

        # Create FTS tombstones
        bucket = self.cluster.buckets[0]
        for _, scope in bucket.scopes.items():
            for c_name, _ in scope.collections.items():
                for index in range(self.num_fts_index,
                                   self.num_fts_index
                                   + self.fts_indexes_to_create_drop):
                    fts_name = fts_generic_name % (bucket.name, scope.name,
                                                   c_name, index)
                    self.__fts_index(self.op_create, fts_name,
                                     bucket, scope.name, c_name)
                    self.__fts_index(self.op_remove, fts_name,
                                     bucket, scope.name, c_name)

        # Release the SDK client
        self.sdk_client_pool.release_client(self.client)

        nodes_involved = list()
        for service in self.target_service_nodes:
            num_nodes_run = 0
            if service == CbServer.Services.KV:
                nodes_to_play = self.cluster.kv_nodes
            elif service == CbServer.Services.INDEX:
                nodes_to_play = self.cluster.index_nodes
            elif service == CbServer.Services.N1QL:
                nodes_to_play = self.cluster.query_nodes
            elif service == CbServer.Services.INDEX:
                nodes_to_play = self.cluster.cbas_nodes
            elif service == CbServer.Services.FTS:
                nodes_to_play = self.cluster.fts_nodes
            elif service == CbServer.Services.EVENTING:
                nodes_to_play = self.cluster.eventing_nodes
            else:
                self.fail("Invalid service %s" % service)

            cluster_node = None
            for node in nodes_to_play:
                if node.ip in nodes_involved:
                    continue

                # Note affected nodes to avoid repeating action on same node
                # Happens when >1 service running on the node
                self.log.info("Master node: %s" % self.cluster.master.ip)
                nodes_involved.append(node.ip)

                for cluster_action in self.cluster_actions:
                    self.log.info("Performing '%s' on node %s"
                                  % (cluster_action, node.ip))
                    nodes_in_cluster = \
                        self.cluster_util.get_nodes_in_cluster(self.cluster)
                    if cluster_action == "rebalance_in":
                        rest = RestConnection(self.cluster.master)
                        self.task.rebalance(nodes_in_cluster, [node], [])
                        self.sleep(30, "Wait for rebalance to start")
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Rebalance_in failed")
                    elif cluster_action == "rebalance_out":
                        rest = RestConnection(self.cluster.master)
                        self.task.rebalance(nodes_in_cluster, [], [node])
                        self.sleep(30, "Wait for rebalance to start")
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Rebalance_out failed")
                        for t_node in nodes_in_cluster:
                            cluster_node = t_node
                            if cluster_node.ip != self.cluster.master.ip:
                                self.cluster_util.find_orchestrator(
                                    self.cluster, cluster_node)
                                break
                    elif cluster_action == "swap_rebalance":
                        rest = RestConnection(self.cluster.master)
                        self.task.rebalance(
                            nodes_in_cluster, [self.spare_node], [node],
                            services=None,
                            check_vbucket_shuffling=False)
                        self.sleep(30, "Wait for rebalance to start")
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Swap_rebalance failed")
                        if node.ip == self.cluster.master.ip:
                            self.cluster.master = self.spare_node
                            self.cluster_util.find_orchestrator(
                                self.cluster, self.spare_node)
                        self.spare_node = node
                    elif cluster_action == "graceful_failover":
                        rest = None
                        for t_node in nodes_in_cluster:
                            if t_node.ip != node.ip:
                                rest = RestConnection(t_node)
                                cluster_node = t_node
                                break
                        rest.fail_over("ns_1@" + node.ip, graceful=True)
                        self.sleep(10, "Wait for failover to start")
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Failover failed for node %s" % node.ip)
                    elif cluster_action == "rebalance_failover_node":
                        current_nodes = list()
                        for t_node in nodes_in_cluster:
                            if t_node.ip != node.ip:
                                current_nodes.append(t_node)

                        # Rebalance_out failed-over node
                        rest = RestConnection(self.cluster.master)
                        self.task.rebalance(current_nodes, [], [])
                        self.sleep(10, "Wait after cluster rebalance")
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Rebalance failed with failover node %s" % node.ip)
                        self.cluster_util.find_orchestrator(self.cluster,
                                                            cluster_node)
                    elif cluster_action == "add_back_failover_node":
                        rest = RestConnection(self.cluster.master)
                        rest.set_recovery_type("ns_1@" + node.ip,
                                               self.recovery_type)
                        self.task.rebalance(nodes_in_cluster, [], [])
                        self.sleep(30)
                        self.assertTrue(
                            rest.monitorRebalance(stop_if_loop=True),
                            "Rebalance failed with failover node %s" % node.ip)
                        self.cluster_util.find_orchestrator(self.cluster,
                                                            cluster_node)

                # Break if max nodes to run has reached per service
                num_nodes_run += 1
                if num_nodes_run >= self.num_nodes_to_run:
                    break


'''**********************************************************************************
You are given a string s. Reorder the string using the following algorithm:

Pick the smallest character from s and append it to the result.
Pick the smallest character from s which is greater than the last appended character to the result and append it.
Repeat step 2 until you cannot pick more characters.
Pick the largest character from s and append it to the result.
Pick the largest character from s which is smaller than the last appended character to the result and append it.
Repeat step 5 until you cannot pick more characters.
Repeat the steps from 1 to 6 until you pick all characters from s.
In each step, If the smallest or the largest character appears more than once you can choose any occurrence and append it to the result.

Return the result string after sorting s with this algorithm.

Example 1:

Input: s = "aaaabbbbcccc"
Output: "abccbaabccba"
Explanation: After steps 1, 2 and 3 of the first iteration, result = "abc"
After steps 4, 5 and 6 of the first iteration, result = "abccba"
First iteration is done. Now s = "aabbcc" and we go back to step 1
After steps 1, 2 and 3 of the second iteration, result = "abccbaabc"
After steps 4, 5 and 6 of the second iteration, result = "abccbaabccba"
**********************************************************************************'''

class Solution:
    def sortString(self, s: str) -> str:
        d = {}
        res = ""

        for i in s:
            if i not in d:
                d[i]=s.count(i)

        print(d)

        while sum(d.values())!=0:
            for i in sorted(d):
                if d[i]>0:
                    res+=i
                    d[i]-=1
            print(d)
            if sum(d.values())!=0:
                for i in sorted(d,reverse=True):
                    if d[i]>0:
                        res+=i
                        d[i]-=1
            else:
                break
        return res

Hash Table/Easy/1370. Increasing Decreasing String.py


import numpy as np
import time

# For time taking
timer = []
for _ in range(1000):
    start = float(round(time.time() * 1000, 5))
    np.set_printoptions(linewidth=np.inf)

    # Parsing data
    with open("input.txt") as data:
        data = data.readlines()

        coordinates = [int(i) for x in data for i in x.strip("\n").split(",") if 0 < len(i) < 10]
        coordinates = [[coordinates[i], coordinates[i+1]] for i in range(0,len(coordinates),2)]

        folds = [[x.strip("\n").split(" ")[-1].split("=")[0], int(x.strip("\n").split("=")[-1])] for x in data if len(x) > 13]

    # Making map
    max_x, max_y = max(x[0] for x in coordinates), max(x[1] for x in coordinates)
    a = np.zeros((max_y + 1, max_x + 1), dtype=int)

    for i in coordinates:
        a[i[1],i[0]] = 55

    # Folding
    def folder(times, fold):
        global a
        if fold == 'y':
            A = a[times+1:]
            cord = np.where(A==55)
            new_coordinates = ((abs(times - cord[0][i] - 1), cord[1][i]) for i in range(len(cord[0])))
            a = a[:times]
            for i in new_coordinates:
                a[i[0],i[1]] = 55
        else:
            B = a[:,times+1:]
            cord = np.where(B==55)
            new_coordinates = ((cord[0][i], abs(times - cord[1][i] - 1)) for i in range(len(cord[0])))
            a = a[:,:times]
            for i in new_coordinates:
                a[i[0],i[1]] = 55

        return np.count_nonzero(a == 55)

    results = [folder(fold[1], fold[0]) for fold in folds]

    end = float(round(time.time() * 1000, 5))
    timer.append(end-start)
    # print(f"{end - start} ms")

print(f"avg time from 1000 runs is: {round(sum(timer) / len(timer), 5)} ms")


from typing import Union

from persephone_client import Persephone

from src.domain.enums.persephone import PersephoneQueue, PersephoneSchema
from src.domain.enums.region import Region
from src.domain.exceptions.service.auditing_trail.model import FailToSaveAuditingTrail
from src.domain.models.account.bank.model import BankAccount
from src.domain.models.account.broker.model import BrokerAccount
from src.domain.models.device_info.dto import DeviceInfo
from src.domain.models.thebes_answer.model import ThebesAnswer
from src.domain.request.funding_and_withdrawal.validators import (
    UserMoneyFlowSameExchange,
    UserMoneyFlowDifferentExchange,
    UserMoneyFlowToExternalBank,
)
from nidavellir import Sindri

from src.infrastructures.env_config import config
from src.services.funding_and_withdraw.money_flow_resolvers import (
    TransferToExternalBank,
)
from src.transport.device_info.transport import DeviceSecurity

class FundingAndWithdrawalService:
    @classmethod
    async def withdrawal_to_external_bank(
        cls,
        money_flow: UserMoneyFlowToExternalBank,
        jwt: ThebesAnswer,
        origin_account: BrokerAccount,
        account_destination: BankAccount,
        device_info: DeviceInfo,
    ):

        await cls._log_in_persephone_to_audit(
            money_flow=money_flow,
            jwt=jwt,
            origin_account=origin_account,
            account_destination=account_destination,
            device_info=device_info,
        )

        transfer_to_external_bank = TransferToExternalBank(
            origin_account=origin_account,
            account_destination=account_destination,
            value=money_flow.value,
        )

        resume = await transfer_to_external_bank()

        return resume

    @staticmethod
    async def _log_in_persephone_to_audit(
        money_flow: UserMoneyFlowToExternalBank,
        jwt: ThebesAnswer,
        origin_account: BrokerAccount,
        account_destination: BankAccount,
        device_info: DeviceInfo,
    ):
        (
            sent_to_persephone,
            status_sent_to_persephone,
        ) = await Persephone.send_to_persephone(
            topic=config("PERSEPHONE_TOPIC_MONEY_MOVEMENT_TRANSFERENCE"),
            partition=PersephoneQueue.CASH_FLOW_WITHDRAWAL_TO_EXTERNAL_BANK.value,
            message={
                "unique_id": jwt.unique_id,
                "origin_account": await origin_account.resume(),
                "account_destination": await account_destination.resume(),
                "value": money_flow.value,
                "device_id": device_info.device_id,
                "device_info": device_info.decrypted_device_info,
            },
            schema_name=PersephoneSchema.CASH_FLOW_WITHDRAWAL_TO_EXTERNAL_BANK.value,
        )
        if sent_to_persephone is False:
            raise FailToSaveAuditingTrail("common.process_issue")

    @classmethod
    async def money_flow_between_user_dtvm_accounts(
        cls,
        money_flow: Union[UserMoneyFlowSameExchange, UserMoneyFlowDifferentExchange],
        jwt_data: dict,
    ):
        jwt = ThebesAnswer(jwt_data=jwt_data)
        jwt.account_br_is_blocked()
        unique_id = jwt.unique_id

        origin_account = BrokerAccount(
            **money_flow.origin_account.dict(),
            user_unique_id=unique_id,
        )
        account_destination = BrokerAccount(
            **money_flow.account_destination.dict(),
            user_unique_id=unique_id,
        )
        await origin_account.validate_accounts_ownership()

        await account_destination.validate_accounts_ownership()

        def money_flow_resolver_class(*args, **kwargs):
            raise Exception()
            # aplica logica aki de tranferian entre contas sinacor
            async def loro():
                pass

            return loro

        money_flow_resolver = money_flow_resolver_class(
            origin_account=origin_account,
            account_destination=account_destination,
            value=money_flow.value,
        )
        resume = await money_flow_resolver()
        Sindri.dict_to_primitive_types(resume)
        return resume


import sys
test_cases = int(input())
while test_cases > 0:
    test_cases -= 1
    m, n = map(int, input().split())
    a = []
    max_value = -sys.maxsize -1

    max_i = 0
    max_j = 0
    for i in range(m):
        tmp = [int(item) for item in input().split()]
        a.append(tmp)

    for i in range(m):
        for j in range(n):
            if a[i][j] > max_value:
                max_i = i
                max_j = j
                max_value = a[i][j]

    max_i += 1
    max_j += 1

    area = max_i * max_j
    area = max(area, max_j * (m - max_i + 1))
    area = max(area, (n - max_j + 1) * (m - max_i + 1))
    area = max(area, (n - max_j + 1) * max_i)
    print(area)


#encoding:utf-8
from colorama import Fore, Back, Style

def avg(first, *rest):
    return (first + sum(rest))/(1 + len(rest))

import html
def make_element(name, value, **attrs):
    keyvals = [' %s="%s"' % item for item in attrs.items()]
    attr_str = ''.join(keyvals)
    element = '<{name}{attrs}>{value}</{name}>'.format(
                      name=name,
                      attrs=attr_str,
                      value=html.escape(value))
    return element

def anyargs(*args, **kwargs):
    print('args=',args)
    print('kwargs=',kwargs)

def main():
    print(avg(1,2))
    print(avg(1,2,3))

    print(Fore.BLUE + "and content:" + Style.RESET_ALL)

    print(make_element('item', 'I am value',size='large',quantity=6))
    print(make_element('p','<spam>'))

    print(Back.GREEN + 'demo2'+ Back.RESET)

    anyargs(1,2,3,t1=10,t2=20)

if __name__ == '__main__':
    main()


from test.common.public_data import PublicData
from utils.config import DataType, TimeMethod
import time
from selenium.webdriver.common.by import By
import traceback
from selenium.common import exceptions
from selenium import webdriver

class WebSelenium(PublicData):
    def iselement(self, xpath):
        """
        判断元素是否存在
        """
        try:
            self.chrlist[self.chrindex].find_element(By.XPATH, xpath)
            return True
        except exceptions.NoSuchElementException:
            return False

    def webPageif(self, value, type):
        """
        seleium 点击，输入，获取文本操作
        type: 1 Click  2 Input  3 Value
        """
        # 获取当前运行的浏览器
        if type == 3:
            self.chrIndex(value[2][0])
        else:
            self.chrIndex(value[0])

        # 获取xpath路径
        xpath = value[2]

        # 重试次数
        for i in range(3):
            try:
                xpath = self.strValue(xpath)
                if self.iselement(xpath):
                    if type == 1:
                        self.webPageElementClick(xpath)
                    elif type == 2:
                        self.webPageElementInput(xpath, value)
                    elif type == 3:
                        self.webElementValue(value)
                    else:
                        print(f"类型不正确:{type}")
                    break
                else:
                    if i == 2:
                        self.logs(f"没有定位到元素位置：{xpath}  -----------------------------------------------")
                    time.sleep(1)
                    pass
            except:
                errorlog = traceback.print_exc()
                self.logs(f"执行报错：{xpath}{errorlog}  -----------------------------------------------")

    # 获取当前打开chrome
    def chrIndex(self, name):
        self.chrindex = self.chrstrlist.index(name)

    # 启动google浏览器
    def webBrowser(self, value):
        name = value[0]

        # 判断当前打开的浏览器是否保存在chrstrlist
        if name not in self.chrstrlist:
            self.chrstrlist.append(name)
            self.chrIndex(name)
            self.chrlist.append(self.chrindex)
        else:
            self.chrIndex(name)
        # 打开浏览器
        if self.system_test == 1:
            option = webdriver.ChromeOptions()
            # 无头模式
            option.add_argument('headless')
            # driver_width, driver_height = pyautogui.size()  # 通过pyautogui方法获得屏幕尺寸
            # option.add_argument('--window-size=%sx%s' % (driver_width, driver_height))  # 设置浏览器窗口大小
            option.add_argument("--start-maximized")
            # 沙盒模式运行
            option.add_argument('no-sandbox')
            # 大量渲染时候写入/tmp而非/dev/shm
            option.add_argument('disable-dev-shm-usage')
            option.add_argument('--disable-gpu')

            self.chrlist[self.chrindex] = webdriver.Chrome(options=option)
            # 窗口最大化
            self.chrlist[self.chrindex].maximize_window()
            self.chrlist[self.chrindex].execute_script("""navigator.geolocation.getCurrentPosition = function(success, failure) {
                  success({
                    coords: {latitude: -43.5333, longitude: 172.633},
                    timestamp: Date.now(),
                  });
                }""")
        else:
            self.chrlist[self.chrindex] = webdriver.Chrome()

        # 窗口最大化
        # self.chrlist[self.chrindex].minimize_window()
        # self.chrlist[self.chrindex].maximize_window()

    # 打开url
    def webBrowserPage(self, value):
        self.chrIndex(value[0])
        if value[1] == "str":
            url = self.strValue(value[2])
        else:
            url = self.strValue(value[1])
        # 打开链接
        self.chrlist[self.chrindex].get(url)
        time.sleep(5)

    # 点击元素
    def webPageElementClick(self, xpath):
        if ("photo-upload-modal" in xpath and "select-from-album" in xpath) or ("img-file" in xpath):
            self.photoUpload(xpath)
        else:
            time.sleep(1)
            try:
                self.chrlist[self.chrindex].find_element(By.XPATH, xpath).click()
            except:
                self.logs(f"执行报错：{xpath}  -----------------------------------------------")
                self.chrlist[self.chrindex].execute_script("arguments[0].click();",
                                                           self.chrlist[self.chrindex].find_element(By.XPATH, xpath))

    # 上传图片
    def photoUpload(self, xpath):
        self.chrlist[self.chrindex].find_element(By.XPATH, xpath).send_keys(f"{self.testData}test.jpg")

    # 输入信息
    def webPageElementInput(self, xpath, value):
        if value[5] == "str":  # 判断参数中是否有拼接
            values = self.strValue(value[6])
        else:
            if len(value[5:]) > 1:
                values = ""
                for x in value[5:]:
                    values += x
                values = DataType().updateStrMake(values) + "\n"
            else:
                values = self.strValue(value[5])

        # 清除输入框内容
        try:
            self.chrlist[self.chrindex].find_element(By.XPATH, xpath).clear()
        except:
            pass
        if "ride-id-search-fld" in xpath:
            if TimeMethod().has_letters(values):
                values = values.replace('"', "")
            elif values[0] == "a":
                values = values.replace('"', "")
            else:
                values = values

            self.chrlist[self.chrindex].find_element(By.XPATH, xpath).send_keys(f"{values}\n")
        else:
            self.chrlist[self.chrindex].find_element(By.XPATH, xpath).send_keys(values)

    # 获取元素值
    def webElementValue(self, value):
        # 把获取的值保存到全局参数self.globalVariable
        self.chrIndex(value[2][0])
        try:
            if value[2][-1] == "value":
                self.globalVariables(value[0], self.chrlist[self.chrindex].
                                     find_element(By.XPATH, self.strValue(value[2][2])).get_attribute('value'))
                self.logs(f"获取元素成功：{value[0]}={self.globalVariable[value[0]]}")
            else:
                self.globalVariables(value[0], self.chrlist[self.chrindex].
                                     find_element(By.XPATH, self.strValue(value[2][2])).get_attribute('innerText'))
                self.logs(f"获取元素成功：{value[0]}={self.globalVariable[value[0]]}")
        except:
            try:
                self.globalVariables(value[0], self.chrlist[self.chrindex].
                                     find_element(By.XPATH, self.strValue(value[2][2])).text)
            except:
                try:
                    self.globalVariables(value[0], self.chrlist[self.chrindex].
                                         find_element(By.XPATH, self.strValue(value[2][2])).get_attribute(
                        'innerText'))
                except:
                    self.logs(f"获取元素失败：{value}")
                    self.globalVariables(value[0], "")

    # 获取元素json
    def webElementInfo(self, value):
        self.chrIndex(value[2][0])
        try:
            displayed = self.chrlist[self.chrindex].find_element(By.XPATH,
                                                                 self.strValue(value[2][2])).is_displayed()
            selected = self.chrlist[self.chrindex].find_element(By.XPATH,
                                                                 self.strValue(value[2][2])).is_selected()
            self.globalVariables(value[0], {"displayed": displayed, "selected": selected})
            self.logs(f"获取元素成功：{value[0]}={self.globalVariable[value[0]]}")
        except:
            self.logs(f"获取元素json失败：{value}")
            self.globalVariables(value[0], "")


import sqlite3 as sql
def initialize_db():
    connection = sql.connect("products.db")
    cursor = connection.cursor()

    #create table with products
    cursor.execute("DROP TABLE IF EXISTS products")
    cursor.execute("CREATE TABLE products (name TEXT, description TEXT, price FLOAT, stock INTEGER, imgpath TEXT)")
    cursor.execute("""INSERT INTO products (name, description, price, stock, imgpath) VALUES \
        ('Penne Pasta', 'Our most popular dish...', 21.99, 10, '/static/images/pasta.jpeg'), \
        ('Chicken Noodle Soup', 'A heartwarming soup made with...', 9.99, 10, '/static/images/soup.jpeg'), \
        ('Strawberry Shortcake Cake', 'A delicious slice of cake...', 4.99, 10, '/static/images/cake.jpeg')
    """)

    # Create empty transactions table
    cursor.execute("DROP TABLE IF EXISTS transactions")
    cursor.execute("CREATE TABLE transactions (timestamp TEXT, productid INTEGER, value INTEGER)")

    # Commit the db changes
    connection.commit()

def get_cursor():
    connection = sql.connect("products.db")
    cursor = connection.cursor()
    return (cursor, connection)

#process information about all the products to be displayed
def process_info():
    (cursor, _) = get_cursor()
    cursor.execute("SELECT rowid, * FROM products")
    rows = cursor.fetchall()

    products = []
    for row in rows:
        products.append({
            "id":    row[0],
            "name":  row[1],
            "description": row[2],
            "price": "$%.2f" % (row[3]),
            "stock": "%d left" % (row[4]),
            "src":   "%s" % (row[5]),
        })
    return products

#get data on specific product
def get_product(product_id):
    (cursor, _) = get_cursor()
    cursor.execute("SELECT name, description, price, stock FROM products WHERE rowid = ?", (product_id,))
    result = cursor.fetchone()
    return result

#update information for specific product
def update_product(name, description, price, stock, product_id):
    (cursor, connection) = get_cursor()
    cursor.execute("UPDATE products SET name = ?, description = ?, price = ?, stock = ? WHERE rowid = ?", (name, description, price, stock, product_id,))
    connection.commit()

#process a buy transaction
def process_transaction(product_id, price):
    (cursor, connection) = get_cursor()
    cursor.execute("INSERT INTO transactions (timestamp, productid, value) VALUES " + \
    "(datetime(), ?, ?)", (product_id, price))

    cursor.execute("UPDATE products SET stock = stock - 1 WHERE rowid = ?", (product_id,))
    connection.commit()

#calculate the total earnings
def calc_sum():
    (cursor, connection) = get_cursor()
    cursor.execute("SELECT SUM(value) FROM transactions")
    result = cursor.fetchone()[0]
    earnings = result if result else 0
    return earnings

#delete item
def delete(product_id):
    (cursor, connection) = get_cursor()
    cursor.execute("DELETE FROM products WHERE rowid = ?", (product_id,))
    connection.commit()


# UPDATE DICTIONARIES AND LISTS

x = [ [5,2,3], [10,8,9] ]
students = [
    {'first_name':  'Michael', 'last_name' : 'Jordan'},
    {'first_name' : 'John', 'last_name' : 'Rosales'}
]
sports_directory = {
    'basketball' : ['Kobe', 'Jordan', 'James', 'Curry'],
    'soccer' : ['Messi', 'Ronaldo', 'Rooney']
}
z = [ {'x': 10, 'y': 20} ]

# 1
x[1][0] = 15
# print(x)

# 2
students[0]["last_name"] = "Bryant"
# print(students[0])

# 3
sports_directory['soccer'][0] = "Andres"
# print(sports_directory['soccer'])

# 4
z[0]['y'] = 30
# print(z)

# ITERATE LISTS OF DICTIONARIES

def iterateDictionary(some_list):
    for index in range(len(some_list)):
        # print(index)
        for key in some_list[index]:
            # print(key)
            print(key + " - " + some_list[index][key])

def itemsIterate(some_list):
    for index in range(len(some_list)):
        for key,value in some_list[index].items():
            print(key + " - " + value)

students = [
        {'first_name':  'Michael', 'last_name' : 'Jordan'},
        {'first_name' : 'John', 'last_name' : 'Rosales'},
        {'first_name' : 'Mark', 'last_name' : 'Guillen'},
        {'first_name' : 'KB', 'last_name' : 'Tonel'}
    ]

iterateDictionary(students)

itemsIterate(students)

# should output: (it's okay if each key-value pair ends up on 2 separate lines;
# bonus to get them to appear exactly as below!)
# first_name - Michael, last_name - Jordan
# first_name - John, last_name - Rosales
# first_name - Mark, last_name - Guillen
# first_name - KB, last_name - Tonel

# GET VALUS FROM A LIST OF DICTIONARIES

def iterateDictionary2(key_name, some_list):
    for index in range(len(some_list)):
        print(some_list[index][key_name])

iterateDictionary2('first_name', students)

iterateDictionary2('last_name', students)

# Iterate through a dict with list values

def printInfo(some_dict):
    for keys in some_dict:
        print(len(some_dict[keys]) , keys.upper())
        for items in some_dict[keys]:
            print(items)
        print("")

dojo = {
    'locations': ['San Jose', 'Seattle', 'Dallas', 'Chicago', 'Tulsa', 'DC', 'Burbank'],
    'instructors': ['Michael', 'Amy', 'Eduardo', 'Josh', 'Graham', 'Patrick', 'Minh', 'Devon']
}
printInfo(dojo)

# output:
# 7 LOCATIONS
# San Jose
# Seattle
# Dallas
# Chicago
# Tulsa
# DC
# Burbank

# 8 INSTRUCTORS
# Michael
# Amy
# Eduardo
# Josh
# Graham
# Patrick
# Minh
# Devon


import torch
import torch.nn as nn
import numpy as np

import matplotlib.pyplot as plt

class TrajNet(nn.Module):
    def __init__(self):
        super(TrajNet, self).__init__()
        self.rnn = nn.LSTM(input_size=6, hidden_size=64, num_layers=2)
        self.fc = nn.Linear(64, 1200)

    def forward(self, x):
        _, (h, _) = self.rnn(x)
        output = self.fc(h[-1])
        return output

class StackingBlock(nn.Module):
    def __init__(self, input, hidden, output):
        super(StackingBlock, self).__init__()

        layer_info = torch.cat((input, hidden, output))
        self.model = self.__gen_model(layer_info)

    def __gen_model(self, layer_info):
        layers = []

        for i in range(len(layer_info)-1):

            if i == len(layer_info)-2:
                layers.append(nn.Linear(layer_info[i], layer_info[i+1]))
                # layers.append(nn.ReLU(inplace=True))
                layers.append(nn.Tanh())

            else:
                layers.append(nn.Linear(layer_info[i], layer_info[i+1]))
                # layers.append(nn.ReLU(inplace=True))
                layers.append(nn.Tanh())
                # layers.append(nn.BatchNorm1d(layer_info[i+1]))

        return nn.Sequential(*layers)

    def forward(self, coords):
        u = self.model(coords)

        return u


#! /usr/bin/env python

with open('./data.txt') as f:
    data = f.read().split()

def position(code, min_no, max_no, min_zone_marker):
    for c in code:
        if c == min_zone_marker:
            # The position is in the lower seat numbers
            max_no = (max_no - min_no) // 2 + min_no
        else:
            min_no = max_no - ((max_no - min_no) // 2)

    return min_no

print(max(
    position(seat_code[:-3], 0, 127, 'F') * 8 + position(seat_code[-3:], 0, 7, 'L')
    for seat_code in data
))


#!usr/bin/env python
# -*- coding: utf-8 -*-
# Title: clean_class.py
# author: clasch
# class: Udacity SQL Project 3
# date: 1/28/2016
# 0 edits from parent /../p3/clean_class.py

import re

class CleanValue(object):
    """CleanValue returns a corrected value (determined by audit).
    It requires the value's key to determine cleaning route."""

    def __init__(self):
        """set variables for Fort Collins data. Mappings were derived from the audit step."""
        self.expected_street_types = fc_st_types = [ "Street", "Avenue", "Boulevard", "Drive", "Court",
                        "Place", "Square", "Lane", "Road", "Trail", "Parkway", "Commons", "Way",
                        "Circle", "East", "West", "North", "South", "Southwest", "Alley", "Square"]
        self.street_mapping = st_mapping = { "st" : "Street",
                       "St" : "Street",
                       "St." : "Street",
                       "Aventue" : "Avenue",
                       "Rd" : "Road",
                       "Dr" : "Drive",
                       "Ct" : "Court" }
        self.foco_manual_updates = fc_manual_st = {"S Summit View #11" : "S Summit View Drive",
                        "Old Town Square #238" : "Old Town Square",
                        "S Timberline" : "S Timberline Road",
                        "West Oak" : "West Oak Street",
                        "Seventh-Day Adventist Church" : "E Pitkin Street" }
        self.expected_postal_codes = fc_posts = ["80521", "80522", "80523", "80524", "80525", "80526",
                    "80527", "80528", "80553", "80547", "80538", "80535" ]

        self.street_type_re = re.compile(r"\b\S+\.?$", re.IGNORECASE)

    def correct_post(self, postal_code):
        """Correct postal code if not expected."""
        if postal_code[:5] not in self.expected_postal_codes:

            re_state_prefix = re.compile(r"\D*\s+", re.IGNORECASE)
            if re_state_prefix.match(postal_code):
                postal_code = re.sub(re_state_prefix, '', postal_code)   # remove any letters/spaces from 'CO 80525'
            elif postal_code == "80701":
                postal_code = "80525"     # specific to correct weigh station postal_code typo

        return postal_code

    def correct_street(self, street_name):
        """Replaces street_type with its mapping if not expected."""
        street_name = street_name.strip()
        street_type = self.street_type_re.search(street_name).group()

        if street_type not in self.expected_street_types:

            if street_type in self.street_mapping:
                street_name = street_name.replace(street_type, self.street_mapping[street_type])
            elif street_name in self.foco_manual_updates:
                street_name = self.foco_manual_updates[street_name]

        return street_name

    # zip and streets in one function accepting any value
    def process(self, key, value):
        """Accepts key and value. Calls the correct value cleaner based on key."""

        if "post" in key:
            return self.correct_post(value)
        elif key == "addr:street":
            return self.correct_street(value)
        else:
            return value


import string
import requests
import os
import shutil

from bs4 import BeautifulSoup
from urllib.parse import urljoin

def download_resource(url, filename):
    """Downloads the content at specified URL to a file."""
    try:
        response = requests.get(url, headers={'Accept-Language': 'en-US,en;q=0.5'})
        response.raise_for_status()
    except requests.RequestException as e:
        print(f'Failed to retrieve {url}: {e}')
        return

    if not response.content:
        print(f'No content to download from {url}')
        return

    try:
        with open(filename, 'wb') as raw_page:
            raw_page.write(response.content)
    except Exception as e:
        print(f'Failed to save {filename}: {e}')
        return

def get_article_link_and_description(html_resource):
    """Parses HTML to find and return article link and description."""
    try:
        soup = BeautifulSoup(html_resource, 'html.parser')
    except Exception as e:
        print(f'Failed to parse HTML: {e}')
        return None, None

    anchor = soup.find('a', {'data-track-action': 'view article'})
    if anchor is not None:
        link = anchor['href']
        description = anchor.text
        return link, description
    return None, None

def save_as_textfile(url, tag, attributes):
    """Saves article content from URL to a text file."""
    try:
        response = requests.get(url, headers={'Accept-Language': 'en-US,en;q=0.5'})
        response.raise_for_status()
    except requests.RequestException as e:
        print(f'Failed to retrieve {url}: {e}')
        return

    if not response.content:
        print(f'No content to download from {url}')
        return

    try:
        soup = BeautifulSoup(response.content, 'html.parser')
    except Exception as e:
        print(f'Failed to parse HTML: {e}')
        return

    title = soup.find('title').text
    resource = soup.find(tag, attributes)
    if resource is not None:
        filename = title_to_filename(title)
        byte_content = resource.text.encode('utf-8')
        try:
            with open(filename, 'wb') as file:
                file.write(byte_content)
        except Exception as e:
            print(f'Failed to save {filename}: {e}')
            return

def search_articles(absolute_url, html_file, article_type):
    """Searches for articles of a specified type in the HTML file."""
    try:
        with open(html_file, 'r', encoding='utf-8') as file:
            soup = BeautifulSoup(file, 'html.parser')
    except Exception as e:
        print(f'Failed to open or parse {html_file}: {e}')
        return

    # Iterate through all articles, filter by type, and save to text files
    articles = soup.find_all('article')
    for article in articles:
        article_type_span = article.find('span', {'data-test': 'article.type'})
        if article_type_span is not None:
            current_article_type = article_type_span.text
            if current_article_type == article_type:
                relative_link, _ = get_article_link_and_description(str(article))
                if relative_link:
                    full_url = urljoin(absolute_url, relative_link)
                    save_as_textfile(full_url, 'p', {'class': 'article__teaser'})

def title_to_filename(title):
    """Converts article title to a filename-friendly format."""
    title_no_punctuation = title.translate(str.maketrans('', '', string.punctuation))
    words = title_no_punctuation.split()
    return f'{"_".join(words)}.txt'

def request_page_count():
    """Prompts user for the number of pages to download and validates input."""
    number_of_pages = input('Enter the number of pages to download: ')
    while not number_of_pages.isdigit():
        number_of_pages = input('Invalid input. Enter the number of pages to download: ')
    return int(number_of_pages)

def extract_and_save_articles(url, pages_count, article_type):
    """Processes and saves articles from multiple pages of a specified type."""
    html_filename = 'source.html'
    for i in range(pages_count):
        dir_name = f'Page_{i + 1}'
        if os.path.exists(dir_name):
            shutil.rmtree(dir_name)
        os.mkdir(dir_name)
        os.chdir(dir_name)
        link = f'{url}&page={i + 1}'
        download_resource(link, html_filename)
        search_articles(link, html_filename, article_type)
        os.chdir('..')
    print('Saved all articles.')

def main():
    journal_url = 'https://www.nature.com/nature/articles?sort=PubDate&year=2020'
    pages_to_download = request_page_count()
    type_of_article = input('Enter the type of article (e.g., News, Editorial, etc.): ')
    extract_and_save_articles(journal_url, pages_to_download, type_of_article)

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt

n1=1.5
n2=1
a=np.linspace(0,np.pi/2,1000)
b=np.arcsin(n1*np.sin(a)/n2)
c=a*180/np.pi

rp=np.tan(a-b)/np.tan(a+b)
Rp=(abs(rp))**2
rs=np.sin(b-a)/np.sin(b+a)
Rs=(abs(rs))**2

plt.plot(c,Rp,color='black')
plt.plot(c,Rs,color='red')
plt.xlim(0,90)
plt.legend()
plt.show()

'''
np.savetxt("C:/Users/maila/OneDrive/デスクトップ/保存先/Rp.txt",Rp)
np.savetxt("C:/Users/maila/OneDrive/デスクトップ/保存先/Rs.txt",Rs)
np.savetxt("C:/Users/maila/OneDrive/デスクトップ/保存先/c.txt",c)
'''


# -*- coding: utf-8 -*-
from selenium import webdriver
import csv

#list of whatsapp group invite links
links = ["https://chat.whatsapp.com/JCFcDOTQO7aA5etQXeUEyB",
"https://chat.whatsapp.com/55blTpaDk4j791aTpP6usH"]
w, h = 2, len(links);
name = [[0 for x in range(w)] for y in range(h)]
i = 0
driver = webdriver.Chrome()
for x in links:
	driver.get(x)
	name[i][0] = driver.find_element_by_class_name("block__title").text
	name[i][1] = driver.current_url
	i += 1

with open('employee_file2.csv', mode='w', encoding='utf-8') as csv_file:
	fieldnames = ['Name', 'URL']
	writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
	writer.writeheader()
	for x in name:
		if len(x[0]) > 0:
			writer.writerow({'Name': x[0], 'URL': x[1]})