import pandas as pd
import google_auth
import googleapiclient.discovery

class Filter:
    def __init__(self, f_id,f_name, f_type, f_update, f_details=None, f_rank=None):
        self.id = f_id
        self.name = f_name
        self.type = f_type
        self.update = f_update
        self.details = f_details
        self.rank = f_rank

def response2df(report):
    """Parses and prints the Analytics Reporting API V4 response"""
    report = report.get('reports', [])[0]
    #Initialize empty data container for the two date ranges (if there are two that is)
    data_csv = []

    #Initialize header rows
    header_row = []

    #Get column headers, metric headers, and dimension headers.
    column_header = report.get('columnHeader', {})

    metric_headers = column_header.get('metricHeader', {}).get('metricHeaderEntries', [])

    dimension_headers = column_header.get('dimensions', [])

    #Combine all of those headers into the header_row, which is in a list format
    for dheader in dimension_headers:
        header_row.append(dheader)
    for mheader in metric_headers:
        header_row.append(mheader['name'])

    #Get data from each of the rows, and append them into a list
    rows = report.get('data', {}).get('rows', [])

    for row in rows:
        row_temp = []
        dimensions = row.get('dimensions', [])
        metrics = row.get('metrics', [])

        for d in dimensions:
            row_temp.append(d)

        for m in metrics[0]['values']:
            row_temp.append(m)

        data_csv.append(row_temp)

    #Putting those list formats into pandas dataframe, and append them into the final result
    result_df = pd.DataFrame(data_csv, columns=header_row)

    return result_df

class AnalyticsReporting:
    API_VER = 'v4'
    API_NAME = 'analyticsreporting'

    def __init__(self, view_id):
        credentials = google_auth.build_credentials()
        self.analytics = googleapiclient.discovery.build(self.API_NAME, self.API_VER, credentials=credentials)
        self.view_id = view_id

    def _get_report_data(self, dimensions, metrics, date_range):
        reportRequests = [
            {
                "viewId": self.view_id,
                "dateRanges": [{
                    "startDate": date_range[0],
                    "endDate": date_range[1],
                }],
                'metrics': metrics,
                'dimensions': dimensions
            }]
        return self.analytics.reports().batchGet(
            body={
                'reportRequests': reportRequests
            }
        ).execute()

    def report(self, dimensions, metrics, date_range):
        """Parses and and returns a pd.DataFrame of the Analytics Reporting API V4 response"""
        report = self._get_report_data(dimensions=dimensions, metrics=metrics, date_range=date_range)
        report = report.get('reports', [])[0]
        #Initialize empty data container for the two date ranges (if there are two that is)
        data = []
        header_row = []

        #Get column headers, metric headers, and dimension headers.
        column_header = report.get('columnHeader', {})
        metric_headers = column_header.get('metricHeader', {}).get('metricHeaderEntries', [])
        dimension_headers = column_header.get('dimensions', [])

        #Combine all of those headers into the header_row, which is in a list format
        for dheader in dimension_headers:
            header_row.append(dheader)
        for mheader in metric_headers:
            header_row.append(mheader['name'])

        #Get data from each of the rows, and append them into a list
        rows = report.get('data', {}).get('rows', [])

        for row in rows:
            row_temp = []
            dimensions = row.get('dimensions', [])
            metrics = row.get('metrics', [])

            for d in dimensions:
                row_temp.append(d)

            for m in metrics[0]['values']:
                row_temp.append(m)

            data.append(row_temp)

        #Putting those list formats into pandas dataframe, and append them into the final result
        result_df = pd.DataFrame(data, columns=header_row)
        return result_df

class AnalyticsManagement:
    API_VER = 'v3'
    API_NAME = 'analytics'

    def __init__(self, account_id, property_id, view_id):
        credentials = google_auth.build_credentials()
        self.analytics = googleapiclient.discovery.build(self.API_NAME, self.API_VER, credentials=credentials)
        self.account_id = account_id
        self.property_id = property_id
        self.view_id = view_id

    def list_accounts(self):
        """returns a list of management.accountSummaries Resource"""
        accounts = self.analytics.management().accounts().list().execute()
        return accounts.get('items', [])

    def list_account_users(self):
        users = self.analytics.management().accountUserLinks().list(accountId=self.account_id).execute()
        return users.get('items', [])

    def list_account_filters(self):
        filters = self.analytics.management().filters().list(accountId=self.account_id).execute()
        return filters.get('items', [])

    def get_property(self):
        """per default pre-defined property is returned"""
        prop = self.analytics.management().webproperties().get(
            accountId=self.account_id,
            webPropertyId=self.property_id).execute()
        return prop

    def list_view_filters(self):
        """per default view filters for pre-defined property and view are returned"""
        response = self.analytics.management().profileFilterLinks().list(
            accountId=self.account_id,
            webPropertyId=self.property_id,
            profileId=self.view_id).execute()
        view_filters = response.get('items', [])
        view_filter_list = []
        for filter_obj in view_filters:
            filter_id = filter_obj['id'].split(':')[1]
            rank = filter_obj['rank']
            view_filter_list.append({'id': filter_id, 'rank': rank})

        return view_filter_list

    def handle_filters(self):
        account_filters = self.list_account_filters()
        filter_dict = []
        for filter in account_filters:
            filter_ins = Filter(f_id=filter['id'], f_name=filter['name'], f_type=filter['type'], f_update=filter['updated'])
            if filter['type'] == "SEARCH_AND_REPLACE":
                filter_ins.details = filter['searchAndReplaceDetails']
            elif filter['type'] == "INCLUDE":
                filter_ins.details = filter["includeDetails"]
            elif filter['type'] == "EXCLUDE":
                filter_ins.details = filter["excludeDetails"]
            elif filter['type'] == "LOWERCASE":
                filter_ins.details = filter["lowercaseDetails"]
            elif filter['type'] == "UPPERCASE":
                filter_ins.details = filter["uppercaseDetails"]
            elif filter['type'] == "ADVANCED":
                filter_ins.details = filter["advancedDetails"]
            else:
                filter_ins.details = {'key': 'value'}
            filter_dict.append(filter_ins)

        return filter_dict


# 918. Maximum Sum Circular Subarray
class Solution:
    def maxSubarraySumCircular(self, A: List[int]) -> int:
        '''
        This is based on 53. Maximum Subarray.
        There are 2 cases in general - whether or not the max subarray is circular.
        1. max subarray is no circular, so it's the same as 53;
        2. max subarray is circular, it seems tricky to handle the circle, but we can
           view it as the {sum - min_subarray_sum};
        (3). this is the edge case: if all numbers are negative, we should return #1;
           We can check max_sum to tell whether all numbers are negative.
        '''
        max_sum = min_sum = A[0]
        total = cur_max_sum = cur_min_sum = 0
        for a in A:
            total += a
            cur_max_sum = max(cur_max_sum + a, a)
            max_sum = max(max_sum, cur_max_sum)
            cur_min_sum = min(cur_min_sum + a, a)
            min_sum = min(min_sum, cur_min_sum)
        return max(max_sum, total - min_sum) if max_sum > 0 else max_sum

medium/918. Maximum Sum Circular Subarray.py


import cv2
import numpy as np
import os
import sys
import tensorflow as tf
import ntpath

from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout

from sklearn.model_selection import train_test_split

EPOCHS = 10
IMG_WIDTH = 30
IMG_HEIGHT = 30
NUM_CATEGORIES = 43
TEST_SIZE = 0.4

def main():

    # Check command-line arguments
    if len(sys.argv) not in [2, 3]:
        sys.exit("Usage: python traffic.py data_directory [model.h5]")

    # Get image arrays and labels for all image files
    images, labels = load_data(sys.argv[1])

    # Split data into training and testing sets
    labels = tf.keras.utils.to_categorical(labels)
    x_train, x_test, y_train, y_test = train_test_split(
        np.array(images), np.array(labels), test_size=TEST_SIZE
    )

    # Get a compiled neural network
    model = get_model()

    # Fit model on training data
    model.fit(x_train, y_train, epochs=EPOCHS)

    # Evaluate neural network performance
    model.evaluate(x_test,  y_test, verbose=2)

    # Save model to file
    if len(sys.argv) == 3:
        filename = sys.argv[2]
        model.save(filename)
        print(f"Model saved to {filename}.")

def load_data(data_dir):
    """
    Load image data from directory `data_dir`.

    Assume `data_dir` has one directory named after each category, numbered
    0 through NUM_CATEGORIES - 1. Inside each category directory will be some
    number of image files.

    Return tuple `(images, labels)`. `images` should be a list of all
    of the images in the data directory, where each image is formatted as a
    numpy ndarray with dimensions IMG_WIDTH x IMG_HEIGHT x 3. `labels` should
    be a list of integer labels, representing the categories for each of the
    corresponding `images`.
    """
    list_of_all_img_arr = []
    list_of_all_labels = []

    list_of_labels = os.listdir(data_dir)
    for label in list_of_labels: # for each label
        label_path = os.path.join(data_dir, label)
        img_filenames = os.listdir(label_path)
        for img_filename in img_filenames: # for each image
            img_path = os.path.join(label_path, img_filename)
            img_arr = cv2.imread(img_path) # read in the image as a numpy array
            img_arr = cv2.resize(img_arr, (IMG_WIDTH, IMG_HEIGHT)) # resize the image
            list_of_all_img_arr.append(img_arr) # Append the image numpy array to a list
            list_of_all_labels.append(label) # Append the labels to a list

    return (list_of_all_img_arr, list_of_all_labels) # return a tuple of image arrays and labels

def get_model():
    """
    Returns a compiled convolutional neural network model. Assume that the
    `input_shape` of the first layer is `(IMG_WIDTH, IMG_HEIGHT, 3)`.
    The output layer should have `NUM_CATEGORIES` units, one for each category.
    """
    model = tf.keras.models.Sequential([
        Conv2D(32, (3, 3), activation = "relu", input_shape = (IMG_WIDTH, IMG_HEIGHT, 3)),
        MaxPooling2D(pool_size= (2, 2)),
        Conv2D(32, (3, 3), activation = "relu"),
        MaxPooling2D(pool_size = (2, 2)),
        # Conv2D(32, (3, 3), activation = "relu"),
        # MaxPooling2D(pool_size = (2, 2)),
        Flatten(),
        Dense(128, activation="relu"),
        Dropout(0.5),
        # Dense(64, activation="relu"),
        # Dropout(0.5),

        Dense(NUM_CATEGORIES, activation = 'softmax')
    ])

    model.compile(
        optimizer = "adam",
        loss = "categorical_crossentropy",
        metrics = ['accuracy']
    )

    return model

if __name__ == "__main__":
    main()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

import time

from socorro.lib.threaded_task_manager import ThreadedTaskManager

class TestThreadedTaskManager:
    def test_start(self):
        ttm = ThreadedTaskManager(
            idle_delay=1,
            number_of_threads=1,
            maximum_queue_size=1,
        )
        try:
            ttm.start()
            time.sleep(0.2)
            assert ttm.queueing_thread.is_alive()
            assert len(ttm.thread_list) == 1
            assert ttm.thread_list[0].is_alive()
            ttm.stop()
            assert not ttm.queueing_thread.is_alive()
        finally:
            # we got threads to join
            ttm.wait_for_completion()

    def test_doing_work_with_one_worker(self):
        my_list = []

        def insert_into_list(anItem):
            my_list.append(anItem)

        ttm = ThreadedTaskManager(
            idle_delay=1,
            number_of_threads=1,
            maximum_queue_size=1,
            task_func=insert_into_list,
        )
        try:
            ttm.start()
            time.sleep(0.2)
            assert len(my_list) == 10
            assert my_list == list(range(10))
            ttm.stop()
        finally:
            # we got threads to join
            ttm.wait_for_completion()

    def test_doing_work_with_two_workers_and_generator(self):
        my_list = []

        def insert_into_list(anItem):
            my_list.append(anItem)

        ttm = ThreadedTaskManager(
            idle_delay=1,
            number_of_threads=2,
            maximum_queue_size=2,
            task_func=insert_into_list,
            job_source_iterator=(((x,), {}) for x in range(10)),
        )
        try:
            ttm.start()
            time.sleep(0.2)
            assert len(ttm.thread_list) == 2
            assert len(my_list) == 10
            assert sorted(my_list) == list(range(10))
        finally:
            # we got threads to join
            ttm.wait_for_completion()

    def test_blocking_start_with_quit_on_empty(self):
        calls = []

        def task_func(index):
            calls.append(index)

        tm = ThreadedTaskManager(
            number_of_threads=2,
            maximum_queue_size=2,
            quit_on_empty_queue=True,
            task_func=task_func,
        )

        tm.blocking_start()
        assert len(calls) == 10


'''Escribir un programa que pregunte al usuario los números ganadores de la lotería primitiva, los almacene en una lista y los muestre por pantalla ordenados de menor a mayor.'''

def ordenMenorMayor(numeros:list) ->list:
    '''ordena los numeros ingresados'''
    numero = len(numeros)
    for i in range(numero-1):       # bucle padre
        for j in range(numero-1-i): # bucle hijo
            if numeros[j] > numeros[j+1]:
                numeros[j], numeros[j+1] = numeros[j+1], numeros[j]
    return numeros

if __name__=="__main__":
    #entrada y proceso
    numeros = []
    while len(numeros) < 8:
        numero = int(input("Escribe los números ganadores de la primitiva: "))
        numeros.append(numero)
    else:
        print("Error: tamaño de la lista alcanzado (sólo 8 digitos).")

    #salida
    ordenados = ordenMenorMayor(numeros)
    print(ordenados)


import re
from typing import List, Dict, Tuple
from loguru import logger
import pandas as pd

from PropertyParserInterface import PropertyParserInterface

class RevisionsParser(PropertyParserInterface):
    def __init__(self, lines: List[str]):
        super().__init__(lines)
        # Pattern to detect date in any format
        self.datePattern = re.compile(r"^[0-9]{2,4}[\/\-\.]([0-9]{2}|[A-Za-z]+)[\/\-\.][0-9]{2,4}$")
        # Pattern to detect possible two columns in one line. Parametrized to use for 2-4 columns.
        self.columnsPattern = "^[ ]{0,5}((?:\S+(?:(?!\s{2}).)+)(?:[ ]{3,}(?:\S+(?:(?!\s{2}).)+)){%d})$"
        self.versionDetectionPattern = re.compile(r"[a-zA-Z\.]{0,10}[0-9]+[\.][0-9]+$")
        self.versionPattern = re.compile(r"([0-9]+[\.][0-9]+)")

    def find_keywords(self, lines: List[str], keywords: List[str]) -> list:
        """ Find keywords in text, return list of line numbers """
        subs = [string.lower() for string in keywords]
        # Find line indexes where the substring is
        start_idxs = [i - 1 for i, line in enumerate(lines) if any(map(line.lower().__contains__, subs))]
        return start_idxs

    def is_table_header(self, string: str, keywords: List[str]):
        """ Check line if it can be a table header """
        score = 0
        columns = re.split(r" {2,}", string)
        if len(columns) > 4 or len(columns) < 2:
            return False
        # Check if does not contain numbers or symbols
        is_alpha_or_space = all(c.isalpha() or c.isspace() for c in string)
        if not is_alpha_or_space: return False
        return True

    def extract_formatting_params(self, line: str) -> List[int]:
        """ Tries to extract start indexes for each column """
        # Split line by two or more spaces
        columns = [x for x in re.split("\s{2,}", line) if x]
        # We need to find index of each column in string.
        # As split does not give indexes, let's try to find by first occurence
        indexes = [line.find(column) for column in columns]
        return indexes

    def most_frequent(self, List):
        return max(set(List), key = List.count)

    def extract_table(self, lines: List[str], number_of_cols: int, pattern: str):
        """ Tries to extract table with given number of columns using
            the given regex pattern """
        table_header_keywords = ['Date', 'Description', 'Rev', 'Revision', 'Version']
        table_header = prev_line_format = table = []
        table_header_found = False
        false_lines = final_lines = 0
        for line in lines:
            result = re.findall(pattern, line)
            columns = [x for x in re.split(" {2,}", line) if x]
            if false_lines > 4: return []
            if final_lines > 1: break
            if len(table) > 0:
                # Check the newline at the end
                if line == '\n':
                    final_lines += 1
                    # Reset format indexes to avoid false positives in case of new line
                    prev_line_format = []
                    continue
                else:
                    final_lines = 0
                # Adds line to the table if line is part of the previous
                formatting = self.extract_formatting_params(line)
                if len(formatting) == 1 and formatting[0] in prev_line_format:
                    for i, col in enumerate(prev_line_format):
                        if(formatting[0] == col):
                            table[-1][i] += " " + line
                            break
                    continue
            # Filtering inconsistent data or ToC
            if len(result) == 0 or len(re.findall("[\.]{4,}", line)) > 0:
                false_lines += 1
                continue
            if not table_header_found and self.is_table_header(line, table_header_keywords):
                if len(columns) == number_of_cols:
                    table_header = columns
                    table_header_found = True
                    # Table line cannot exist before header
                    table = []
                continue

            table.append(list(columns))
            # Find start indexes
            prev_line_format = self.extract_formatting_params(line)
            false_lines = 0
        if table_header_found:
            return pd.DataFrame(table, columns=table_header)
        else:
            return pd.DataFrame(table)

    def find_table(self, start: List[int], lines: List[str]):
        """ Try to find tables """
        possible_tables = []
        for id in start:
            # We take next line after found keyword
            next_line = id + 1
            for i in range(2, 5):
                pattern = re.compile(self.columnsPattern % (i - 1))
                table = []
                try:
                    table = self.extract_table(lines[next_line:], i, pattern)
                    table = self.postprocess(table)
                except Exception as e:
                    logger.error(f"Parsing table failed. Skipping this one. (exception: {e})")
                    table = []
                if len(table) > 0:
                    possible_tables.append(table)
        # TODO: include table score to select better
        best_table = []
        if len(possible_tables) > 0:
            best_table = max(possible_tables, key=len)
        return best_table

    def process_ver(self, input: str) -> str:
        """ Extract version from string """
        res = re.findall(self.versionPattern, input)
        if len(res) == 0:
            return ''
        return res[0]

    def process_desc(self, input: str) -> str:
        """ Format description nicely """
        return " ".join(input.split())

    def postprocess(self, table: pd.DataFrame) -> pd.DataFrame:
        """ Find data types of columns. If found correctly
        remove unwanted spaces and symbols from the revision table """

        allowed_cols = ['version', 'date', 'description']
        ver_kwords = ['Version', 'Revision', 'Rev']
        date_kwords = ['Date']

        if len(table) == 0:
            return []

        # Try to extract data types from columns based on score
        for col in list(table.columns):
            date_score = ver_score = other_score = 0
            for i in range(len(table)):
                val = table[col].iloc[i]
                date_match = re.findall(self.datePattern, val)
                if len(date_match) > 0:
                    date_score += 1
                ver_match = re.findall(self.versionDetectionPattern, "".join(val.split()))
                if len(ver_match) > 0:
                    ver_score += 1
            if ver_score > date_score:
                table.rename(columns={col : 'version'}, inplace=True)
            if date_score > ver_score:
                table.rename(columns={col : 'date'}, inplace=True)
        # Description is always the last
        table.columns = [*table.columns[:-1], 'description']
        # Remove unused columns
        resulting_table = table[table.columns.intersection(allowed_cols)].copy(deep=True)
        # Postprocess data
        # If version and description columns were not detected, this table is bad
        if not ('version' in resulting_table and 'description' in resulting_table):
            return []
        # Clean-up of values
        resulting_table.loc[:,'version'] = resulting_table['version'].map(self.process_ver)
        resulting_table.loc[:,'description'] = resulting_table['description'].map(self.process_desc)

        return resulting_table

    def dataframe_to_dict(self, table: pd.DataFrame) -> List[dict]:
        """ Convert dataframe to dictionary. Accepts only columns named
            'version', 'date' and 'description' """
        out = []
        for i, row in table.iterrows():
            revision_item = dict()
            if 'version' in table.columns:
                revision_item['version'] = row['version']
            else: revision_item['version'] = ""
            if 'date' in table.columns:
                revision_item['date'] = row['date']
            else: revision_item['date'] = ""
            if 'description' in table.columns:
                revision_item['description'] = row['description']
            else: revision_item['description'] = ""
            out.append(revision_item)
        return out

    def parse(self) -> List[dict]:
        """ Main parsing function """
        keywords = ['Revision History', 'Version Control']
        indexes = self.find_keywords(self.lines, keywords)
        table = self.find_table(indexes, self.lines)
        if len(table) > 0:
            return self.dataframe_to_dict(table)
        return []


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

# entry to get model given dictionary input
def get_model(kw):
        return Model(**kw['model_args'])

# https://raw.githubusercontent.com/Bjarten/early-stopping-pytorch/master/pytorchtools.py
class EarlyStopper:

    def __init__(self, patience, delta):
        self.patience = int(patience)
        self.delta = float(delta)
        self.prev_val = None
        self.counter = 0

    def __call__(self, val):
        if self.prev_val is None:
            self.prev_val = val
            return False
        else:
            improvement = self.prev_val - val
            self.counter += 1 if improvement < self.delta else 0
        if self.counter >= self.patience:
            return True
        self.prev_val = val
        return False

class Model(nn.Module):

    def __init__(self, dimsin, hidden_nodes, dimsout, loss_function, \
                        optimizer_function, lr, wd, epochs, scoring_heuristic, \
                        early_stop_patience, early_stop_delta):
        super().__init__()
        self.dims_in = int(dimsin)
        self.hidden_nodes = int(hidden_nodes)
        self.dims_out = int(dimsout)
        self.loss_func = loss_function
        self.optim_func = optimizer_function
        self.optim_params = {'lr': float(lr), 'weight_decay': float(wd)}
        self.epochs = epochs
        self.scoring_heuristic = scoring_heuristic
        self.early_stop_patience = early_stop_patience
        self.early_stop_delta = early_stop_delta
        self.l1 = nn.Linear(self.dims_in, self.hidden_nodes)
        self.l2 = nn.Linear(self.hidden_nodes, self.dims_out)
        self.current_loss = None
        self.current_state_dict = None
        self.dev = torch.device('cpu:0')

    # learn an (x,y) dataset and return performance and parameters in dict
    def learn(self, x, y, sd=None, allow_earlystop=True):
        x = torch.from_numpy(x.astype(np.float32)).to(self.dev)
        y = torch.from_numpy(y.astype(np.float32)).to(self.dev)
        loss_func = getattr(nn, self.loss_func)()
        optimizer = getattr(optim, self.optim_func)
        optimizer = optimizer(self.parameters(), **self.optim_params)
        epochs = self.epochs
        stopper = EarlyStopper(self.early_stop_patience, self.early_stop_delta)
        if sd is not None and self.scoring_heuristic == 'gradient_heuristic':
            self.load_state_dict(sd)
            epochs = 1
        for i in range(epochs):
            pred = self.forward(x)
            loss = loss_func(pred, y)
            self.current_loss = loss.item()
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if epochs != 1 and stopper(self.current_loss) and allow_earlystop:
                print(f"Stopped @ {i} / {epochs}")
                break
        self.current_state_dict = self.state_dict()
        out = {}
        out['loss'] = loss.item()
        out['score'] = self.score
        out['state_dict'] = self.current_state_dict
        out['trained_for'] = i
        return out

    # forward pass on x
    def forward(self, x):
        relu = self.l1(x).clamp(min=0)
        return self.l2(relu)

    # forward pass with no backprop on x
    def predict(self, x):
        x = torch.from_numpy(x.astype(np.float32)).to(self.dev)
        with torch.no_grad():
            return self.forward(x)

    # calculate score of model
    @property
    def score(self):
        if self.scoring_heuristic == 'norm_heuristic':
            return self.norm_heuristic()
        elif self.scoring_heuristic == 'gradient_heuristic':
            return self.gradient_heuristic()

    # calculate sum of norms of weights
    def norm_heuristic(self):
        norm = 0
        l1 = self.l1.weight.detach().numpy()
        l2 = self.l2.weight.detach().numpy()
        norm += np.linalg.norm(l1)
        norm += np.linalg.norm(l2)
        return norm

    # calculate norm of gradient after one forward pass
    def gradient_heuristic(self):
        loss = self.current_loss
        wi_grads = np.array([])
        for w in [self.l1, self.l2]:
            w = w.weight.grad.numpy().flatten()
            wi_grads = np.append(wi_grads, w)
        wi_grads *= loss
        norm = np.linalg.norm(wi_grads)
        return norm


import numpy as np
import matplotlib.pyplot as plt
from ortools.sat.python import cp_model
import sys
import time

if len(sys.argv) < 2:
    quit()
try:
    istanza = int(sys.argv[1])
except:
    print("Metti un numero da 1 a 11 per runnure un'istanza, 0 per banale e -1 per banale ma non troppo!")
    quit()

file = ""
if istanza == 0:
    file = "inst_banale"
elif istanza == -1:
    file = "inst_banale_ma_non_troppo"
elif istanza < 10:
    file = "inst0{}".format(istanza)
else:
    file = "inst{}".format(istanza)
f = open('tsp_inst/' + file + ".dzn", "r")
text = f.read()
f.close()
content = text.split('\n')[:-2]

lower_boud = int(content[6][16:-1])

content = [content[i][4:-1] for i in range(len(content)) if content[i][0] != '%']
content[2] = content[2][1:-3]
content[3] = content[3][1:-3]
content[4] = content[4][1:-3]
content[5] = content[5][1:-3]

n = int(content[1])
m = int(content[0])

l = [int(L) for L in content[2].split(',')]
s = [int(L) for L in content[3].split(',')]
x = [int(L) for L in content[4].split(',')]
y = [int(L) for L in content[5].split(',')]

x = x + [x[-1]]*(m-1)
y = y + [y[-1]]*(m-1)

N = n + m
ITEMS = list(range(n))
NODES = list(range(N))
COURIERS = list(range(m))

D = np.array([[0 for j in range(N)] for i in range(N)])

for i in range(N):
    for j in range(N):
        if i > n and j > n:
            D[i, j] = 0
        elif i > n:
            D[i, j] = abs(x[n] - x[j]) + abs(y[n] - y[j])
        elif j > n:
            D[i, j] = abs(x[i] - x[n]) + abs(y[i] - y[n])
        else:
            D[i, j] = abs(x[i] - x[j]) + abs(y[i] - y[j])

model = cp_model.CpModel()

# VARIABLES
table = []
for i in NODES:
    rows = []
    for j in NODES:
        rows.append(model.NewBoolVar(name=f't[{i},{j}]'))
    table.append(rows)

# VARIABLES
courier = []
for k in COURIERS:
    rows = []
    for i in ITEMS:
        rows.append(model.NewBoolVar(name=f'courier[{k},{i}]'))
    courier.append(rows)

# ------ initialization constraints ----

# Ongi oggetto raggiunto esattamente una volta
for i in ITEMS:
    model.AddExactlyOne(table[i][j] for j in NODES)
    model.AddExactlyOne(table[j][i] for j in NODES)

for i in ITEMS:
    model.Add(table[i][i] == False)

for i in NODES:
    for j in NODES:
        if i != j:
            model.Add(table[i][j] == False).OnlyEnforceIf(table[j][i])

for i in range(n, N):
    for j in range(n, N):
        if i != j:
            model.Add(table[i][j] == False)

for i in ITEMS:
    model.AddExactlyOne(courier[k][i] for k in COURIERS)

for i in range(n, N):
    for j in ITEMS:
        model.Add(courier[i-n][j] == True).OnlyEnforceIf(table[i][j])

for k in COURIERS:
    for i in ITEMS:
        for j in ITEMS:
            model.Add(courier[k][i] == courier[k][j]).OnlyEnforceIf(table[i][j])

for k in COURIERS:
    model.Add(sum(courier[k][j]*s[j] for j in ITEMS) <= l[k])

"""
# SYMMETRY BREAKING
for i in range(n, N):
    model.Add(sum(table[i][j]*(j+1)*(j+1) for j in ITEMS) >= sum(table[j][i]*(j+1)*(j+1) for j in ITEMS))

for j in range(n, N):
    for jj in range(j+1, N):
        model.Add(sum(table[i][j]*(i+1)*(i+1) for i in ITEMS) <= sum(table[i][jj]*(i+1)*(i+1) for i in ITEMS))
"""

for k in COURIERS:
    for kk in range(k+1, m):
        if l[k] == l[kk]:
            model.Add(sum(courier[k][j]*(j+1)*(j+1) for j in ITEMS) <= sum(courier[kk][j]*(j+1)*(j+1) for j in ITEMS))
"""
        if l[k] >= l[kk]:
            model.Add(sum(courier[k][j]*s[j] for j in ITEMS) >= sum(courier[kk][j]*s[j] for j in ITEMS))
"""

arcs = []
for i in NODES:
    for j in NODES:
        arcs.append((i, j, table[i][j]))
model.AddCircuit(arcs)

obj = cp_model.LinearExpr.Sum([table[i][j]*D[i, j] for i in NODES for j in NODES])
model.Add(obj >= lower_boud)

model.Minimize(obj)
solver = cp_model.CpSolver()
# Massimo 5 minuti
solver.parameters.max_time_in_seconds = 300.0

solver.num_search_workers = 4
print("Start solving...")
status = solver.Solve(model)
print(status)

if status == cp_model.OPTIMAL or status == cp_model.FEASIBLE:

    print(f"\nPlan:\n")
    for i in NODES:
        for j in NODES:
            print(f'{solver.Value(table[i][j])}', end=" ")
        print("")
    print("--------------------------------------")

    for k in COURIERS:
        print(f"\nCourier{k}:", end=" ")
        for i in ITEMS:
            print(f'{solver.Value(courier[k][i])}', end=" ")

    print(f'\nMinimum of objective function: {solver.ObjectiveValue()}\n')
    for k in COURIERS:
        somma = 0
        for j in ITEMS:
            somma += solver.Value(courier[k][j])*s[j]
        print("sum[{}] = {} and l[{}] = {}".format(k, somma, k, l[k]))
    for i in range(len(x)):
        plt.gca().add_patch(plt.Circle((x[i],y[i]), 0.1, color='r'))
    x1s = []
    y1s = []
    x2s = []
    y2s = []
    color_i = 1
    color = 'C1'
    for k in range(m):
        x1s.clear()
        y1s.clear()
        x2s.clear()
        y2s.clear()
        for i in range(N):
            for j in range(N):
                if i != j:
                    if solver.Value(table[i][j]):
                        x1s.append(x[i])
                        y1s.append(y[i])
                        x2s.append(x[j]-x[i])
                        y2s.append(y[j]-y[i])
        plt.quiver(x1s, y1s, x2s, y2s, angles='xy', scale_units='xy', color=[color], scale=1, width=0.003)
        color_i += 1
        color = 'C'+str(color_i)
    plt.xlim(min(x)-2,max(x)+2)
    plt.ylim(min(y)-2,max(y)+2)
    plt.show()
else:
    print('No solution found.')

"""
plt.scatter(x[:-1], y[:-1], c='b')
for i in range(n):
    plt.annotate(f'q{i}', (x[i]+0.05, y[i]+0.05))

for k in COURIERS:
    for i in NODES:
        for j in NODES:
            if solver.Value(table[k][i][j]) == 1:
                plt.plot([x[i], x[j]], [y[i], y[j]], c='y', alpha=0.4)

plt.plot(x[-1], y[-1], c='r', marker='s')
plt.show()
"""


from scipy.signal import butter, filtfilt
import scipy.io as sio
from numpy.fft import rfftfreq, rfft
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import freqz
from PIL import Image
import matplotlib.ticker as ticker

def butter_bandpass_filter(data, lowcut, highcut, fs, order=4):
    b, a = butter(N=order, Wn=[lowcut, highcut], fs=fs, btype='band')
    y = filtfilt(b, a, data)
    return y

def item_i(fs, lowcut, highcut):
    plt.figure(1)
    plt.clf()
    order = 4
    b, a = butter(N=order, Wn=[lowcut, highcut], fs=fs, btype='band')
    w, h = freqz(b, a, fs=fs, worN=2000)
    plt.plot(w / (10 ** 6), abs(h), label="order = %d" % order)
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Gain')
    plt.grid(True)
    plt.legend(loc='best')
    plt.show()

def _call_plot_stuff_ii():
    plt.grid()
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Amplitude')
    plt.legend(loc='upper right')

def item_ii(rf, fs, lowcut, highcut, line=100):
    N = rf.shape[0]
    frequencies = rfftfreq(N, 1. / fs) / (10 ** 6)
    d_sig = rf[:, line]
    one_sided_power_spectrum = abs(rfft(d_sig))
    filtered_signal = butter_bandpass_filter(d_sig, lowcut, highcut, fs, order=4)
    one_sided_filtered = abs(rfft(filtered_signal))
    plt.subplot(3, 1, 1)
    plt.plot(frequencies, one_sided_power_spectrum, label='Original signal')
    plt.title('Signal power spectrum for a line')
    _call_plot_stuff_ii()

    plt.subplot(3, 1, 2)
    plt.plot(frequencies, one_sided_filtered, label='Filtered signal')
    _call_plot_stuff_ii()

    plt.subplot(3, 1, 3)
    plt.plot(frequencies, one_sided_power_spectrum, label='Original signal')
    plt.plot(frequencies, one_sided_filtered, label='Filtered signal')
    _call_plot_stuff_ii()

    plt.show()

def _call_plot_stuff_iii():
    fig = plt.figure()
    ax = fig.add_subplot(111)
    scale_x = 20
    make_it_millimeters = ticker.FuncFormatter(lambda x, pos: '{0:g}'.format(x / scale_x))
    ax.xaxis.set_major_formatter(make_it_millimeters)
    ax.yaxis.set_major_formatter(make_it_millimeters)
    plt.xlabel('Lateral, millimeters')
    plt.ylabel('Depth, millimeters')

def item_iii(rf, fs, lowcut, highcut):
    L = rf.shape[1]
    img = np.empty_like(rf)
    for line in range(L):
        img[:, line] = butter_bandpass_filter(rf[:, line], lowcut, highcut, fs, order=4)
    img = Image.fromarray(img).resize((400, 600))
    mine, maxe = np.min(img), np.max(img)
    _call_plot_stuff_iii()
    plt.title('Raw resulting image')
    plt.imshow(img, cmap='gray')
    plt.show()

    print(np.min(img), np.mean(img), np.max(img))
    img = (255 * (img - mine) / (maxe - mine)).astype(int)
    print(np.min(img), np.mean(img), np.max(img))
    _call_plot_stuff_iii()
    plt.title('Normalized to 0-255')
    plt.imshow(img, cmap='gray')
    plt.show()

if __name__ == "__main__":
    # settings

    rf_file = sio.loadmat('./data/mousekidney.mat')['RawRF']
    fs = 500 * 10 ** 6
    lowcut = 5 * 10 ** 6
    highcut = 50 * 10 ** 6

    # item calls

    # item_i(fs=fs, lowcut=lowcut, highcut=highcut)
    # item_ii(rf=rf_file, fs=fs, lowcut=lowcut, highcut=highcut)
    item_iii(rf=rf_file, fs=fs, lowcut=lowcut, highcut=highcut)


"""
Created on 23:07, Apr. 22nd, 2021
Author: fassial
Filename: main.py
"""
import os
import copy
import pickle
import numpy as np
import brainpy as bp
# local dep
import model
import stimulus

# macro
DIR_ROOT = os.getcwd()
DIR_FIGS = os.path.join(DIR_ROOT, "figs")
if not os.path.exists(DIR_FIGS): os.mkdir(DIR_FIGS)
DIR_OUTPUTS = os.path.join(DIR_ROOT, "outputs")
if not os.path.exists(DIR_OUTPUTS): os.mkdir(DIR_OUTPUTS)
DIR_OUTPUTS_STIM = os.path.join(DIR_OUTPUTS, "stimulus")
if not os.path.exists(DIR_OUTPUTS_STIM): os.mkdir(DIR_OUTPUTS_STIM)
DIR_OUTPUTS_SPIKE = os.path.join(DIR_OUTPUTS, "spike")
if not os.path.exists(DIR_OUTPUTS_SPIKE): os.mkdir(DIR_OUTPUTS_SPIKE)

## default params
# default stim_params
default_stim_params = {
    "ipRGC": {
        "white": stimulus.stim_params(
            name = "white",
            height = 100,
            width = 1,
            duration = 1000,
            others = {
                "noise": .0,
            }
        ),
        "black": stimulus.stim_params(
            name = "black",
            height = 100,
            width = 1,
            duration = 1000,
            others = {
                "noise": 0.,
            }
        ),
        "normal": stimulus.stim_params(
            name = "normal",
            height = 100,
            width = 1,
            duration = 1000,
            others = {
                "freqs": np.full((100,), 20., dtype = np.float32),
                "noise": 0.,
            }
        ),
        "frate_increase": stimulus.stim_params(
            name = "frate_increase",
            height = 100,
            width = 1,
            duration = 1000,
            others = {
                "freqs": np.full((100,), 20., dtype = np.float32),
                "factor": 4.,   # (1,16)
                "ratio": .2,
                "noise": 0.,
            }
        ),
    },
    "PAC": {
        "white": stimulus.stim_params(
            name = "white",
            height = 50,
            width = 1,
            duration = 1000,
            others = {
                "noise": .0,
            }
        ),
        "black": stimulus.stim_params(
            name = "black",
            height = 50,
            width = 1,
            duration = 1000,
            others = {
                "noise": 0.,
            }
        ),
        "normal": stimulus.stim_params(
            name = "normal",
            height = 50,
            width = 1,
            duration = 1000,
            others = {
                "freqs": np.full((50,), 20., dtype = np.float32),
                "noise": 0.,
            }
        ),
        "frate_increase": stimulus.stim_params(
            name = "frate_increase",
            height = 50,
            width = 1,
            duration = 1000,
            others = {
                "freqs": np.full((50,), 20., dtype = np.float32),
                "factor": 4.,   # (1,16)
                "ratio": .2,
                "noise": 0.,
            }
        ),
    },
}
# default net_params
default_net_params = {
    "ipRGC": {
        ## neurons params
        # shape params
        "size": (100,),
        # dynamic params
        "V_init": "gaussian",
        "tau": 5.,  # tau > t_refractory
        "t_refractory": 1.,
        "noise": .2,
    },
    "PAC": {
        ## neurons params
        # shape params
        "size": (50,),
        # dynamic params
        "V_init": "gaussian",
        "tau": 5.,
        "t_refractory": 1.,
        "noise": .2,
    },
    "GJ_RP": {
        # gap junction
        "neighbors": 3,
        "weight": .3,
        "k_spikelet": .1,
        "conn": model.connector.IndexConnector(),
    },
    "ES_RP": {
        # exp synapses
        "neighbors": 5,
        "weight": .5,
        "delay": .5,    # random dists
        "tau": .5,
        "conn": model.connector.IndexConnector(),
    },
}

def main(dt = 0.01):
    # init seed & expr_curr
    np.random.seed(0)
    expr_curr = "white"
    # init backend
    bp.backend.set(dt = dt)
    bp.backend.set(backend = "numba")
    model.set_backend(backend = "numba")

    ## get stim
    # get stim_fname
    stim_iprgc_fname = os.path.join(
        DIR_OUTPUTS_STIM,
        expr_curr + "-" + str(default_stim_params["ipRGC"][expr_curr].duration) + "-iprgc" + ".csv"
    )
    stim_pac_fname = os.path.join(
        DIR_OUTPUTS_STIM,
        expr_curr + "-" + str(default_stim_params["PAC"][expr_curr].duration) + "-pac" + ".csv"
    )
    # get stim_iprgc
    if os.path.exists(stim_iprgc_fname):
        # load stim
        stim_iprgc = np.loadtxt(
            fname = stim_iprgc_fname,
            delimiter = ","
        )
    else:
        # get stim
        stim_iprgc, _ = stimulus.stimulus.get(
            stim_params = default_stim_params["ipRGC"][expr_curr]
        )
        # save stim
        np.savetxt(fname = stim_iprgc_fname, X = stim_iprgc, delimiter = ",")
    # get stim_pac
    if os.path.exists(stim_pac_fname):
        # load stim
        stim_pac = np.loadtxt(
            fname = stim_pac_fname,
            delimiter = ","
        )
    else:
        # get stim
        stim_pac, _ = stimulus.stimulus.get(
            stim_params = default_stim_params["PAC"][expr_curr]
        )
        # save stim
        np.savetxt(fname = stim_pac_fname, X = stim_pac, delimiter = ",")
    # rescale stim
    stim_pac *= .9; stim_iprgc *= .9

    ## exec sim
    # inst RPNet
    net = model.RPNet(net_params = default_net_params, run_params = {
        "inputs": {
            "ipRGC": stim_iprgc,
            "PAC": stim_pac,
        },
        "dt": 0.01,
        "duration": default_stim_params["ipRGC"][expr_curr].duration,
    })
    # net run
    net.run(report = True)
    # show net.mon
    net_monitors = net.get_monitors()
    net.show(img_fname = os.path.join(DIR_FIGS, expr_curr + ".png"))

if __name__ == "__main__":
    main()


import sys
sys.stdin = open("baek18404.txt")
from collections import deque

def bfs(k_row, k_col):

    visited = [[-1] * N for _ in range(0, N)]
    visited[k_row][k_col] = 0
    que = deque()
    que.append((k_row, k_col, 0))

    dr = [-1, -2, -2, -1, 1, 2, 2, 1]
    dc = [-2, -1, 1, 2, 2, 1, -1, -2]

    while que:
        row, col, path = que.popleft()

        for d in range(0, 8):
            next_row = row + dr[d]
            next_col = col + dc[d]

            if next_row < 0 or next_col < 0 or next_row >= N or next_col >= N:
                continue
            if visited[next_row][next_col] == -1:
                visited[next_row][next_col] = path+1
                que.append((next_row, next_col, path+1))
    return visited

N, M = map(int, input().split())
knight_row, knight_col = map(int, input().split())

stones_pos = [0] * M
for i in range(0, M):
    r, c = map(int, input().split())
    stones_pos[i] = [r-1, c-1]

res_visited = bfs(knight_row-1, knight_col-1)
answer = [0] * M
for i in range(0, M):
    row, col = stones_pos[i]
    answer[i] = res_visited[row][col]
print(" ".join(map(str, answer)))


#encoding=utf-8

from scrapy.selector import HtmlXPathSelector
from scrapy.spider import BaseSpider
from scrapy.contrib.loader import XPathItemLoader
from scrapy.http import Request
import re

from douban_crawler.items import GroupUserItem
from douban_crawler.init import get_start_urls

class GroupSpider(BaseSpider):
    name = "group"
    allowed_domains = ["douban.com"]
    start_urls = get_start_urls("group_user")

    def parse(self, response):
        url = response.url
        group_name = url[url.find('group'):].split('/')[1]
        hxs = HtmlXPathSelector(response)

        dls = hxs.select('//dl[@class="obu"]')
        items = []
        for dl in dls:
            item = GroupUserItem()
            l = XPathItemLoader(item=item, selector=dl)
            l.add_xpath('homepage', 'dt/a/@href')
            l.add_xpath('image', 'dt/a/img/@src')
            l.add_xpath('name', 'dd/a/text()')
            l.add_value('group', group_name)
            yield l.load_item()

        links = hxs.select('//span[@class="next"]/a/@href').extract()
        for url in links:
            yield Request(url, callback=self.parse)
        if len(links) < 1:
            p = re.compile('.*?',re.S)
            m = p.search(response.body_as_unicode())
            if m:
                url = m.group(1)
                yield Request(url, callback=self.parse)


class Solution(object):
    def isValidSudoku(self, board):
        """
        :type board: List[List[str]]
        :rtype: bool
        """
        seen = sum(([(c, i), (j, c), (i/3, j/3, c)]
                for i in range(9) for j in range(9)
                for c in [board[i][j]] if c != '.'), [])
        return len(seen) == len(set(seen))


import sys

n = int(sys.stdin.readline())
matrix = [list(map(int, sys.stdin.readline().split())) for _ in range(n)]
ans = [[0]*n for _ in range(n)]
graph = [[] for _ in range(n+1)]

for i in range(n):
    for j in range(n):
        if matrix[i][j] == 1:
            graph[i+1].append(j+1)

def dfs(i):
    stack = [i]
    vdfs = [0]*(n+1)

    while stack:
        node = stack.pop()
        if vdfs[node] == 0:
            vdfs[node] = 1
            stack.extend(graph[node])
            for j in graph[node]:
                ans[i-1][j-1] = 1

for a in range(1, n+1):
    dfs(a)

for b in range(n):
    for c in range(n):
        print(ans[b][c], end=" ")
    print()


def calc(arr1, arr2):
    value = 0
    for i in range(len(arr1)):
        value += arr1[i] * arr2[i]
    return value

if __name__ == "__main__":
    good = [1, 2, 3, 3, 4, 10]
    evil = [1, 2, 2, 2, 3, 5, 10]
    for i in range(1, int(input()) + 1):
        t_good = list(map(int, input().split()))
        t_evil = list(map(int, input().split()))
        v_good = calc(good, t_good)
        v_evil = calc(evil, t_evil)
        if v_good == v_evil:
            print(f"Battle {i}: No victor on this battle field")
        elif v_good < v_evil:
            print(f"Battle {i}: Evil eradicates all trace of Good")
        else:
            print(f"Battle {i}: Good triumphs over Evil")


from collections import deque

def yosqp():
    for i in range(1, K + 1):
        temp = queue.popleft()
        if i != K:
            queue.append(temp)
        else:
            if queue:
                print(temp, end=", ")
            else:
                print(temp, end=">")

if __name__ == '__main__':
    N, K = map(int, input().split())
    queue = deque([i for i in range(1, N + 1)])
    print("<", end="")
    while queue:
        yosqp()

Solved/자료구조/리스트/조세퍼드문제.py


def attr_path(name):
    return (
        "./AdditionalAttributes/AdditionalAttribute"
        f"[Name='{name}']/Values/Value"
    )

def get_field_paths():
    paths = {
        'absoluteOrbit':            "./OrbitCalculatedSpatialDomains/OrbitCalculatedSpatialDomain/OrbitNumber",
        'ascendingNodeTime':        attr_path('ASC_NODE_TIME'),
        'baselinePerp':             attr_path('INSAR_BASELINE'),
        'beamMode':                 attr_path('BEAM_MODE_TYPE'),
        'beamModeType':             attr_path('BEAM_MODE_TYPE'),
        'bytes':                    attr_path("BYTES"),
        'centerLat':                attr_path('CENTER_LAT'),
        'centerLon':                attr_path('CENTER_LON'),
        'collectionName':           attr_path('MISSION_NAME'),
        'configurationName':        attr_path('BEAM_MODE_DESC'),
        'doppler':                  attr_path('DOPPLER'),
        'downloadUrl':              "./OnlineAccessURLs/OnlineAccessURL/URL",
        'farEndLat':                attr_path('FAR_END_LAT'),
        'farEndLon':                attr_path('FAR_END_LON'),
        'farStartLat':              attr_path('FAR_START_LAT'),
        'farStartLon':              attr_path('FAR_START_LON'),
        'faradayRotation':          attr_path('FARADAY_ROTATION'),
        'finalFrame':               attr_path('CENTER_ESA_FRAME'),
        'firstFrame':               attr_path('CENTER_ESA_FRAME'),
        'flightDirection':          attr_path('ASCENDING_DESCENDING'),
        'flightLine':               attr_path('FLIGHT_LINE'),
        'granuleName':              "./DataGranule/ProducerGranuleId",
        'granuleType':              attr_path('GRANULE_TYPE'),
        'groupID':                  attr_path('GROUP_ID'),
        'insarBaseline':            attr_path('INSAR_BASELINE'),
        'insarGrouping':            attr_path('INSAR_STACK_ID'),
        'insarStackSize':           attr_path('INSAR_STACK_SIZE'),
        'lookDirection':            attr_path('LOOK_DIRECTION'),
        'md5sum':                   attr_path('MD5SUM'),
        'missionName':              attr_path('MISSION_NAME'),
        'nearEndLat':               attr_path('NEAR_END_LAT'),
        'nearEndLon':               attr_path('NEAR_END_LON'),
        'nearStartLat':             attr_path('NEAR_START_LAT'),
        'nearStartLon':             attr_path('NEAR_START_LON'),
        'offNadirAngle':            attr_path('OFF_NADIR_ANGLE'),
        'pointingAngle':            attr_path('POINTING_ANGLE'),
        'polarization':             attr_path('POLARIZATION'),
        'processingDate':           "./DataGranule/ProductionDateTime",
        'processingDescription':    attr_path('PROCESSING_DESCRIPTION'),
        'processingLevel':          attr_path('PROCESSING_TYPE'),
        'processingType':           attr_path('PROCESSING_LEVEL'),
        'processingTypeDisplay':    attr_path('PROCESSING_TYPE_DISPLAY'),
        'productName':              "./DataGranule/ProducerGranuleId",
        'product_file_id':          "./GranuleUR",
        'relativeOrbit':            attr_path('PATH_NUMBER'),
        'sceneDate':                attr_path('ACQUISITION_DATE'),
        'sceneId':                  "./DataGranule/ProducerGranuleId",
        'sensor':                   './Platforms/Platform/Instruments/Instrument/ShortName',
        'sizeMB':                   "./DataGranule/SizeMBDataGranule",
        'startTime':                "./Temporal/RangeDateTime/BeginningDateTime",
        'stopTime':                 "./Temporal/RangeDateTime/EndingDateTime",
        'thumbnailUrl':             attr_path('THUMBNAIL_URL'),
        'track':                    attr_path('PATH_NUMBER'),
        'pgeVersion':               "./PGEVersionClass/PGEVersion",
        'additionalUrls':           "./OnlineAccessURLs",

        # BURST FIELDS
        'absoluteBurstID':          attr_path('BURST_ID_ABSOLUTE'),
        'relativeBurstID':          attr_path('BURST_ID_RELATIVE'),
        'fullBurstID':              attr_path('BURST_ID_FULL'),
        'burstIndex':               attr_path('BURST_INDEX'),
        'azimuthTime':              attr_path('AZIMUTH_TIME'),
        'azimuthAnxTime':           attr_path('AZIMUTH_ANX_TIME'),
        'samplesPerBurst':          attr_path('SAMPLES_PER_BURST'),
        'subswath':                 attr_path('SUBSWATH_NAME'),

        # OPERA RTC FIELDS
        'operaBurstID':                  attr_path('OPERA_BURST_ID'),
    }
    return paths


from pyecharts.charts import Line
from pyecharts.charts import Boxplot
import numpy as np
import pandas as pd
from pyecharts.globals import CurrentConfig, NotebookType
CurrentConfig.NOTEBOOK_TYPE = NotebookType.JUPYTER_LAB

import pyecharts.options as opts
from pyecharts.globals import ThemeType

city_name = ['beijing', 'shanghai', 'guangzhou', 'shenzhen']
cityes_PM = []
for i in range(4):
    filename = '..//csv//' + city_name[i] + '_AQI' + '_2018.csv'
    pm_data = pd.read_csv(filename)

    get_data = pm_data[['Date', 'PM']]
    month_for_data = []
    for j in get_data['Date']:
        time = j.split('/')[1]
        month_for_data.append(time)
    #   获取每行数据的月份
    pm_data['Month'] = month_for_data
    #   求每个月PM平均值
    month_data = pm_data.groupby(['Month'])
    month_PM = month_data['PM'].agg(['mean'])
    month_PM.reset_index(inplace=True)
    month_PM_average = month_PM.sort_index()
    #   获取每个城市月均PM的数据，转化为int数据类型
    month_PM_data = np.array(month_PM_average['mean'])
    month_PM_data_int = [int(i) for i in month_PM_data]
    cityes_PM.append(month_PM_data_int)
months = ['{}'.format(str(i) + '月') for i in range(1, 13)]

line = (
    Line(init_opts=opts.InitOpts(theme=ThemeType.DARK))
        .add_xaxis(months)
        .add_yaxis("北京", cityes_PM[0])
        .add_yaxis("上海", cityes_PM[1])
        .add_yaxis("广州", cityes_PM[2])
        .add_yaxis("深圳", cityes_PM[3])
        .set_series_opts(label_opts=opts.LabelOpts(is_show=False))
        .set_global_opts(title_opts=opts.TitleOpts(title='2018年北上广深PM2.5全年走势图'),
                         legend_opts=opts.LegendOpts(pos_top='8%')
                         )
)
line.render("..//HTML//8.4.5多个数据源2.html")

大数据可视化技术/py/8.4.5多个数据源2.py


import os.path
import os
import sys

import config

# функция убирает из файлов коментарии и пустые строки
def prepare(namefile, newfile):
    f = open(namefile)
    fn = open(newfile, 'w', encoding='utf-8')
    while True:
        string = f.readline()
        if string == '':
            break
        elif string[0] == '#' or string == '\n':
            continue
        else:
            fn.write(string)
    f.close()
    fn.close()

# закрывает файлы
def close_file(*args):
    for i in args:
        i.close()

def check(service, old, new, new_file_name):
    name_log = '{}.log'.format(service)
    log = open(name_log, 'a', encoding='utf-8')
    fold = open(old)
    fnew = open(new)
    log = open(name_log, 'a', encoding='utf-8')
    while True:
        old_str = fold.readline()
        new_str = fnew.readline()
        if old_str == '' or new_str == '':
            log.write('PASS {}\n'.format(new_file_name))
            close_file(fold, fnew, log)
            os.remove(old)
            os.remove(new)
            break
        elif old_str == new_str:
            pass
        else:
            log.write('FAIL {}\n'.format(new_file_name))
            close_file(fold, fnew, log)
            os.remove(old)
            os.remove(new)
            break

if __name__ == '__main__':

    service_name = sys.argv[1]
    files = 'config.{}'.format(service_name)
    print(files)
    for i in range(len(eval(files)['old'])):
        old_path, old_file = os.path.split(eval(files)['old'][i+1])
        new_path, new_file = os.path.split(eval(files)['new'][i+1])
        prepare('{}/{}'.format(old_path, old_file), 'oldcache.txt')
        prepare('{}/{}'.format(new_path, new_file), 'newcache.txt')
        check(service_name, 'oldcache.txt', 'newcache.txt', new_file)


# -*- coding: utf-8 -*-
import html
import re
import json
import time
import scrapy
import requests
from hashlib import md5

from overseaSpider.util.utils import isLinux
from overseaSpider.items import ShopItem, SkuAttributesItem, SkuItem

website = 'bintiva'
# 全流程解析脚本

class BintivaSpider(scrapy.Spider):
    name = website
    allowed_domains = ['bintiva.com']
    start_urls = ['https://bintiva.com/']

    @classmethod
    def update_settings(cls, settings):
        # settings.setdict(getattr(cls, 'custom_debug_settings' if getattr(cls, 'is_debug', False) else 'custom_settings', None) or {}, priority='spider')
        custom_debug_settings = getattr(cls, 'custom_debug_settings' if getattr(cls, 'is_debug', False) else 'custom_settings', None)
        system = isLinux()
        if not system:
            # 如果不是服务器, 则修改相关配置
            custom_debug_settings["HTTPCACHE_ENABLED"] = True
            # custom_debug_settings["HTTPCACHE_DIR"] = "/Users/cagey/PycharmProjects/mogu_projects/scrapy_cache"
            custom_debug_settings["MONGODB_SERVER"] = "127.0.0.1"
        settings.setdict(custom_debug_settings or {}, priority='spider')

    def __init__(self, **kwargs):
        super(BintivaSpider, self).__init__(**kwargs)
        self.counts = 0
        setattr(self, 'author', "守约")

    is_debug = True
    custom_debug_settings = {
        'MONGODB_COLLECTION': website,
        'CONCURRENT_REQUESTS': 4,
        'DOWNLOAD_DELAY': 1,
        'LOG_LEVEL': 'DEBUG',
        'COOKIES_ENABLED': False,
        'HTTPCACHE_ENABLED': True,
         # 'HTTPCACHE_EXPIRATION_SECS': 7 * 24 * 60 * 60, # 秒
        'DOWNLOADER_MIDDLEWARES': {
            #'overseaSpider.middlewares.PhantomjsUpdateCookieMiddleware': 543,
            #'overseaSpider.middlewares.OverseaspiderProxyMiddleware': 400,
            'overseaSpider.middlewares.OverseaspiderUserAgentMiddleware': 100,
        },
        'ITEM_PIPELINES': {
            'overseaSpider.pipelines.OverseaspiderPipeline': 300,
        },
        'HTTPCACHE_POLICY': 'overseaSpider.middlewares.DummyPolicy',
    }

    def start_requests(self):
        url_list = [
            'https://bintiva.com/',
        ]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
            )

        #url = "http://bintiva.com/"
        #yield scrapy.Request(
        #    url=url,
        #)

    def parse(self, response):
        url_list = response.xpath("//nav[@class='navPages']//a/@href").getall()
        url_list.pop(0)
        url_list = [response.urljoin(url) for url in url_list]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
                callback=self.parse_list,
            )

    def parse_list(self, response):
        """列表页"""
        url_list = response.xpath("//ul[@class='productGrid productGrid--maxCol3']/li//figure/a/@href").getall()
        url_list = [response.urljoin(url) for url in url_list]
        for url in url_list:
            print(url)
            yield scrapy.Request(
                url=url,
                callback=self.parse_detail,
            )

        # response_url = parse.unquote(response.url)
        # split_str = ''
        # base_url = response_url.split(split_str)[0]
        # page_num = int(response_url.split(split_str)[1])+1
        # next_page_url = base_url + split_str + str(page_num)
        next_page_url = response.xpath('//link[@rel="next"]/@href').get()
        if next_page_url:
           next_page_url = response.urljoin(next_page_url)
           print("下一页:"+next_page_url)
           yield scrapy.Request(
               url=next_page_url,
               callback=self.parse_list,
           )
    def filter_html_label(self, text):
        text = str(text)
        text = html.unescape(text)
        # 注释，js，css，html标签
        filter_rerule_list = [r'(<!--[\s\S]*?-->)', r'<script[\s\S]*?</script>', r'<style[\s\S]*?</style>', r'<[^>]+>']
        for filter_rerule in filter_rerule_list:
            html_labels = re.findall(filter_rerule, text)
            for h in html_labels:
                text = text.replace(h, ' ')
        filter_char_list = [
            u'\x85', u'\xa0', u'\u1680', u'\u180e', u'\u2000', u'\u200a', u'\u2028', u'\u2029', u'\u202f', u'\u205f',
            u'\u3000', u'\xA0', u'\u180E', u'\u200A', u'\u202F', u'\u205F', '\t', '\n', '\r', '\f', '\v',
        ]
        for f_char in filter_char_list:
            text = text.replace(f_char, '')
        text = re.sub(' +', ' ', text).strip()
        return text

    def filter_text(self, input_text):
        filter_list = [u'\x85', u'\xa0', u'\u1680', u'\u180e', u'\u2000-', u'\u200a',
                       u'\u2028', u'\u2029', u'\u202f', u'\u205f', u'\u3000', u'\xA0', u'\u180E',
                       u'\u200A', u'\u202F', u'\u205F']
        for index in filter_list:
            input_text = input_text.replace(index, "").strip()
        return input_text

    def parse_detail(self, response):
        """详情页"""
        items = ShopItem()
        items["url"] = response.url
        # price = re.findall("", response.text)[0]
        original_price = response.xpath("//div[@itemprop='offers']/span[@class='price price--withoutTax price--main']/text()").get()
        original_price=original_price.split("$")[-1]
        current_price = response.xpath("//div[@itemprop='offers']/following-sibling::div[@class='price-section rrp-price--withoutTax price-section--withoutTax']/span/text()").get()
        current_price=self.filter_text(self.filter_html_label(current_price))
        if current_price:
            current_price=current_price.split("$")[-1]
        else:
            current_price=original_price
        items["original_price"] = "" + str(original_price) if original_price else "" + str(current_price)
        items["current_price"] = "" + str(current_price) if current_price else "" + str(original_price)
        items["brand"] = response.xpath("//h5[@class='productView-brand']/a/span/text()").get()
        items["name"] = response.xpath("//h1[@class='productView-title']//text()").get()
        # attributes = list()
        # items["attributes"] = attributes
        # items["about"] = response.xpath("").get()
        des=response.xpath('//section[@class="productView-desc productView-desc--full"]//text()').getall()
        des=' '.join(des)
        des=self.filter_text(self.filter_html_label(des))
        items["description"] = des

        items["source"] = '	bintiva.com'
        images_list = response.xpath("//div[@class='slick-track']/li/a/@href").getall()
        items["images"] = images_list

        Breadcrumb_list = response.xpath("//ul[@class='breadcrumbs']/li/a/span/text()").getall()
        br_list='/'.join(Breadcrumb_list)
        items["cat"] = Breadcrumb_list[-1]
        items["detail_cat"] = br_list

        label_list=response.xpath("//div[@class='productView-optionsGrid']/div/div/label")
        label_list.pop(0)
        color_list=[]
        for label in label_list:
            color=label.xpath("./span/@title").get()
            color_list.append({'color':color})

        sku_list = list()

        for sku in color_list:
            sku_item = SkuItem()
            sku_item["original_price"] = items["original_price"]
            sku_item["current_price"] = items["current_price"]

            sku_item["url"] = response.url

            attributes = SkuAttributesItem()
            attributes["colour"] = sku["color"]

            sku_item["attributes"] = attributes
            sku_list.append(sku_item)

        items["sku_list"] = sku_list
        items["measurements"] = ["Weight: None", "Height: None", "Length: None", "Depth: None"]
        status_list = list()
        status_list.append(items["url"])
        status_list.append(items["original_price"])
        status_list.append(items["current_price"])
        status_list = [i for i in status_list if i]
        status = "-".join(status_list)
        items["id"] = md5(status.encode("utf8")).hexdigest()

        items["lastCrawlTime"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
        items["created"] = int(time.time())
        items["updated"] = int(time.time())
        items['is_deleted'] = 0

        print(items)
        # yield items


"""initial migration

Revision ID: 004b53756417
Revises: 71f9651db5d6
Create Date: 2017-04-21 22:44:54.798195

"""

# revision identifiers, used by Alembic.
revision = '004b53756417'
down_revision = '71f9651db5d6'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('roles', sa.Column('rolename', sa.String(length=64), nullable=True))
    op.drop_index('name', table_name='roles')
    op.create_unique_constraint(None, 'roles', ['rolename'])
    op.drop_column('roles', 'name')
    op.add_column('users', sa.Column('e_mail', sa.String(length=64), nullable=True))
    op.drop_index('email', table_name='users')
    op.create_unique_constraint(None, 'users', ['e_mail'])
    op.drop_column('users', 'email')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('email', mysql.VARCHAR(length=64), nullable=True))
    op.drop_constraint(None, 'users', type_='unique')
    op.create_index('email', 'users', ['email'], unique=True)
    op.drop_column('users', 'e_mail')
    op.add_column('roles', sa.Column('name', mysql.VARCHAR(length=64), nullable=True))
    op.drop_constraint(None, 'roles', type_='unique')
    op.create_index('name', 'roles', ['name'], unique=True)
    op.drop_column('roles', 'rolename')
    ### end Alembic commands ###


import numpy as np
import os
import torch

def compute_apfd(true_labels, predicted_labels):
    n = len(true_labels)

    potential_faults = []
    fault_positions = []

    for i, (true_label, predicted_label) in enumerate(zip(true_labels, predicted_labels)):
        true_label = true_label[0]
        predicted_label = predicted_label[0]
        if true_label != predicted_label and [true_label, predicted_label] not in potential_faults:
            fault_positions.append(i)
            potential_faults.append([true_label, predicted_label])

    num_faults = len(potential_faults)
    print(num_faults)
    apfd = 1 - (sum(fault_positions) / (num_faults * n)) + 1 / (2 * n)
    return apfd * 100

dataset = 'cifar10'
model = 'resnet20'
pred_labels = np.flip(np.load('./neuron_sensitive_samples_benign_' + dataset + '_' + model + '_pred_labels.npy')[:,None])[:500]
true_labels = np.flip(np.load('./neuron_sensitive_samples_benign_' + dataset + '_' + model + '_labels.npy'))[:500]
print(pred_labels.shape, true_labels.shape)
print('APFD:', compute_apfd(true_labels, pred_labels))


import benjmark

keys = ["cpp", "geex", "java", "clojure"];
#keys = ["geex", "java"];

circle_root = "../benchmarks/circle";

circle_settings = benjmark.default_settings.set('outputprefix', '../latex/images/benchmarks/circle').set('sizeformat', '{:d} points').set('xlabel', 'Number of points').set('ylabel', 'Computation time (seconds)');

benjmark.render_barplots(keys, circle_root, circle_settings);

benjmark.render_lineplot(keys, circle_root, circle_settings);


from threading import Thread
import queue
import cv2
from time import sleep
from pyzbar.pyzbar import decode
import re
from datetime import datetime

class ImageCaption:
    def __init__( self, tello, queue, max_height, log, cam_stop_queue ):
        # tello instance
        self.tello = tello
        # Queue for communication
        self.controller_comm = queue
        self.stop_queue = cam_stop_queue
        self.src_thread = Thread( target=self.routine, args=() )
        self.max_height = max_height
        self.log = log

    def thread_init( self ):
        self.src_thread.start()
        self.src_thread.join()

    def routine( self ):
        end = False
        while not end:
            qr_found = False
            retries = 0

            print("[INFO] ImageCaption: Waiting message from Controller")
            self.log.print("INFO","ImageCaption","Waiting message from Controller")
            # espera pasiva por mensaje desde controlador
            self.controller_comm.get()
            print("[INFO] ImageCaption: Message taken from Controller")
            self.log.print("INFO","ImageCaption","Message taken from Controller")

            # inicia toma de video
            self.tello.streamon()

            while not qr_found:
                # se revisa si hay que detener la ejecucion del hilo
                if not (self.stop_queue.empty()):
                    self.controller_comm.put("STOP")
                    self.log.print("INFO","ImageCaption","Received a stop event")
                    exit(0)

                # se empieza a buscar el QR
                instruction = self.search_qr()
                # se chequea la instruccion decodificada
                value = self.check_instruction( instruction )
                if value == 1: # instruccion correcta
                    self.controller_comm.put( instruction )
                    qr_found = True
                elif value == 2: # instruccion con END correcta
                    self.controller_comm.put( instruction.split( ",END" )[0] )
                    end = True
                    qr_found = True
                if retries < 1: # se reintenta encontrar el QR
                    retries = retries + 1
                else: # se desiste en buscar el QR
                    end = True
                    break
            sleep(1) # para esperar a que el controlador tome el mensaje primero
        if not qr_found:
            print("[INFO] ImageCaption: No QR found")
            self.log.print("INFO","ImageCaption","No QR found")
            self.controller_comm.put( "NOQR" )
        else:
            # se espera a que se completen los ultimos comandos
            self.controller_comm.get()
            self.controller_comm.put( "END" )

    def search_qr( self ):
        decoded_instruction = None

        frame_reader = self.tello.get_frame_read()
        # se captura imagen
        img = frame_reader.frame

        #now = datetime.now()
        #current_time = now.strftime("%H:%M:%S")
        #cv2.imwrite("picture " + current_time + ".jpg", img)

        # se intenta decodificar algun QR
        decoded_instruction = self.try_to_decode( img )
        if decoded_instruction == None:
            # se empieza a buscar el QR hacia arriba
            decoded_instruction = self.look_up( frame_reader )
            if( decoded_instruction == None ):
                # se empieza a buscar el QR hacia abajo
                decoded_instruction = self.look_down( frame_reader )
        return decoded_instruction

    def look_up( self, frame_reader ):
        decoded_instruction = None
        # se obtiene la altura actual del dron
        curr_height = self.tello.get_height()
        # seguir subiendo mientras no se encuentre QR o no se sobrepase el limite
        while( self.max_height - curr_height > 40 and decoded_instruction == None ):
            print("[INFO] ImageCaption: current height: " + str(curr_height))
            self.log.print("INFO","ImageCaption","current height: "+str(curr_height))

            # el dron se eleva 20 centimetros
            self.tello.move_up( 20 )

            # se revisa si hay que detener la ejecucion del hilo
            if not(self.stop_queue.empty()):
                self.controller_comm.put("STOP")
                self.log.print("INFO","ImageCaption","Received a stop event")
                exit(0)

            # se captura imagen
            img = frame_reader.frame

            #now = datetime.now()
            #current_time = now.strftime("%H:%M:%S")
            #cv2.imwrite("picture " + current_time + ".jpg", img)

            # se intenta decodificar algun QR
            decoded_instruction = self.try_to_decode( img )

            # se actualiza altura actual
            curr_height = self.tello.get_height()
            sleep(1)
        return decoded_instruction

    def look_down( self, frame_reader ):
        decoded_instruction = None
        # se obtiene la altura actual
        curr_height = self.tello.get_height()
        # seguir bajando mientras no se aterrice o no se encuentre QR
        while( curr_height > 30 and decoded_instruction == None ):
            print("[INFO] ImageCaption: current height: " + str(curr_height))
            self.log.print("INFO","ImageCaption","current height: "+str(curr_height))

            # el dron desciende 20 centimetros
            self.tello.move_down( 20 )

            # se revisa si hay que detener la ejecucion del hilo
            if not(self.stop_queue.empty()):
                self.controller_comm.put("STOP")
                self.log.print("INFO","ImageCaption","Received a stop event")
                exit(0)

            # se captura imagen
            img = frame_reader.frame

            #now = datetime.now()
            #current_time = now.strftime("%H:%M:%S")
            #cv2.imwrite("picture " + current_time + ".jpg", img)

            # se intenta decodificar algun QR
            decoded_instruction = self.try_to_decode( img )

            # se actualiza altura actual
            curr_height = self.tello.get_height()
            sleep(1)
        return decoded_instruction

    def check_instruction( self, instruction ):
        if( instruction == None ):
            return 0
        elif( re.search( "^((([RLFUB]|R(R|L)):\d+,)+(([RLFUB]|R(R|L)):\d+))$",
            instruction ) ):
                return 1
        elif( re.search( "^(([RLFUB]|R(R|L)):\d+)$",
            instruction ) ):
                return 1
        elif( re.search( "^((([RLFUB]|R(R|L)):\d+,)+END)$",
            instruction ) ):
                return 2
        return 0

    def try_to_decode( self, image ):
        # se intenta decodificar algun QR
        dec_img = decode( image )
        if( len(dec_img) != 0 ):
            decoded_instruction = dec_img[0].data.decode( 'utf8' )
            # se chequea la instruccion decodificada
            if self.check_instruction( decoded_instruction ):
                print("[INFO] ImageCaption: QR found and decoded!")
                self.log.print("INFO","ImageCaption","QR found and decoded!")
                return decoded_instruction
        return None


from twarc import Twarc
import json
from pathlib import Path
import os
import gzip
import time
from tqdm import tqdm
from multiprocessing import Process, SimpleQueue

from utils import load_configs

def hydrate(cfs):
    jobs = []
    squeue = SimpleQueue()
    data_dirs = ['2020-01', '2020-02', '2020-03', '2020-04']

    # create folder here to avoid data racing
    if not os.path.isdir('hydrated'):
        os.makedirs('hydrated')

    for data_dir in data_dirs:
        full = os.path.join('hydrated', data_dir)
        print(full)
        if not os.path.isdir(full):
            os.makedirs(full)

    fc_process = Process(target=file_creator, args=(squeue, len(cfs),))
    jobs.append(fc_process)
    fc_process.daemon = True
    fc_process.start()

    for cf in cfs:
        # unpack keys
        account = cf['account']
        p = Process(target=hydrated_cycle, args=(account, squeue,))
        jobs.append(p)
        p.daemon = True
        p.start()

    [j.join() for j in jobs]

def file_creator(squeue, n_workers):
    data_dirs = ['2020-01', '2020-02', '2020-03', '2020-04']

    for data_dir in data_dirs:

        for p in Path("COVID-19-TweetIDs/" + data_dir).iterdir():

            if p.name.endswith(".txt"):
                partial_path = os.path.join(data_dir, p.name)
                print("inserting:", partial_path)
                squeue.put(partial_path)

    # n work done indicator
    for _ in range(n_workers):
        squeue.put("done")

def hydrated_cycle(account, squeue):
    twarc = Twarc(**account)

    while True:
        working_path = squeue.get()

        if(working_path == "done"):
            print('job done')

        gzip_path = os.path.join(
            "hydrated/",
            working_path.split("/")[0],
            working_path.split("/")[-1][:-4]+".jsonl.gz")

        print(gzip_path)

        hydrate_file(working_path, twarc, gzip_path)

def hydrate_file(id_file, twarc, gzip_path):
    print("hydrating ", id_file)

    # getting use the cached work
    if os.path.isfile(gzip_path):
        print("skipping jsonl file already exists: ", gzip_path)
        return

    # # set prograss bar totoal ids
    # f = open(id_file, 'rb')
    # f_gen = _reader_generator(f.raw.read)
    # total_ids = sum(buf.count(b'\n') for buf in f_gen)

    with gzip.open(gzip_path, 'w') as output:
        for tweet in twarc.hydrate(open("COVID-19-TweetIDs/"+id_file)):
            # simply write to a gzip file and remove pbar

            if(tweet['place']):
                if(tweet['place']["country_code"].strip() == 'AU'):
                    print(tweet['place']["country_code"], gzip_path)
                    output.write(json.dumps(tweet).encode('utf-8') + b"\n")

def _reader_generator(reader):
    b = reader(1024 * 1024)
    while b:
        yield b
        b = reader(1024 * 1024)

if __name__ == "__main__":
    cfs = load_configs()
    hydrate(cfs)


from django.conf.urls import patterns, include, url
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.views.generic.simple import direct_to_template

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'kippit.views.home', name='home'),
    # url(r'^kippit/', include('kippit.foo.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),

     url(r'^$','bookmarks.views.main_page'),

     url(r'^bookmarks/',include('bookmarks.urls')),

     url(r'^accounts/login/$','django.contrib.auth.views.login'),

     url(r'^logout/$','django.contrib.auth.views.logout',{'next_page': '/'}),

     url(r'^signup/$','user_registration.views.register_page'),

     url(r'^signup/success/$',direct_to_template,{'template':'sign_up_success.html'}),

     #urls for comments application

     url(r'^comments/',include('django.contrib.comments.urls')),
)

urlpatterns += staticfiles_urlpatterns()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2021/6/15 16:44
# @Author  : Lelsey
# @File    : base.py

import copy
import numpy as np
from all_nets.cong_net import Cong_net
from all_tools import utils

class BaseAlg:
    def __init__(self, net:Cong_net):
        """
        所有算法的基类，
        diagnose()  该函数进行拥塞链路的诊断
        evaluation() 该函数用于诊断结果的相关指标计算

        算法生成的推测信息：
        self.links_cong_inferred：list  # 诊断为拥塞的链路编号列表
        self.links_state_inferred：ndarray  # 诊断的链路拥塞的状态
        self.dr:float
        self.fpr:float
        self.f1_score:float
        :param net:  拥塞网络类
        """
        self.net = net

        # topo基本信息
        self.leaf_nodes = net.leaf_nodes  #叶子结点数组
        self.route_matrix = net.route_matrix  #路由矩阵
        self.num_links=net.num_links   #链路数量

        # 链路拥塞相关属性
        self.links_cong_pro = net.links_cong_pro  #链路失效概率
        self.loss_model = net.loss_model   #丢包率模型
        self.threshold = net.threshold  #丢包率门限

        # 链路相关信息
        self.links_state_true = net.links_state_true    #真实链路状态数组
        self.links_loss_rate = net.links_loss_rate      #真实链路丢包率数组
        self.links_cong_true = net.links_cong_true      #真实链路拥塞数组

        # 真实路径相关属性
        self.paths_loss_rate = net.paths_loss_rate    #真实路径丢包率数组
        self.paths_state_true = net.paths_state_true  #真实路径状态数组
        self.paths_cong_true =net.paths_cong_true     #真实拥塞路径数组
        self.paths_no_cong_true=net.paths_no_cong_true  #真实非拥塞路径数组

        # 观测的路径相关属性
        # self.paths_state_obv =copy.deepcopy(net.paths_state_obv )   #观测路径状态数组
        # self.paths_cong_obv=copy.deepcopy(net.paths_cong_obv)       #观测拥塞路径数组
        # self.paths_no_cong_obv=net.paths_no_cong_obv  #观测非拥塞路径数组
        #注意，目前将观测的路径状态设置为真实的路径状态
        self.paths_state_obv = copy.deepcopy(net.paths_state_true)  # 观测路径状态数组
        self.paths_cong_obv = copy.deepcopy(net.paths_cong_true)  # 观测拥塞路径数组
        self.paths_no_cong_obv = net.paths_no_cong_true  # 观测非拥塞路径数组

        #推测结果信息
        self.links_cong_inferred = []  # 诊断为拥塞的链路列表
        self.links_state_inferred = None  # 诊断的链路拥塞的状态
        self.dr = None
        self.fpr = None
        self.f1_score=None

    def diagnose(self):
        """
        拥塞链路诊断，由子类继承
        :return:
        """
        pass

    def evaluation(self):
        """
        拥塞链路评估方法1
        dr和fpr的计算方式1
        dr=(推测拥塞 & 真实拥塞)/(真实拥塞数量)
        fpr=(推测拥塞-真实拥塞)/(真实不拥塞数量)
        :return:
        """
        if np.any(self.net.links_state_true):  # 判断是否不存在拥塞链路

            self._dr = len(set(self.links_cong_inferred).intersection(set(self.links_cong_true))) \
                      / len(self.links_cong_true)
        else:
            self._dr = 1.0

        if len(self.links_cong_true) < self.num_links:  # 判断是否不存在正常链路
            self._fpr = len(set(self.links_cong_inferred).difference(set(self.links_cong_true))) \
                       / (self.num_links - len(self.links_cong_true))
        else:
            self._fpr = 0.0

        #计算f1_score
        self.f1_score=utils.cal_f1_score_xt(list(self.links_cong_true),list(self.links_cong_inferred))

        #计算链路的dr和fpr

        #统计链路的tp，fp，fn，tn

    def evaluation_1(self):
        """
        拥塞链路评估方法2
        dr和fpr的计算方式2
        dr=(推测拥塞 & 真实拥塞)/(真实拥塞数量)
        fpr=(推测拥塞-真实拥塞)/(推测拥塞的数量)
        :return:
        """
        if np.any(self.links_state_true):  # 判断是否不存在拥塞链路

            self._dr = len(set(self.links_cong_inferred).intersection(set(self.links_cong_true))) \
                      / len(self.links_cong_true)
        else:
            self._dr = 1.0

        # if len(self._links_congested) < self._num_links:  # 判断是否不存在正常链路
        if len(self.links_cong_inferred)!=0:
            self._fpr = len(set(self.links_cong_inferred).difference(set(self.links_cong_true))) \
                       / (len(set(self.links_cong_inferred)))
        else:
            self._fpr = 0.0

        self._f1_score = utils.cal_f1_score_xt(list(self.links_cong_true), list(self.links_cong_inferred))


import json,os
from tools_cpython import *

if os.name=='nt':
    copy_command="copy"
else:
    copy_command='cp'

# if not os.path.exists('tagged.json.bkup'):
#     print(os.popen('%s tagged.json tagged.json.bkup'%copy_command).read())

g=line_generator('bkup/tagged.json.old')

of=open("tagged.json","w",encoding='utf8')

for line in g:
    article=json.loads(line)
    sentences=article['sentences']

    new_standard_sentences=[]
    for sentence in sentences:
        index,parse,enhancedPlusPlusDependencies=sentence['index'],"NOT_IMPLEMENTED",sentence["enhancedPlusPlusDependencies"]
        dep=sentence["enhancedPlusPlusDependencies"]
        # sentenceStr=" ".join(map(lambda x:x[1]
        #                                 ,sorted([(k['index']
        #                                 ,k['word']+("/"+lac2penn[k['lac_pos']]) if lac2penn[k['lac_pos']] else "") for k in dep if k!=0])))

        tagged=sorted([{"index":w['index'],"word":w['word'],"type":w['lac_pos'],"start":w['characterOffsetBegin'],"length":w['characterOffsetEnd']-w['characterOffsetBegin']+1} for w in dep],key=lambda x:x['index'])
        tokenDict=call_stanford(tagged)

        for w in tagged:
            if w['type']=="w":
                tokenDict[str(w['index'])]=new_token(index=str(w['index']),word=w['word'],pos="PU",lac_pos="w",dep="-",head="0",begin=w["start"],end=w["start"]+w['length'])
        # for k in special_r:
        #     tokenDict[k] = new_token(index=str(k), word=special_r[k], pos=None, lac_pos=None, dep=None,
        #                                       head=None, begin=None, end=None)
        # tokenDict[0] = new_token(index='0', word="[ROOT]", pos=None, lac_pos=None, dep=None, head=None, begin=None,
        #                            end=None)

        ranked_token={}
        for i in sorted([int(k) for k in tokenDict]):
            ranked_token[str(i)]=tokenDict[str(i)]

        # for k in tokenDict:
        #     assert isinstance(k,int)
        #     break
        standard_sentence=new_sentence_repr(index=sentence['index'],tokenDict=ranked_token,manual_relations=sentence['manual_relations'])
        new_standard_sentences.append(standard_sentence)

        # print(ranked_token)

    newArticle={}

    newArticle['fact']=article['fact']
    newArticle['meta']=article['meta']
    newArticle['sentences']=new_standard_sentences

    of.write(json.dumps(newArticle,ensure_ascii=False)+"\n")