import csv
import math

from django.forms.models import BaseInlineFormSet
from django.contrib import admin
from django.conf import settings
from django.db.models import Sum, Count
from django.http import HttpResponse

from . import models
from meet.admin import MeetDependentAdmin
from registration.models import Level

from django.utils.translation import ugettext_lazy as _

LED_SIGN_CODES = """
    The following are special codes you can use to customize the LED sign display

    MODES (defaults to rotate)

    |MODE_ROTATE|
    |MODE_HOLD|
    |MODE_FLASH|
    |MODE_ROLL_UP|
    |MODE_ROLL_DOWN|
    |MODE_ROLL_LEFT|
    |MODE_ROLL_RIGHT|
    |MODE_WIPE_UP|
    |MODE_WIPE_DOWN|
    |MODE_WIPE_RIGHT|
    |MODE_SCROLL|
    |MODE_AUTO_MODE|
    |MODE_ROLL_IN|
    |MODE_ROLL_OUT|
    |MODE_WIPE_IN|
    |MODE_WIPE_OUT|
    |MODE_TWINKLE|
    |MODE_SPARKLE|
    |MODE_SNOW|
    |MODE_INTERLOCK|
    |MODE_SWITCH|

"""

def export_as_csv(self, request, queryset):
    """ Generic csv export admin action. """
    opts = self.model._meta
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename={}.csv'.format(str(opts).replace('.', '_'))
    writer = csv.writer(response)
    field_names = [field.name for field in opts.fields]
    # Write a first row with header information
    writer.writerow(field_names)
    # Write data rows
    for obj in queryset:
        writer.writerow([getattr(obj, field) for field in field_names])
    return response
export_as_csv.short_description = "Export selected objects as csv file"

# Filters

class SessionFilter(admin.SimpleListFilter):
    # Human-readable title which will be displayed in the
    # right admin sidebar just above the filter options.
    title = _('session')

    # Parameter for the filter that will be used in the URL query.
    parameter_name = 'session'

    def lookups(self, request, model_admin):
        return [(s.id, s.name) for s in models.Session.objects.all()]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(division__session__id=self.value())
        else:
            return queryset

class LevelFilter(admin.SimpleListFilter):
    title = _('level')
    parameter_name = 'level'

    def lookups(self, request, model_admin):
        return [(s['level'], s['group']) for s in Level.objects.all().values('level', 'group').distinct().order_by('level')]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(level__level=self.value())
        else:
            return queryset

class LevelDivisionFilter(admin.SimpleListFilter):
    title = _('level division')
    parameter_name = 'level_division'

    def lookups(self, request, model_admin):
        return [(s.id, s.name) for s in Level.objects.all()]

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(level__id=self.value())
        else:
            return queryset

class AgeDivisionFilter(admin.SimpleListFilter):
    title = _('age division')
    parameter_name = 'age_division'

    def lookups(self, request, model_admin):
        return [(s.id, s) for s in models.Division.objects.all()]

    def queryset(self, request, queryset):
        value = self.value()
        if queryset and value is not None:
            return queryset.filter(division__id=value)
        else:
            return queryset

class TeamFilter(admin.SimpleListFilter):
    title = _('team')
    parameter_name = 'team'

    def lookups(self, request, model_admin):
        return [(s.id, s.team) for s in models.Team.objects.all()]

    def queryset(self, request, queryset):
        value = self.value()
        if value is not None:
            return queryset.filter(team__id=value)
        else:
            return queryset

class StartingEventFilter(admin.SimpleListFilter):
    title = _('starting event')
    parameter_name = 'starting_event'

    def lookups(self, request, model_admin):
        lookups = [(s.id, s.name) for s in models.Event.objects.filter(active=True)]  # competition.Event
        lookups.append(('', '(None)'))
        return lookups

    def queryset(self, request, queryset):
        if self.value() is '':
            return queryset.filter(starting_event__isnull=True)
        elif self.value() is not None:
            return queryset.filter(starting_event__id=self.value())
        else:
            return queryset

class GymnastEventFilter(admin.SimpleListFilter):
    title = _('gymnast event')
    parameter_name = 'gymnast_event'

    def lookups(self, request, model_admin):
        lookups = [(s.id, s.name) for s in models.Event.objects.filter(active=True)]  # competition.Event
        return lookups

    def queryset(self, request, queryset):
        if self.value() is not None:
            return queryset.filter(event__id=self.value())
        else:
            return queryset

# Admins

class GymnastEventInlineFormset(BaseInlineFormSet):
    def __init__(self, *args, **kwargs):
        super(GymnastEventInlineFormset, self).__init__(*args, **kwargs)
        self.can_delete = False

class GymnastEventInlineAdmin(admin.TabularInline):
    model = models.GymnastEvent
    formset = GymnastEventInlineFormset
    extra = 0
    max_num = 0
    readonly_fields = ('event', )
    fields = ('event', 'score',)
    verbose_name_plural = 'Gymnast Events'

@admin.register(models.Gymnast)
class GymnastAdmin(MeetDependentAdmin):
    ''' Base admin for the Mens Artistic and Women's Artistic admins. '''
    search_fields = ['athlete_id', 'last_name', 'first_name']
    inlines = (GymnastEventInlineAdmin, )
    readonly_fields = ('overall_score', 'rank', 'place', 'tie_break', 'age', 'team')
    list_filter = (TeamFilter, SessionFilter, StartingEventFilter, AgeDivisionFilter, LevelFilter, LevelDivisionFilter)
    list_per_page = 50
    list_display = ['athlete_id', 'last_name', 'first_name', 'show_team', 'division', 'session', 'starting_event']

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(GymnastAdmin, self).get_fieldsets(request, obj)
        fieldsets += (
            (None, {'fields': ('discipline', 'athlete_id', 'usag', 'last_name', 'first_name', 'team', ), }),
            ('Meet',
                {'fields':
                    ('is_scratched',
                     'age',
                     'division',
                     'level',
                     'starting_event',
                     'overall_score',
                     'rank',
                     'place',
                     'tie_break', ), }), )
        return fieldsets

    def get_readonly_fields(self, request, obj=None):
        # copying this from MeetDependentAdmin because I need the empty value display to be different here (empty string)
        # models.Meet.objects.filter(is_current_meet=True)[0]
        # if self.fieldsets and self.fieldsets[0][1]['fields'][0] == 'meet' and request.session.get('meet', ''):
        if self.fieldsets and self.fieldsets[0][1]['fields'][0] == 'meet' and models.Meet.objects.filter(is_current_meet=True).count() == 1:
            self.readonly_fields += ('meet',)
            self.empty_value_display = ''
            return self.readonly_fields
        else:
            return []

    def session(self, gymnast):
        if gymnast.division:
            return models.Session.objects.get(divisions=gymnast.division).name
        return '-'
    session.admin_order_field = 'division__session__name'

    def get_queryset(self, request):
        qs = super(GymnastAdmin, self).get_queryset(request)
        qs = qs.annotate(aa=Sum('events__score'))
        return qs

    def all_around(self, obj):
        return '{:.1f}'.format(obj.aa)
    all_around.admin_order_field = 'aa'
    all_around.short_description = 'AA'

    def show_team(self, obj):
        return obj.team.team
    show_team.short_description = "Team"
    show_team.admin_order_field = 'team__team'

    def has_add_permission(self, request, obj=None):
            return False

@admin.register(models.MensArtisticGymnast)
class MensArtisticGymnastAdmin(GymnastAdmin):
    pass

    def get_list_display(self, request):
        result = ['athlete_id',
                  'last_name',
                  'first_name',
                  'show_team',
                  'division',
                  'session',
                  'starting_event',
                  'fx',
                  'ph',
                  'sr',
                  'vt',
                  'pb',
                  'hb']
        result += ['all_around', ]
        return result

@admin.register(models.WomensArtisticGymnast)
class WomensArtisticGymnastAdmin(GymnastAdmin):
    pass

    def get_list_display(self, request):
        result = ['athlete_id',
                  'last_name',
                  'first_name',
                  'show_team',
                  'division',
                  'session',
                  'starting_event',
                  'vt',
                  'ub',
                  'bb',
                  'fx']
        result += ['all_around', ]
        return result

@admin.register(models.TeamAward)
class TeamAwardAdmin(MeetDependentAdmin):
    list_display = ('name', 'award_count', 'number_qualified_teams', 'order',)
    filter_horizontal = ('levels',)
    list_editable = ('order',)
    actions = ['meet_awards_percentage']

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(TeamAwardAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('name', 'levels', 'award_count', 'order',),
                'description': ''}), )
        return fieldsets

    def number_qualified_teams(self, obj):
        qualified_teams = obj.qualified_teams()
        return qualified_teams.count()
    number_qualified_teams.short_description = '# Qualified Teams'

    def meet_awards_percentage(self, request, queryset):
        ''' Count how many teams, with at least 3 gymnasts, registered to be considered for team awards.
            Calculate percentage of that number, using value stored in Meet Admin.
            Save calculated percentage to the award.

            gymnasts__level__group__in=award.levels.all().values('group') needed to handle the
            edge case of an award that spans two levels, which the BWI does for levels 9/10
        '''
        for award in queryset:
            awards = models.Team.objects.filter(
                team_awards=award, gymnasts__is_scratched=False,
                gymnasts__level__group__in=award.levels.all().values('group'))\
                .annotate(num_gymnasts=Count('id')).exclude(num_gymnasts__lt=3)

            if awards.count() == 1:
                award.award_count = 1
            else:
                award.award_count = math.ceil(awards.count() * award.meet.team_award_percentage)
            award.save()
    meet_awards_percentage.short_description = "Calculate team awards from %% in Meet admin"

@admin.register(models.TeamAwardRank)
class TeamAwardRankAdmin(MeetDependentAdmin):
    list_display = ('team', 'team_award', 'rank', 'score')
    list_filter = ['team', 'team_award']

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.TeamAwardRankEvent)
class TeamAwardRankEventAdmin(MeetDependentAdmin):
    list_display = ('team_award_rank', 'event', 'gymnast_event', 'rank')
    list_filter = ['team_award_rank__team', GymnastEventFilter]
    search_fields = ['gymnast_event__gymnast__first_name', 'gymnast_event__gymnast__last_name']

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.GymnastEvent)
class GymnastEventAdmin(MeetDependentAdmin):
    list_display = ('gymnast', 'event', 'score', 'rank', 'place')
    list_filter = [GymnastEventFilter]
    search_fields = ['gymnast__first_name', 'gymnast__last_name', 'id', ]

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(GymnastEventAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('gymnast', 'event', 'score', 'rank', 'place'),
                'description': ''}), )
        return fieldsets

    def has_add_permission(self, request, obj=None):
        return False

@admin.register(models.Division)
# TODO: Come back to this one
class DivisionAdmin(MeetDependentAdmin):
    list_display = ('name', 'level', 'num_gymnasts', 'min_age', 'max_age', 'event_award_count', 'all_around_award_count')
    list_editable = ('min_age', 'max_age', 'event_award_count', 'all_around_award_count')
    ordering = ('level', 'min_age')
    actions = ['meet_awards_percentage']
    list_filter = ['level', 'level__group']

    def meet_awards_percentage(self, request, queryset):
        # Make sure to exclude scratched gymnasts when running the award count
        for division in queryset:
            division.event_award_count = math.ceil(division.gymnasts.filter(is_scratched=False).count() * division.meet.event_award_percentage)
            division.all_around_award_count = math.ceil(division.gymnasts.filter(is_scratched=False).count() * division.meet.all_around_award_percentage)
            division.save()
    meet_awards_percentage.short_description = "Calculate AA & event awards from %%s in Meet admin"

    def get_queryset(self, request):
        # TODO: annotate query with num_gymnasts, so the column can be sortable in the admin
        qs = super(DivisionAdmin, self).get_queryset(request)
        return qs

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(DivisionAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('level', 'name', 'short_name', 'min_age', 'max_age', 'event_award_count', 'all_around_award_count'),
                'description': ''}), )
        return fieldsets

@admin.register(models.Event)
class EventAdmin(admin.ModelAdmin):
    list_display = ('name', 'initials', 'order', 'is_mag', 'is_wag', 'active')
    actions = ['toggle_mag_wag']

    def toggle_mag_wag(self, request, queryset):
        for event in queryset:
            # Don't let this work on trampoline events
            if not (event.is_wag is False and event.is_mag is False):
                event.is_wag = False if event.is_wag else True
                event.is_mag = False if event.is_mag else True
                event.active = False if event.active else True
                event.save()
    toggle_mag_wag.short_description = "Switch active events, mens and womens"

@admin.register(models.Session)
class SessionAdmin(MeetDependentAdmin):
    list_display = ('name', 'num_gymnasts', 'warmup', 'session_start', 'timed_warmup_start', 'presentation_start', 'competition_start', )
    filter_horizontal = ('divisions',)

    def get_fieldsets(self, request, obj=None):
        fieldsets = super(SessionAdmin, self).get_fieldsets(request, obj)
        # If there's no active meet, hide fields until active meet has been set
        if request.session.get('meet', ''):
            fieldsets += ((None, {
                'fields': ('name',
                           'divisions',
                           'warmup',
                           'session_start',
                           'timed_warmup_start',
                           'presentation_start',
                           'competition_start'),
                'description': ''}), )
        return fieldsets

@admin.register(models.ScoreRankEvent)
class ScoreRankEventAdmin(MeetDependentAdmin):
    list_display = ('gymnast', 'fx', 'ph', 'sr', 'vt', 'pb', 'hb', 'ub', 'bb')
    search_fields = ['gymnast__last_name', 'gymnast__first_name', 'gymnast__usag']

@admin.register(models.LEDSign)
class LEDSignAdmin(admin.ModelAdmin):
    list_display = ('name', 'device', 'connect')

    def connect(self, obj):
        return 'Connect Sign'.format(obj.device)
    connect.allow_tags = True

    class Media:
        js = ("{}js/ledsign.js".format(settings.STATIC_URL),)

class LEDShowMessageInline(admin.TabularInline):
    model = models.LEDShowMessage
    fields = ('led_sign', 'message')
    extra = 0
    min_num = 6

@admin.register(models.LEDShow)
class LEDShowAdmin(admin.ModelAdmin):
    model = models.LEDShow
    list_display = ('name',)
    inlines = (LEDShowMessageInline, )
    fieldsets = ((None, {
        'fields': ('name',),
        'description': LED_SIGN_CODES}), )


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import LinearRegression

base =  pd.read_csv('/workspace/tensorFlow/udemyDSPythonAndR/Estatistica_II/mt_cars.csv')
base = base.drop(['Unnamed: 0'], axis = 1)

base.iloc[0:2, 1:2]

X = base.iloc[:, 2].values
X
y = base.iloc[:, 0]
y
correlacao = np.corrcoef(X, y)
correlacao
X = X.reshape(-1, 1)
modelo = LinearRegression()
modelo.fit(X, y)
modelo.intercept_
modelo.coef_
modelo.score(X, y)

previsoes = modelo.predict(X)
previsoes

plt.scatter(X, y)
plt.plot(X, previsoes, color= 'red')

X1 = base.iloc[:, 1:4].values
modelo2 = LinearRegression()
modelo2.fit(X1, y)
previsoes2 = modelo2.predict(X1)
previsoes2

fig = plt.subplot()
ax = fig.scatter(X, previsoes2, color= 'green')
fig.plot(X, previsoes2, color= 'green')
plt.show()


"""
2016 Day 19
https://adventofcode.com/2016/day/19
"""

# The problem presented here in 2016 Day 19 is also known as The Josephus Problem:
# https://www.youtube.com/watch?v=uCsD3ZGzMgE

from typing import Sequence
import numpy as np
import aocd  # type: ignore

def who_wins(elfcount: int) -> int:
    """
    Use the general solution of the Josephus problem to solve for the given quantity of elves.
    Return the number of the winning elf.
    """
    binary = np.binary_repr(elfcount)
    return int(binary[1:] + binary[0], 2)

class Elf:
    """
    An elf, sat in the circle and ready for the Josephus-inspired game.
    """

    number: int
    prev: "Elf"
    next: "Elf"

    def __init__(self, number: int):
        self.number = number

    def delete(self) -> None:
        """
        Remove this elf from the circle and update its neighbours to point at each other instead.
        """
        self.next.prev = self.prev
        self.prev.next = self.next

    @classmethod
    def circle(cls, length: int) -> "Sequence[Elf]":
        """
        Instatiate a circle of elves of the given length, each aware of its neighbours.
        """
        circle = [cls(x) for x in range(1, length + 1)]
        for index in range(length):
            circle[index].prev = circle[(index - 1) % length]
            circle[index].next = circle[(index + 1) % length]
        return circle

def who_wins_opposite(elfcount: int) -> int:
    """
    Return the number of the Elf that wins in the 'eliminate opposite' version of the game.
    """
    circle = Elf.circle(elfcount)

    elf = circle[0]
    mid = circle[int(elfcount / 2)]

    for theft in range(elfcount - 1):
        mid.delete()
        mid = mid.next
        if (elfcount - theft) % 2 == 1:
            mid = mid.next
        elf = elf.next

    return elf.number

def main() -> None:
    """
    Calculate and output the solutions based on the real puzzle input.
    """
    data = aocd.get_data(year=2016, day=19)
    elfcount = int(data)

    print(f"Part 1: {who_wins(elfcount)}")
    print(f"Part 2: {who_wins_opposite(elfcount)}")

if __name__ == "__main__":
    main()


# The app configuration defined as a pydantic model, plus some helpers.

import enum
from typing import Dict, List, Optional, Union
from typing_extensions import Annotated, Literal

import yaml
from pydantic import BaseModel, Field

from . import _yaml_tools

class StrEnum(str, enum.Enum):
    """String enum base class provide actual string value in str(...) output.

    Overrides the default IdEnum.xxx output of the Enum class.
    """

    def __str__(self):
        # Avoid IdEnum.xxx output of Enum class, provide actual string value
        return self.value

@_yaml_tools.dump_as_str
@enum.unique
class ExposeRoutesEnum(StrEnum):
    """Enumeration of exposable endpoints.
    """
    get_all = 'get_all'
    get_one = 'get_one'
    delete_all = 'delete_all'
    delete_one = 'delete_one'
    create = 'create'
    update = 'update'

@_yaml_tools.dump_as_str
@enum.unique
class AuthnEnum(StrEnum):
    """Enumeration of authentication mechanisms.
    """
    HTTPBasic_LDAP = 'HTTPBasic+LDAP'
    OAuth2PasswordBearer_LDAP = 'OAuth2PasswordBearer+LDAP'

@_yaml_tools.dump_as_str
@enum.unique
class SchemaSpecEnum(StrEnum):
    """Enumeration of the supported schema specifications.
    """
    sqlalchemy = "https://www.sqlalchemy.org/"
    data_resource = "https://specs.frictionlessdata.io/data-resource/"

# The rather complex Table + Datatables implementation is needed to allow for
# using the specialized SQLAlchemyTable and TableschemaTable classes where a
# Table is expected, discriminated by their schema_spec attribute. This
# provides for putting the concrete table implementation where a base Table is
# expected and for proper serialization.

# Helper classes to enforce intended field order.

# Common Table fields
class _TableFields(BaseModel):
    dbtable: str
    paginate: int = 10
    expose_routes: Optional[List[ExposeRoutesEnum]] = [
        ExposeRoutesEnum.get_one]
    query_params: Optional[List[str]] = []

class _SQLAlchemyTableFields(BaseModel):
    schema_spec: Literal[SchemaSpecEnum.sqlalchemy]

class _TableschemaTableFields(BaseModel):
    schema_spec: Literal[SchemaSpecEnum.data_resource]
    # schema is used by pydantic so work with an alias
    schema_: str = Field(..., alias='schema')

# The actual Table implementation classes.
class SQLAlchemyTable(_SQLAlchemyTableFields, _TableFields):
    pass

class TableschemaTable(_TableschemaTableFields, _TableFields):
    pass

Table = Annotated[
    Union[TableschemaTable, SQLAlchemyTable],
    Field(discriminator='schema_spec')
    ]

class Datatables(BaseModel):
    __root__: Dict[str, Table]

    def __getattr__(self, attr):  # if you want to use '.'
        try:
            return self.__root__[attr]
        except KeyError as exc:
            raise AttributeError from exc

    def items(self):
        return self.__root__.items()

class App(BaseModel):
    title: str
    description: Optional[str] = ""
    version: str

class LDAP(BaseModel):
    bind_dn: str
    server: str

class Authn(BaseModel):
    authn_type: AuthnEnum
    ldap: Optional[LDAP] = None

class Fastapi(BaseModel):
    app: App
    authn: Optional[Authn] = None

class Database(BaseModel):
    connect_string: str

class Datarest(BaseModel):
    fastapi: Fastapi
    database: Database
    datatables: Datatables

class AppConfig(BaseModel):
    datarest: Datarest

# TODO: Design a saner API for setting authn configuration, including
# init-phase config checks (e.g. LDAP-based configs must ensure proper LDAP
# bind_dn + server settings, ...)
def app_config(
        table,
        title=None,
        description='',
        version='0.1.0',
        connect_string="sqlite:///app.db",
        expose_routes=(ExposeRoutesEnum.get_one, ),
        query_params=(),
        authn_type=None,
        ldap_bind_dn=None,
        ldap_server=None,
        ):
    """Return AppConfig object.
    """
    title = table if title is None else title

    authn = None
    ldap = None
    if authn_type is not None:
        # Using or here catches a missing ldap attribute, at least
        if ldap_bind_dn is not None or ldap_server is not None:
            ldap = LDAP(
                bind_dn=ldap_bind_dn,
                server=ldap_server,
                )
        authn = Authn(
            authn_type=authn_type,
            ldap=ldap
            )
    # expose_routes = list(expose_routes)
    config = AppConfig(
        datarest=Datarest(
            fastapi=Fastapi(
                app=App(
                    title=f"{title} API",
                    description=description,
                    version=version,
                    ),
                authn=authn,
                ),
            database=Database(connect_string=connect_string, ),
            datatables=Datatables(
                __root__={
                    table:
                    TableschemaTable(
                        schema_spec=
                        "https://specs.frictionlessdata.io/data-resource/",
                        schema=f"{table}.yaml",
                        dbtable=table,
                        expose_routes=expose_routes,
                        query_params=query_params,
                        )
                    }
                ),
            )
        )
    return config

def write_app_config(cfg_path, app_config):
    """Write app.yaml config YAML string to cfg_path file.
    """
    app_config_yaml = yaml.safe_dump(
        app_config.dict(by_alias=True, exclude_unset=True, exclude_none=True),
        default_flow_style=False,
        sort_keys=False
        )
    with open(cfg_path, encoding='utf-8', mode='w') as cfg_file:
        cfg_file.write(app_config_yaml)

def read_app_config(cfg_path='app.yaml'):
    """Read app.yaml config YAML string from cfg_path file.

    Returns AppConfig model.
    """
    with open(cfg_path, 'rb') as config_file:
        dct = yaml.safe_load(config_file)
        config = AppConfig.parse_obj(dct)
    return config


number = [1,1,1,1,1]
target = 3
def solution(numbers,target):
    answer=[0]
    for a in numbers:
        temp=[]
        for b in answer:
            temp.append(b+a)
            temp.append(b-a)
        answer=temp
    result = answer.count(target)
    return result
print(solution(number,target))


from pyspark.sql import SparkSession

def exe_spark():
    spark = SparkSession.builder \
        .master("local[*]") \
        .appName("spark sql hint test") \
        .getOrCreate()
    # sc = spark.sparkContext

    spark.read.json("../data1.json").createOrReplaceTempView("t1")
    spark.read.json("../data2.json").createOrReplaceTempView("t2")

    """
    -- Join Hints for broadcast join
    SELECT /*+ BROADCAST(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
    SELECT /*+ BROADCASTJOIN (t1) */ * FROM t1 left JOIN t2 ON t1.key = t2.key;
    SELECT /*+ MAPJOIN(t2) */ * FROM t1 right JOIN t2 ON t1.key = t2.key;

    -- Join Hints for shuffle sort merge join
    SELECT /*+ SHUFFLE_MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
    SELECT /*+ MERGEJOIN(t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
    SELECT /*+ MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;

    -- Join Hints for shuffle hash join
    SELECT /*+ SHUFFLE_HASH(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;

    -- Join Hints for shuffle-and-replicate nested loop join
    SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;

    -- 多个hints同时指定时，有以下优先级：
    -- BROADCAST > MERGE > SHUFFLE_HASH > SHUFFLE_REPLICATE_NL
    -- 下面的例子会Warning：org.apache.spark.sql.catalyst.analysis.HintErrorLogger: Hint (strategy=merge)
    -- MERGE会被覆盖，不会生效
    SELECT /*+ BROADCAST(t1), MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
    """

    # 在实际运用中，对小表进行广播，可以增加计算性能
    # t1 订单表， t2 用户维度
    # 对于right outer join 只能广播左表；
    # 对于left outer join，left semi join，left anti join，internal join等只能广播右表；
    # inner join 可以指定广播
    spark.sql("""
        SELECT /*+ MAPJOIN(t2) */
               t2.area
              ,COUNT(DISTINCT t1.id) as order_count
              ,SUM(t1.amount) as order_amount
          FROM t2
          JOIN t1
            on t1.user = t2.user
      GROUP BY t2.area
    """).show()
    """
    执行计划显示，BuildLeft对左边也就是t2进行了广播，否则默认是会对右边进行广播，hint生效
    == Physical Plan ==
    *(4) HashAggregate(keys=[area#22], functions=[count(distinct id#9L)], output=[area#22, order_count#28L])
    +- Exchange hashpartitioning(area#22, 200), true, [id=#77]
       +- *(3) HashAggregate(keys=[area#22], functions=[partial_count(distinct id#9L)], output=[area#22, count#34L])
          +- *(3) HashAggregate(keys=[area#22, id#9L], functions=[], output=[area#22, id#9L])
             +- Exchange hashpartitioning(area#22, id#9L, 200), true, [id=#72]
                +- *(2) HashAggregate(keys=[area#22, id#9L], functions=[], output=[area#22, id#9L])
                   +- *(2) Project [area#22, id#9L]
                      +- *(2) BroadcastHashJoin [user#24], [user#10], Inner, BuildLeft
                         :- BroadcastExchange HashedRelationBroadcastMode(List(input[1, string, true])), [id=#63]
                         :  +- *(1) Project [area#22, user#24]
                         :     +- *(1) Filter isnotnull(user#24)
                         :        +- FileScan json [area#22,user#24] Batched: false, DataFilters: [isnotnull(user#24)], Format: JSON, Location: InMemoryFileIndex[file:/C:/Users/panda/Desktop/WORK/sp/spark_practice/data2.json], PartitionFilters: [], PushedFilters: [IsNotNull(user)], ReadSchema: struct<area:string,user:string>
                         +- *(2) Project [id#9L, user#10]
                            +- *(2) Filter isnotnull(user#10)
                               +- FileScan json [id#9L,user#10] Batched: false, DataFilters: [isnotnull(user#10)], Format: JSON, Location: InMemoryFileIndex[file:/C:/Users/panda/Desktop/WORK/sp/spark_practice/data1.json], PartitionFilters: [], PushedFilters: [IsNotNull(user)], ReadSchema: struct<id:bigint,user:string>
    """

if __name__ == "__main__":
    exe_spark()


import requests
from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
from PyPDF2 import PdfFileWriter, PdfFileReader
from requests_html import HTMLSession

# initialize the set of links (unique links)
internal_urls = set()
external_urls = set()
urls = set()

# This will make sure that a proper scheme (protocol, e.g http or https) and domain name exists in the URL.
def is_valid(url):
    """Checks whether `url` is a valid URL"""
    parsed = urlparse(url)
    return bool(parsed.netloc) and bool(parsed.scheme)

def get_all_website_links(url):

    session = HTMLSession()
    try:
        response = session.get(url)
    except:
        print("Check your url")
        return {}

    try:
        response.html.render()
    except:
        pass
#     soup = BeautifulSoup(requests.get(url).content, "html.parser")
    domain_name = urlparse(url).netloc
    soup = BeautifulSoup(response.html.html, "html.parser")  # this is assumed for javascript loaded pages
    for a_tag in soup.findAll("a"):
        href = a_tag.attrs.get("href")
        if href == "" or href is None:
            # href empty tag
            continue

        href = urljoin(url, href)
#         print(href)
        parsed_href = urlparse(href)
        if parsed_href.query != "":
            href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path + "?" + parsed_href.query
        else:
            href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path

        if not is_valid(href):
            # not a valid URL
            continue
        if href in internal_urls:
            # already in the set
            continue
        if domain_name not in href:
            # its external link
            if href not in external_urls:
                external_urls.add(href)
            continue
        # else its internel links
        urls.add(href)
        internal_urls.add(href)
    return urls

def make_dir(dirName):
    if not os.path.exists(dirName):
        os.mkdir(dirName)
        print("Directory " , dirName ,  " Created ")
    else:
        print("Directory " , dirName ,  " already exists")

total_urls_visited = 0

# help to call give number of times inside the website
def crawl(url, max_urls=10):
    """
    Crawls a web page and extracts all links.
    You'll find all links in `external_urls` and `internal_urls` global set variables.
    params:
        max_urls (int): number of max urls to crawl, default is 10.
    """
    global total_urls_visited
    total_urls_visited += 1
    links = get_all_website_links(url)
    links = list(links)
    if len(links) > 0:
        for link in links:
            if total_urls_visited > max_urls:
                break
            crawl(link, max_urls=max_urls)

if __name__ == "__main__":

    total_urls_visited = 0

    root_url = input("Enter the url:")
    max_url = int(input("Max number internel url to crawl:"))
    select_number_of_pdf = int(input("Number of pdf to dowload:"))

    links = get_all_website_links(root_url)
    print(len(links))
    list_links = list(links)
    if len(list_links) < max_url:
    	max_url = len(list_links)
    for i in range(max_url):
        get_all_website_links(list_links[i])

    print("[+] Total External links:", len(external_urls))
    print("[+] Total Internal links:", len(internal_urls))
    print("[+] Total:", len(external_urls) + len(internal_urls))

    #Create pdfs and txts
    import os
    parsed = urlparse(root_url)
    netloc = parsed.netloc

    if not os.path.exists("pdfs"):
        os.mkdir("pdfs")

    if not os.path.exists("txts"):
        os.mkdir("txts")

    pdf_dirName = os.path.join("pdfs",netloc.split(".")[1])
    txt_dirName = os.path.join("txts",netloc.split(".")[1])

    make_dir(pdf_dirName)
    make_dir(txt_dirName)

    internal_urls = list(internal_urls)
    pdf_urls = [iu for iu in internal_urls if iu.endswith(".pdf")]

    total_pdfs = len(pdf_urls)
    print(f"Total pdfs {total_pdfs}!")

    def dowload_pdf_save_txt(url):
        pdf_name = url.split("/")[-1]

        print(pdf_name)
        try:
            response = requests.get(url)

            with open(f'{pdf_dirName}/{pdf_name}', 'wb') as f:
                f.write(response.content)
                f.close()
            output = PdfFileWriter()
            inputs = PdfFileReader(open(f'{pdf_dirName}/{pdf_name}', "rb"))
            print (f"{pdf_name} has {inputs.getNumPages()} pages.")
            total_pages = inputs.getNumPages()
            txt = url+"\n" + ""
            for i in range(total_pages):
                txt+=inputs.getPage(i).extractText()

            with open(f'{txt_dirName}/{pdf_name.split(".")[0]}.txt', 'w+') as f:
                f.write(txt)
                f.close()
        except:
            pass

    import time
    t1 = time.time()
    count = 0

    for pdf_url in pdf_urls[:select_number_of_pdf]:
        count+=1
        dowload_pdf_save_txt(pdf_url)
    print(f"Total {count} pdfs downloaded:")
    t2 = time.time()
    print(f'!{t2-t1} " seconds take!')
    print("Lets see pdfs and txts folder")


n = float(input("Dame el numero: "))

def mayoresn (lista):
    c = 0
    for elem in lista:
        if elem > n:
            c += 1
    return c

lis = [3, 5, 7, 9, 12]

print((mayoresn(lis)))


# -*- coding: utf-8 -*-
"""
Materia     : Laboratorio de datos - FCEyN - UBA
Clase       : Clase Regresion Lineal
Detalle     : Modelo KNN
Autores     : Manuela Cerdeiro y Pablo Turjanski
Modificacion: 2023-10-21
"""

# Importamos bibliotecas
import pandas as pd
import numpy as np
from sklearn.neighbors import KNeighborsRegressor
import matplotlib.pyplot as plt
import seaborn as sns

#%%
####################################################################
########  DEFINICION DE FUNCIONES AUXILIARES
####################################################################

# Dibuja una recta. Toma como parametros pendiente e intercept
def plotRectaRegresion(slope, intercept):
    """Plot a line from slope and intercept"""
    axes = plt.gca()
    x_vals = np.array(axes.get_xlim())
    y_vals = intercept + slope * x_vals
    plt.plot(x_vals, y_vals, color="red")

#%%
####################################################################
########  MAIN
####################################################################
# Cargamos el archivo
carpeta = '/home/clinux01/Descargas/Clase-20-KNN/'
data_train = pd.read_csv(carpeta+"datos_roundup.txt", sep=" ", encoding='utf-8')
#%%

# ----------------------------------
# ----------------------------------
#       Modelo KNN
# ----------------------------------
# ----------------------------------
#  X = RU (variable predictora) [Dosis de Roundup]
#  Y = ID (variable a predecir) [Damage Index]
########################
## Generacion del modelo
########################
# Declaramos las variables
X = data_train[['RU']] # doble corchete es importante
Y = data_train[['ID']] # doble corchete es importante
k = 3                  # cantidad de vecinos
# Declaramos el tipo de modelo
neigh = KNeighborsRegressor(n_neighbors=k)

# Entrenamos el modelo
neigh.fit(X, Y)

#####################################
## Evaluacion del modelo contra TRAIN
#####################################
# Calculamos el R2. Recordar que 1 es en el caso de una prediccion perfecta
print("R^2 (train ): %.2f" % neigh.score(X, Y))

#####################################
## Prediccion
#####################################
# Cargamos el archivo (no posee valores para ID)
carpeta = '/home/clinux01/Descargas/Clase-20-KNN/'
data_a_predecir = pd.read_csv(carpeta+"datos_a_predecir.txt", sep=" ", encoding='utf-8')

# Realizamos la prediccion de ID utilizando el modelo y
# la asignamos a la columna ID
data_a_predecir[['ID']] = neigh.predict(data_a_predecir[['RU']])

# Graficamos una dispersion de puntos de ID en funcion de la Dosis de RU
# Graficamos tanto los puntos de entrenamiento del modelo como los predichos
ax = sns.scatterplot(data=data_train, x="RU", y="ID", s = 40, color='black')
ax = sns.scatterplot(data=data_a_predecir, x="RU", y="ID", s = 40, color='olive')
ax.set_xlabel("Dosis de RU (ug/huevo)",fontsize=12)
ax.set_ylabel("Indice de daño",fontsize=12)
# Graficamos la recta que habiamos obtenido en el modelo RLS la clase pasada
plotRectaRegresion(0.037, 106.5)


import tensorflow as tf
import numpy as np

import utils

class StackedDenoisingAutoencoder:
    """
    Implementation of a stacked denoising autoencoder using tensorflow
    """

    def __init__(self, corruption_type, corruption_fraction, weight_stddev, bias_init, learning_rate, epochs, batch_size):
        # Input
        self.corruption_type = corruption_type
        self.corruption_fraction = corruption_fraction

        # Initialization
        self.weight_stddev = weight_stddev
        self.bias_init = bias_init

        # Training
        self.learning_rate = learning_rate
        self.epochs = epochs
        self.batch_size = batch_size

        # Model Variables
        self.input_data = None
        self.corrupted_input_data = None
        self.weights = None
        self.biases = None
        self.encode = None
        self.flat_pool = None
        self.decode = None
        self.cost = None
        self.train_step = None

        # Build the model
        self.build_model2()

    def build_model(self):
        self.weights = tf.Variable(tf.truncated_normal([5, 5, 3, 32], stddev=self.weight_stddev))
        self.biases = tf.Variable(tf.constant(self.bias_init, shape=[32]))

        n_features = 3072

        self.input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        samples = tf.shape(self.input_data)[0]
        self.corrupted_input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        corrupted_image = tf.reshape(self.corrupted_input_data, [samples, 32, 32, 3])

        # Encode
        self.encode = tf.nn.relu(
            tf.nn.conv2d(corrupted_image, self.weights, strides=[1, 1, 1, 1], padding='SAME') + self.biases
        )

        # Flat Pool Encoding
        self.flat_pool = tf.reshape(
            tf.nn.max_pool(self.encode, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME'), [samples, 8192]
        )

        # Decode
        self.decode = tf.nn.conv2d_transpose(self.encode, self.weights, output_shape=[samples, 32, 32, 3],
                                             strides=[1, 1, 1, 1])

        # Flatten
        flat_decode = tf.reshape(self.decode, [samples, n_features])

        # Cost
        self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.input_data - flat_decode)))

        # Optimizer
        self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.cost)

    def build_model2(self):
        self.weights = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=self.weight_stddev))
        self.biases = tf.Variable(tf.constant(self.bias_init, shape=[64]))

        n_features = 8192

        self.input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        samples = tf.shape(self.input_data)[0]
        self.corrupted_input_data = tf.placeholder(tf.float32, shape=[None, n_features])
        corrupted_image = tf.reshape(self.corrupted_input_data, [samples, 16, 16, 32])

        # Encode
        self.encode = tf.nn.relu(
            tf.nn.conv2d(corrupted_image, self.weights, strides=[1, 1, 1, 1], padding='SAME') + self.biases
        )

        # Pool Encoding
        self.flat_pool = tf.reshape(
            tf.nn.max_pool(self.encode, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME'), [samples, 4096]
        )

        # Decode
        self.decode = tf.nn.conv2d_transpose(self.encode, self.weights, output_shape=[samples, 16, 16, 32],
                                             strides=[1, 1, 1, 1])

        # Flatten
        flat_decode = tf.reshape(self.decode, [samples, n_features])

        # Cost
        self.cost = tf.sqrt(tf.reduce_mean(tf.square(self.input_data - flat_decode)))

        # Optimizer
        self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.cost)

    def train(self, training_x, validation_x):
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())

        corruption_ratio = np.round(self.corruption_fraction * training_x.shape[1]).astype(np.int)
        for epoch in range(self.epochs):
            print("Epoch: {}".format(epoch))

            x_corrupted = self.corrupt_input(training_x, corruption_ratio)

            shuff = list(zip(training_x, x_corrupted))
            np.random.shuffle(shuff)

            batches = [_ for _ in utils.gen_batches(shuff, self.batch_size)]

            for batch in batches:
                x_batch, x_corrupted_batch = zip(*batch)
                sess.run(self.train_step, feed_dict={self.input_data: x_batch,
                                                     self.corrupted_input_data: x_corrupted_batch})

            if epoch % 5 == 0:
                if validation_x is not None:
                    error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
                                                           self.corrupted_input_data: validation_x})
                    print("Validation cost is {}".format(error))

        if validation_x is not None:
            error = sess.run(self.cost, feed_dict={self.input_data: validation_x,
                                                   self.corrupted_input_data: validation_x})
            print("Validation cost is {}".format(error))

        # Save the model
        #saver = tf.train.Saver()
        #saver.save(sess, 'SDAE_model2')

    def corrupt_input(self, data, v):

        if self.corruption_type == 'masking':
            x_corrupted = utils.masking_noise(data, v)

        elif self.corruption_type == 'salt_and_pepper':
            x_corrupted = utils.salt_and_pepper_noise(data, v)

        elif self.corruption_type == 'gaussian':
            x_corrupted = utils.gaussian_noise(data, v)

        elif self.corruption_type == 'none':
            x_corrupted = data

        else:
            x_corrupted = None

        return x_corrupted

    def transform(self, x, model_name):
        # load saved model
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())
        saver = tf.train.import_meta_graph(model_name)
        saver.restore(sess, tf.train.latest_checkpoint('./'))

        # Run against some data until pooling step
        pool = sess.run(self.flat_pool, feed_dict={self.input_data: x, self.corrupted_input_data: x})

        return pool

    def save_weights(self, x, model_name):
        # load saved model
        sess = tf.Session()
        sess.run(tf.global_variables_initializer())
        saver = tf.train.import_meta_graph(model_name)
        saver.restore(sess, tf.train.latest_checkpoint('./'))

        weights, biases = sess.run([self.weights, self.biases], feed_dict={self.input_data: x, self.corrupted_input_data: x})
        return weights, biases


keywords = ['class', 'constructor', 'function', 'method', 'field', 'static', 'var', 'int', 'char', 'boolean', 'void', 'true', 'false', 'null', 'this', 'let', 'do', 'if', 'else', 'while', 'return']
symbols = ['{', '}', '(', ')', '[', ']', '.', ',', ';', '+', '-', '*', '/', '&', '|', '<', '>', '=', '~']

def tokenize(program_arg, output_type):
    import os, sys

    print(program_arg)
    if os.path.isfile(program_arg) and program_arg.endswith('.jack'):
        tokenize_file(program_arg, output_type)
    elif os.path.isdir(program_arg):
        import glob
        for jack_file in glob.glob("{0}/*.jack".format(program_arg)):
            tokenize_file(jack_file, output_type)

def tokenize_file(jack_file, output_type):
    tokens = get_tokens(jack_file)

    if output_type is 'xml':
        tokens_as_xml = map(get_xml, tokens)
        output_file = jack_file.replace('.jack', '_tokenized.xml')
        write_xml(output_file, tokens_as_xml)
    else:
        output_file = jack_file.replace('.jack', '_tokens')
        write_output(output_file, tokens)

def get_tokens(jack_file):
    tokens = []
    with open(jack_file, 'r') as f:
        for line in f:
            if (not line.isspace() and
                    not line.startswith('//') and
                    not line.startswith('/*') and
                    not line.startswith(' *')):
                code_without_comments = remove_comments(line).strip()
                tokens.extend(get_symbols(code_without_comments))
    return tokens

def remove_comments(line_of_code):
    if '//' in line_of_code:
        return line_of_code[0:line_of_code.index('//') - 1]
    if '/*' in line_of_code:
        return line_of_code[0:line_of_code.index('/*') - 1]
    return line_of_code

'''
get_symbols( string )
returns an array of strings

Iterates through the characters of a line of code to get the tokens associated with the Jack language
'''
def get_symbols(line_of_code):
    symbols_from_code = []
    symbol_parts = [] # to be used per symbol/word
    isCharPartOfString = False
    for char in line_of_code:
        if char == '"' or char == "'":
            if not isCharPartOfString:
                isCharPartOfString = True
            else:
                isCharPartOfString = False
                symbol = "(str){0}".format(''.join(symbol_parts)) # Add a way to identify the token as a string
                symbols_from_code.append(symbol)
                symbol_parts.clear()
        elif char.isspace() and len(symbol_parts) > 0 and not isCharPartOfString:
            symbol = ''.join(symbol_parts)
            symbols_from_code.append(symbol)
            symbol_parts.clear()
        elif char in symbols and not isCharPartOfString:
            if len(symbol_parts) > 0:
                symbol = ''.join(symbol_parts)
                symbols_from_code.append(symbol)
                symbol_parts.clear()
            symbols_from_code.append(char)
        elif not char.isspace() or isCharPartOfString:
            symbol_parts.append(char)
    return symbols_from_code

def get_symbol_code(symbol):
    code = symbol
    if symbol == '<':
        code = '<'
    elif symbol == '>':
        code = '>'
    elif symbol == '&':
        code = '&'
    return code

def get_xml(token):
    xml = ''
    if token in keywords:
        xml = "<keyword> {0} </keyword>".format(token)
    elif token in symbols:
        xml = "<symbol> {0} </symbol>".format(get_symbol_code(token))
    elif "(str)" in token:
        xml = "<stringConstant> {0} </stringConstant>".format(token[5:])
    elif token.isdigit():
        xml = "<integerConstant> {0} </integerConstant>".format(token)
    else:
        xml = "<identifier> {0} </identifier>".format(token)
    return xml

def write_xml(output_file, tokens_as_xml):
    with open(output_file, 'w') as f:
        f.write("<tokens>\n")
        for token in tokens_as_xml:
            f.write(token)
            f.write("\n")
        f.write('</tokens>')
    print("All done writing XML to {0}".format(output_file))

def write_output(output_file, tokens):
    with open(output_file, 'w') as f:
        for token in tokens:
            f.write("{0}\n".format(token))
    print("All done writing XML to {0}".format(output_file))

if __name__ == '__main__':
    import sys

    if len(sys.argv) > 2:
        tokenize(sys.argv[1], sys.argv[2])

    tokenize(sys.argv[1], '')


import torch.nn as nn
import torch
import torch.nn.functional as F
from Network.tcn import TemporalConvNet

class Embedder(nn.Module):
  def __init__(self, module='gru', time_stamp=82, input_size=34, hidden_dim=17, output_dim=24, num_layers=10, activate_function=nn.Tanh()):
    super(Embedder, self).__init__()
    self.module = module
    self.input_size = input_size
    self.time_stamp = time_stamp
    self.hidden_dim = hidden_dim
    self.num_layers = num_layers
    self.hidden_dim_layers = []
    self.output_dim = output_dim

    if self.module == 'gru':
      self.r_cell = nn.GRU(
        input_size = self.input_size,
        hidden_size = self.hidden_dim,
        num_layers = self.num_layers,
        batch_first = True
      )
    elif self.module == 'lstm':
      self.r_cell = nn.LSTM(
        input_size = self.input_size,
        hidden_size = self.hidden_dim,
        num_layers = self.num_layers,
        batch_first = True
      )
    elif self.module == 'bi-lstm':
      self.r_cell = nn.LSTM(
        input_size = self.input_size,
        hidden_size = self.hidden_dim,
        num_layers = self.num_layers,
        batch_first = True,
        bidirectional = True
      )

    elif self.module == 'tcn':

      for i in range(num_layers):
        self.hidden_dim_layers.append(self.hidden_dim)

      self.r_cell = TemporalConvNet( num_inputs=self.input_size,
        num_channels=self.hidden_dim_layers,
        kernel_size=2,
        dropout=0.2
      )

    if self.module == 'bi-lstm':
      self.fc = nn.Linear(self.hidden_dim * 2, self.output_dim)
    else:
      self.fc = nn.Linear(self.hidden_dim, self.output_dim)

    self.activate = activate_function

  def forward(self, X, H):
    if self.module == 'tcn':
      # Input X shape: (batch_size, seq_len, input_dim)
      X = torch.transpose(X, 1, 2)
      # Input X shape: (batch_size, input_dim, seq_len)
      output = self.r_cell(X)
      # H shape: (batch_size, input_dim, seq_len)
      # H_transpose: (batch_size, seq_len, input_dim)
      output = torch.transpose(output, 1, 2)
    else:
      # Inpu t X shape: (seq_len, batch_size, input_dim)
      output, _ = self.r_cell(X, H)
      # Outputs shape: (seq_len, batch_size, input_dim)

    H = self.fc(self.activate(output))

    return H

# gpu-used
CUDA_DEVICES = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def test():

  model = Embedder(
    module='tcn',
    time_stamp = 82,
    input_size = 27,
    hidden_dim = 100,
    output_dim = 100,
    num_layers = 10,
    activate_function=nn.Tanh()
  )

  model = model.to(CUDA_DEVICES)
  model.train()

  criterion = nn.MSELoss()

  optimizer =  torch.optim.Adam(params=model.parameters(), lr=0.001)

  optimizer.zero_grad()

  labels = torch.randn(32, 82, 100)
  inputs = torch.randn(32, 82, 27)

  inputs = inputs.to(CUDA_DEVICES)
  labels = labels.to(CUDA_DEVICES)

  # for name, param in model.named_parameters():
  #   if param.requires_grad:
  #     print(name, param.data)

  outputs = model(inputs, None)

  loss = criterion(outputs, labels)

  loss.backward()

  optimizer.step()

  # for name, param in model.named_parameters():
  #   if param.requires_grad:
  #     print(name, param.data)

  model.eval()
  print("[embedder.py] model: {}".format(model))

if __name__ == '__main__':
  test()


from collections import Counter

from flask import Flask, render_template, request
from apache_log_parser import make_parser
import pandas as pd

app = Flask(__name__)

def parse_log_file(file):
    try:
        parser = make_parser('%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"')

        with open(file, 'r') as f:
            logs = f.readlines()
            parsed_logs = [parser(line) for line in logs]
        # Преобразуем данные в DataFrame
        df = pd.DataFrame(parsed_logs)

        return df

    except:
        print(f"Failed to parse log file")
        return None

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        file = request.files['file']

        if file:
            file_path = f"{file.filename}"
            df = parse_log_file(file_path)

            if df is not None:
                # all
                data = df.to_dict()
                data_list = list(
                    zip(data['remote_host'], data['remote_host'].values(), data['request_method'].values(),
                        data['request_header_referer'].values(),
                        data['status'].values(), data['time_received_datetimeobj'].values(),
                        data['request_header_user_agent'].values()))

                # ban
                ban = []
                i = 0
                data_ban = list(data['remote_host'].values())
                res = Counter(data_ban).most_common(10)

                for key, val in res:
                    if val > 5:
                        ban.append({'N': i, 'ip': key, 'frequency': val})
                        i += 1

                return render_template('index.html', data=data_list, ban=ban)

    return render_template('index.html')

if __name__ == '__main__':
    app.run(debug=True)


import datetime
from typing import List

from . import setup, data_classes
from ..resources import values
from ..settings import hyper_parameters

def usd_to_quote(amount_usd, product_index):
    return amount_usd/setup.quote_currency_prices[product_index]

def usd_to_base(amount_usd, price_per_unit_usd):
    return amount_usd/price_per_unit_usd

def quote_to_usd(amount_quote, product_index):
    return amount_quote*setup.quote_currency_prices[product_index]

def base_to_usd(amount_base, price_per_unit_usd):
    return amount_base/price_per_unit_usd

def get_forecast(prediction_graph) -> int:
    dates = prediction_graph['dates']
    forecast = round(dates[-1] - dates[-2]) / 24 / 3600
    return int(forecast)

def investment_condition_satisfied(prediction_graph):
    investment_condition = hyper_parameters.investment_condition_factor*get_forecast(prediction_graph)
    currency_values = prediction_graph['predictions']
    prediction = currency_values[-1] / currency_values[-2] - 1
    if prediction > investment_condition:
        return True
    else:
        return False

def get_prediction_from_prediction_graph(prediction_graph):
    currency_values = prediction_graph['predictions']
    prediction = currency_values[-1] / currency_values[-2] - 1
    return prediction

def get_current_timestamp():
    current_datetime = datetime.datetime.combine(datetime.date.today(), values.min_time)
    current_timestamp = datetime.datetime.timestamp(current_datetime)
    return current_timestamp

def setup_investment_sizes(pending_investments: List[data_classes.PendingInvestment]):
    if len(pending_investments) == 0:
        return []
    investments = []
    forecast = pending_investments[0].get_forecast()
    for pending_investment in pending_investments:
        investment_condition = hyper_parameters.investment_condition_factor*forecast
        split_for_investment = hyper_parameters.investment / investment_condition * pending_investment.prediction / forecast
        investments.append(split_for_investment)
    capital = hyper_parameters.capital / forecast
    if sum(investments) > capital:
        factor = capital / sum(investments)
        investments = [n * factor for n in investments]
    for pending_investment, split_usd in zip(pending_investments, investments):
        pending_investment.setup_size(split_usd)
    return pending_investments

def get_product_id(base_currency):
    try:
        index = setup.base_currencies.index(base_currency)
        return setup.product_ids[index]
    except ValueError:
        return None

def update_trade_data_entry(trade_data, order_id, size, sell_date):
    for trade_data_entry in trade_data:
        if trade_data_entry['order_id'] == order_id:
            trade_data_entry['size'] = size
            trade_data_entry['sell_date'] = sell_date
    return trade_data

def remove_trade_data_entry(trade_data, order_id):
    trade_data = list(filter(lambda it: it['order_id'] != order_id, trade_data))
    return trade_data

def create_new_trade_data_entry(order_id, pending_investment: data_classes.PendingInvestment):
    trade_data_entry = {
        'order_id': order_id,
        'ticker': pending_investment.ticker,
        'size': None,
        'product_id': pending_investment.product_id,
        'buy_date': pending_investment.buy_date,
        'sell_date': pending_investment.sell_date
    }
    return trade_data_entry

def get_account_balance(currency):
    balance = None
    for account in setup.portfolio:
        if account['currency'] == currency:
            balance = account['balance']
    return float(balance)

def trade_history_entry_from_pending_order(pending_order, prediction_graphs, order_id):
    if isinstance(pending_order, data_classes.PendingInvestment):
        side = 'buy'
    elif isinstance(pending_order, data_classes.PendingDump):
        side = 'sell'
    else:
        raise ValueError('Input type {} not supported.'.format(type(pending_order)))
    current_market_price = None
    if get_current_market_price_usd(pending_order.ticker[0], prediction_graphs) is not None:
        current_market_price = \
            get_current_market_price_usd(pending_order.ticker[0], prediction_graphs)*pending_order.size
    trade_history_entry = {
        'order_id': order_id,
        'ticker': pending_order.ticker,
        'product_id': pending_order.product_id,
        'side': side,
        'size': pending_order.size,
        'value_usd': current_market_price,
        'date': datetime.datetime.now().timestamp()
    }
    return trade_history_entry

def get_current_market_price_usd(ticker, prediction_graphs):
    for prediction_graph in prediction_graphs:
        if prediction_graph['ticker'][0] == ticker:
            return prediction_graph['predictions'][-2]
    else:
        return None

def get_current_portfolio(portfolio_history, prediction_graphs):
    value_history = []
    for portfolio_history_entry in portfolio_history:
        value = 0
        for account in portfolio_history_entry['portfolio']:
            market_price_usd = get_current_market_price_usd(account['currency'], prediction_graphs)
            if market_price_usd is not None:
                value += market_price_usd*float(account['balance'])
            elif account['currency'] == 'USD':
                value += float(account['balance'])
            elif account['currency'] == 'EUR':
                value += float(account['balance'])*setup.euro_price_usd
        value_history.append({'value': value, 'date': datetime.datetime.now().timestamp()})

    return {'portfolio': portfolio_history[-1], 'value_history': value_history}

def get_filled_size(client, order_id):
    fills = client.get_fills(order_id=order_id)
    size = 0
    for fill in fills:
        size += float(fill['size'])
    return size

def float_to_string(value, decimal_places):
    return '{:.{}f}'.format(value, decimal_places)


import re

from flask_swagger_generator.components.swagger_models import SwaggerModel
from flask_swagger_generator.components.version_three.request_type \
    import SwaggerRequestType
from flask_swagger_generator.utils import RequestType
from flask_swagger_generator.components.version_three.tag import SwaggerTag
from flask_swagger_generator.components.version_three.operation_id import \
    SwaggerOperationId
from flask_swagger_generator.components.version_three.parameters import \
    SwaggerParameters
from flask_swagger_generator.components.version_three.path_parameter import \
    SwaggerPathParameter

class SwaggerPath(SwaggerModel):

    def __init__(self, func_name, group, path, request_types):
        super(SwaggerPath, self).__init__()
        self.func_name = func_name
        self.path = path
        self.group = group
        self.add_request_types(func_name, request_types)

    def add_request_types(self, function_name, request_types):

        for request_type in request_types:

            if request_type not in ['OPTIONS', 'HEAD']:
                swagger_request_type = SwaggerRequestType(
                    function_name,
                    RequestType.from_string(request_type)
                )
                tag = SwaggerTag(self.group)
                swagger_request_type.add_swagger_model(tag)
                swagger_request_type.add_swagger_model(
                    SwaggerOperationId(function_name)
                )
                self.add_swagger_model(swagger_request_type)

    def perform_write(self, file):
        self.format_path()
        path_entry = "'{path}':".format(path=self.path)
        path = self.indent(path_entry, self.TAB)
        file.write(path)
        file.write("\n")

    def format_path(self):

        if len(re.findall("<(.*?)>", self.path)) > 0:
            swagger_request_types = self.get_swagger_child_models_of_type(
                SwaggerRequestType
            )
            parameters = swagger_request_types[-1]\
                .get_swagger_child_models_of_type(
                    SwaggerParameters
            )

            if parameters:
                path_parameters = parameters[-1]\
                    .get_swagger_child_models_of_type(
                        SwaggerPathParameter
                )

                for path_parameter in path_parameters:
                    self.path = self.path.replace(
                        "<{}:{}>".format(
                            path_parameter.input_type
                            .get_flask_url_variable_type_value(),
                            path_parameter.name
                        ),
                        "{" + path_parameter.name + "}"
                    )


from django.shortcuts import render
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.shortcuts import get_object_or_404
from .models import News
# Create your views here.
def news(request):
    news = News.objects.all().order_by('-id')

    page_num = request.GET.get('page', 1)

    paginator = Paginator(news, 6) # 6 employees per page

    try:
        page_obj = paginator.page(page_num)
    except PageNotAnInteger:
        # if page is not an integer, deliver the first page
        page_obj = paginator.page(1)
    except EmptyPage:
        # if the page is out of range, deliver the last page
        page_obj = paginator.page(paginator.num_pages)

    for i in news:

        print(i.slug)

    context = {
        'page_obj': page_obj,
        'news': page_obj.object_list
    }

    return render(request, 'news.html', context)

def news_details(request, slug):
    news = get_object_or_404(News, slug = slug)
    related_news = News.objects.exclude(slug=news.slug).order_by('-id')[0:2]

    context = {
        'news' : news,
        'related_news' : related_news
        # 'navbar': 'news'
    }

    return render(request, 'news-details.html', context)


#coding=utf-8
import os,re
os.environ['DJANGO_SETTINGS_MODULE'] = 'decoder.settings'
import django
django.setup()
from baike.models import *

data = open('cn.txt').read().split('\n')
iterator = iter(data)
while True:
    try:
        t = iterator.next()
        s = iterator.next()
        d = iterator.next()
    except StopIteration:
        break

    data = t.split(':')
    if len(data) == 2:
        rsid = data[1]
        snp = Snp.objects.get(rsid=rsid)
        snp.summary_cn = s
        snp.description_cn = d
        snp.save()
    if len(data) == 3:
        rsid = data[1]
        allele = data[2]
        snp = Snp.objects.get(rsid=rsid)
        g = snp.genotype_set.get(allele=allele)
        g.summary_cn = s
        g.description_cn = d
        g.save()


from itertools import count
from typing import Optional
from flask import Blueprint, jsonify, request
from pydantic import BaseModel, Field
from flask_pydantic_spec import Response, Request
from lista_tarefas.extensions.pydantic import spec
from tinydb import TinyDB, Query

api = Blueprint('api', __name__)
database = TinyDB('database.json')
c = count()

class Item(BaseModel):
    id: Optional[int] = Field(default_factory=lambda: next(c))
    tarefa: str

class Itens(BaseModel):
    itens: list[Item]
    count: int

@api.route('/lista')
@spec.validate(resp=Response(HTTP_200=Itens))
def exibir_lista():
    """Exibe a lista completa junto com um contador de número de itens na lista"""
    return jsonify(
        Itens(
            itens=database.all(),
            count=len(database.all())
        ).dict()
    )

@api.route('/lista', methods=['POST'])
@spec.validate(body=Request(Item), resp=Response(HTTP_201=Item))
def inserir_item_na_lista():
    """Insere um item de tarefa na lista de tarefas."""
    body = request.context.body.dict()
    database.insert(body)
    return body

@api.route('/lista/<int:id>')
@spec.validate(resp=Response(HTTP_200=Item))
def buscar_item_lista(id):
    """Retorna um item da lista de tarefas passando na url seu id"""
    try:
        item = database.search(Query().id == id)[0]
    except IndexError:
        return {'message': 'Item not found'}, 404
    return jsonify(item)

@api.route('/lista/<int:id>', methods=['PUT'])
@spec.validate(body=Request(Item), resp=Response(HTTP_201=Item))
def altera_pessoa(id):
    """Altera os itens da lista baseado no id do item"""
    Item = Query()
    body = request.context.body.dict()
    database.update(body, Item.id == id)
    return jsonify(body)

@api.route('/lista/<int:id>', methods=['DELETE'])
@spec.validate(resp=Response('HTTP_204'))
def deleta_item(id):
    """Deleta um item da lista através do id repassado na URL do recurso"""
    Item = Query()
    database.remove(Item.id == id)
    return jsonify({})


'''
Created on 08.08.2017

@author: Laptop8460
'''

import threading
import time
import traceback
import socket
import json
from sys import platform

import logging
import stopAll

logger = logging.getLogger('subscription')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler("log/subscription.log")
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
fmtr = logging.Formatter("%(threadName)s %(asctime)s  %(levelname)s: %(message)s","%H:%M:%S")
ch.setFormatter(fmtr)
logger.addHandler(fh)
logger.addHandler(ch)

stopAllEvent=threading.Event()

startSubscriptionUpdateEvent=threading.Event()
publishEvent = threading.Event()
subscriberUpdaterDebug = threading.Event()
subscriberListenerDebug= threading.Event()
subscriberListenerStopEvent = threading.Event()
subscriberSenderStopEvent=threading.Event()

def startSubscriberListener(broadcastIp, broadcastPort):
    # start register interface
    subscriberListener = threading.Thread(name="subscriptionService",
                                          target=runSubscriberListener,
                                          args=[broadcastIp,broadcastPort])
    subscriberListener.start()
    stopAll.AddShutdownEvent(subscriberListenerStopEvent)   # triggers a stop
    stopAll.AddShutdownEvent(publishEvent)      # triggers a send

def runSubscriberListener(ip,port):
    subscriberListenerStopEvent.clear()
    sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
    if platform=="linux" or platform=="linux2":
        ipa=ip.split('.')[0:3]
        ipa.append('255')
        sock.bind(('.'.join(ipa), port))
        logger.info("START SUBSCRIBER LISTENER AT %s:%s (LINUX:%s)"%(str(ip),str(port),str(ipa)))
    else:
        sock.bind((ip, port))
        logger.info("START SUBSCRIBER LISTENER AT %s:%s (WINDOWS)"%(str(ip),str(port)))
    while True:
        # wait for message
        #logger.debug("Wait for receive")
        recv,info = sock.recvfrom(1024)
        if subscriberListenerStopEvent.isSet():
            subscriberListenerStopEvent.clear()
            logger.info("STOP SUBSCRIBER LISTENER")
            break

        #logger.debug("RECEIVED from %s:%s Data:%s"%(str(info[0]),str(info[1]), str(recv) ))
        if info[0]==ip:
            # own broadcast
            #logger.debug("OWN BROADCAST RECEIVED from %s:%s Data:%s"%(str(info[0]),str(info[1]), str(recv) ))
            if not broadcastSentEvent.isSet():
                logger.error("Error: own broadcast was not received")
            broadcastSentEvent.clear()
        else:
            jsonData = json.loads(recv.decode())
            action=jsonData["action"]
            paydeskObj=jsonData["paydesk"]
            paydesk = paydesk_ctrl.post(paydeskObj)


try:
    from methods.regular.regular_api import *
    from methods.source_control.file.remove import remove_core as file_remove_core
    from methods.source_control.file.file_browser import File_Browser

except:
    from default.methods.regular.regular_api import *
    from default.methods.source_control.file.remove import remove_core as file_remove_core
    from default.methods.source_control.file.file_browser import File_Browser

from sqlalchemy.orm.session import Session
from flasgger import swag_from

@routes.route('/api/v1/project/<string:project_string_id>/file/<int:file_id>/update-metadata', methods = ['PUT'])
@Project_permissions.user_has_project(
    Roles = ["admin", "Editor"],
    apis_user_list = ['api_enabled_builder', 'security_email_verified'])
@swag_from('../../../docs/files/file_update_metadata.yml')
def api_file_update_metadata(project_string_id: str, file_id: int):
    """
        Updates a file ID metadata.
    :param project_string_id: the project where the file belongs.
    :param file_id: the file ID to update
    :return:
    """

    spec_list = [
        {'rotation_degrees': {'type': int, 'required': False}},
        {'ordinal': {'type': int, 'required': False}}
    ]

    log, input, untrusted_input = regular_input.master(request = request,
                                                       spec_list = spec_list)
    if len(log["error"].keys()) >= 1:
        return jsonify(log = log), 400

    with sessionMaker.session_scope() as session:

        member = get_member(session)
        project = Project.get(session, project_string_id)

        updated_file, log = file_update_metadata_core(
            session = session,
            project = project,
            file_id = file_id,
            rotation_degrees = input['rotation_degrees'],
            ordinal = input['ordinal'],
            log = log
        )

        if len(log["error"].keys()) >= 1:
            return jsonify(log = log), 400
        log['success'] = True
        return jsonify(log = log, file = updated_file), 200

def file_update_metadata_core(session: Session,
                              project: Project,
                              file_id: int,
                              rotation_degrees: int,
                              ordinal: int,
                              log: dict = regular_log.default()):
    file = File.get_by_id(session = session, file_id = file_id)
    if file.project_id != project.id:
        log['error']['project'] = f'File does not belong to project {project.project_string_id}'
        return None, log

    if file.type == 'image':
        if rotation_degrees is not None:
            valid_degrees = [0, 90, 180, 270]
            if rotation_degrees not in valid_degrees:
                log['error']['rotation_degrees'] = f'Invalid rotation degrees can only be {valid_degrees}'
                return None, log
            file.image.rotation_degrees = rotation_degrees
            session.add(file.image)
    if ordinal is not None:
        file.ordinal = ordinal
    session.add(file)
    result = file.serialize_with_type(session = session, regen_url = False)
    return result, log


from calculation.sum import acumulator

#funcao de criacao da matriz que armazenara os valores de ∑(x^n) e os incrementos seguintes
def criaA(n, vX):
    matriz = list() #declaro a matriz
    mat_aux = list() #declaro um vetor auxiliar para a alocacao temporaria dos valores
    for l in range(0, n):
        for c in range(0, n):
            if l == 0 and c == 0:
                mat_aux.append(n)
            elif l == 0 and c > 0:
                soma = acumulator.somaX(n, vX, c)
                mat_aux.append(soma)
            elif l > 0 and c == 0:
                soma = acumulator.somaX(n, vX, l)
                mat_aux.append(soma)
            else:
                #if l == c:
                soma = acumulator.somaX(n, vX, (l + c))
                mat_aux.append(soma)
        matriz.append(mat_aux[:]) #criacao do vetor linha auxiliar, com valores zerados
        mat_aux.clear() #apago os valores no vetor auxiliar
    return(matriz)


import glob
import os

from ..job import Job

class Cleaner(Job):
    def __init__(self, **kwargs) -> None:
        super().__init__(**kwargs)

    def run(self):
        links = []
        links.extend(glob.glob('status/tasks/status-*.txt'))
        links.extend(glob.glob('status/tasks/init-*.json'))
        for link in links:
            os.remove(link)
            self.logger.debug(f'Status file {link} removed.')


import os
import subprocess
import time
from datetime import datetime

from win32comext.shell import shellcon, shell

from Base.FileDialogType import FileDialogType
from Base.MainQuit import MainQuit
from tkinter import Tk, filedialog, ttk
import tkinter as tk
from tkinter import *

from Base.TypeBgColor import TypeBgColor

class Merge(MainQuit):
    def selectFileDialogs(self,tv:Label,type:FileDialogType,showContext):
        self.setTvContext(showContext,"",TypeBgColor.defend)
        #选择目录
        path=None
        if type==FileDialogType.DIRECTORY:
            dialog=filedialog.askdirectory()
            if dialog is not None:
                path=dialog
        elif type==FileDialogType.DOCUMENT:#文件
            fileType=[("Xlsx",".xlsx")]
            dialog=filedialog.askopenfile(filetypes=fileType)
            if dialog is not  None:
                path=dialog.name
        tv.config(text=path)
    def showDialog(self, mainRoot):
        root = Tk()
        root.title("合成ts文件")
        root.geometry("640x600")
        root.config(bg="light gray")
        mFont = 10
        mSticky = W + E + S + N
        mWraplength = 300,
        mRlief = "groove"
        rowNum = 0
        showContext: Label
        Label(master=root, text="文件目录:", font=mFont, relief=mRlief).grid(row=rowNum, column=0, sticky=mSticky,
                                                                             pady=4)
        selectTsFileTv = Label(master=root, text="", font=mFont,
                               wraplength=mWraplength, relief=mRlief)
        selectTsFileTv.grid(row=rowNum, column=1, sticky=mSticky, pady=4)

        Button(master=root, text="选择", font=mFont, relief=mRlief,
               command=lambda: self.selectFileDialogs(selectTsFileTv, FileDialogType.DIRECTORY, showContext)).grid(
            row=rowNum, column=2,
            sticky=mSticky,
            pady=4)
        childLength = root.children.__len__()

        for index in range(childLength):
            root.columnconfigure(index, weight=1)

        rowNum += 1
        Label(master=root, text="保存目录:", font=mFont, relief=mRlief).grid(row=rowNum, column=0, sticky=mSticky,
                                                                             pady=4)
        savceFileTsTV = Label(master=root, text="", font=mFont,
                              wraplength=mWraplength, relief=mRlief)
        savceFileTsTV.grid(row=rowNum, column=1, sticky=mSticky, pady=4)

        Button(master=root, text="选择", font=mFont, relief=mRlief,
               command=lambda: self.selectFileDialogs(savceFileTsTV, FileDialogType.DIRECTORY, showContext)).grid(
            row=rowNum, column=2,
            sticky=mSticky,
            pady=4)

        rowNum += 1
        Label(master=root, text="保存文件名称:", font=mFont, relief=mRlief).grid(row=rowNum, column=0, sticky=mSticky,
                                                                                pady=4)
        editName = Entry(master=root, font=mFont, relief=mRlief)
        editName.grid(row=rowNum, column=1, columnspan=2, sticky=mSticky, pady=4)

        rowNum += 1
        Label(master=root, text="保存格式:", font=mFont, relief=mRlief).grid(row=rowNum, column=0, sticky=mSticky,
                                                                             pady=4)
        comboboxCopySaveType = ttk.Combobox(font=mFont, master=root, justify=tk.CENTER)
        comboboxCopySaveType.grid(row=rowNum, column=1, sticky=mSticky, pady=4)
        comboboxCopySaveType['values'] = ["TS", "MP4"]
        comboboxCopySaveType['state'] = "readonly"
        comboboxCopySaveType.config(font=mFont)
        comboboxCopySaveType.current(0)
        Button(master=root, text="copy /b 合成", font=mFont, relief=mRlief,
               command=lambda: self.mergefunction(selectTsFileTv, savceFileTsTV, editName, comboboxCopySaveType,showContext)).grid(
            row=rowNum, column=2,
            sticky=mSticky,
            pady=4)
        rowNum += 1
        Label(master=root, text="保存格式:", font=mFont, relief=mRlief).grid(row=rowNum, column=0, sticky=mSticky,
                                                                                pady=4)
        comboboxSaveType = ttk.Combobox(font=mFont, master=root, justify=tk.CENTER)
        comboboxSaveType.grid(row=rowNum, column=1, sticky=mSticky, pady=4)
        comboboxSaveType['values'] = ["TS", "MP4"]
        comboboxSaveType['state'] = "readonly"
        comboboxSaveType.config(font=mFont)
        comboboxSaveType.current(0)
        Button(master=root, text="ffmpeg  合成", font=mFont, relief=mRlief,
               command=lambda: self.mergefunctionffmpeg(selectTsFileTv, savceFileTsTV, editName, comboboxSaveType,showContext)).grid(
            row=rowNum, column=2, sticky=mSticky,
            pady=4)

        rowNum += 1
        Button(master=root, text="删除FFmpeg 合成txt文件", font=mFont, relief=mRlief,
               command=lambda: self.delecetFfmpegTxt(selectTsFileTv,showContext)).grid(row=rowNum, column=0,columnspan=3, sticky=mSticky,
                                                                  pady=4)
        rowNum += 1
        showContext = Label(master=root, height=10, font=mFont, wraplength=600, bg="#5EA4DE")
        showContext.grid(row=rowNum, column=0, columnspan=3, sticky=mSticky, pady=4)

        self.registLisetener(root, mainRoot)
        mainloop()
        pass

    pass

    def mergefunction(self, selectTsFileTv, savceFileTsTV, editName, comboboxCopySaveType,showContext):
        selectTsFile = selectTsFileTv.cget("text")
        if selectTsFile == "" or savceFileTsTV is None:
            self.setTvContext(showContext, "========Waring========\n   请选择文件目录", TypeBgColor.waring)
            return

        saveTsFile = savceFileTsTV.cget("text")
        if saveTsFile == "" or saveTsFile is None:
            self.setTvContext(showContext, "========Waring========\n   请选择保存文件目录", TypeBgColor.waring)
            return
        name = editName.get()
        if name == "" or name is None:
            self.setTvContext(showContext, "========Waring========\n   请输入保持的名称", TypeBgColor.waring)
            return
        savetype = comboboxCopySaveType.get()
        savetxt = f"{saveTsFile}/{name}.{savetype.lower()}"
        pathlist = selectTsFile + "/*.ts"
        print(f"savetxt=={savetxt}")
        print(f"pathlist=={selectTsFile}")
        saveReplace = savetxt.replace("/", "\\")
        selectReplace = pathlist.replace("/", "\\")
        cmdd = f"copy /b {selectReplace}  {saveReplace}"
        os.system(f"start cmd.exe /k   {cmdd}")
        self.setTvContext(showContext, "========Success========\n   执行成功", TypeBgColor.Success)

        pass

    def mergefunctionffmpeg(self, selectTsFileTv, savceFileTsTV, editName, comboboxSaveType,showContext):
        selectTsFile = selectTsFileTv.cget("text")
        if selectTsFile == "" or savceFileTsTV is None:
            self.setTvContext(showContext, "========Waring========\n   请选择文件目录", TypeBgColor.waring)
            return

        saveTsFile = savceFileTsTV.cget("text")
        if saveTsFile == "" or saveTsFile is None:
            self.setTvContext(showContext, "========Waring========\n   请选择保存文件目录", TypeBgColor.waring)
            return
        name = editName.get()
        if name == "" or name is None:
            self.setTvContext(showContext, "========Waring========\n   请输入保持的名称", TypeBgColor.waring)
            return

        selectPath = os.listdir(selectTsFile)
        len__ = selectPath.__len__()
        if len__==0:
            self.setTvContext(showContext,"========Waring========\n   选择的目录下没有ts文件", TypeBgColor.waring)
            return
        selectPath.sort()
        # start = datetime.now()
        # print('开始合成，初始时间为:', datetime.now())
        savetype = comboboxSaveType.get()
        pathlist = saveTsFile + f"/{name}.{savetype.lower()}"
        fileStr = "concat:"
        # for index in range(len__):
        #     print(f"index=={index}")
        #     if index == len__ - 1:
        #         fileStr += f"{selectTsFile}/{selectPath[index]}"
        #     else:
        #         fileStr += f"{selectTsFile}/{selectPath[index]}|"

        if 'file_list.txt' in selectPath:
            os.remove(saveTsFile + 'file_list.txt')

        f = open(saveTsFile + '/file_list.txt', 'w+')

        for index in selectPath:
            f.write("file '"+f"{selectTsFile}/{index}"+"'\n")
        time.sleep(1)
        # for one in file_names:
        #     f.write("file '" + one + "'\n")
        filenme = saveTsFile + f"/file_list.txt"
        # cmdd = f"ffmpeg -i "concat:1.ts|2.ts" -c copy output.mp4"
        print(f"filename =={filenme}")
        # cmdd = f"ffmpeg -i \"{fileStr}\" -c copy {pathlist}"
        cmdd = f"ffmpeg -f concat -safe 0 -i  \"{filenme}\" -c copy {pathlist}"
        print(cmdd)
        os.system(f"start cmd.exe /k   {cmdd}")
        # print('合成后的当前时间为：', datetime.now())
        # print('合成视频完成！用时：' + str(datetime.now() - start))
        self.setTvContext(showContext, "========Success========\n   执行成功", TypeBgColor.Success)
        pass

        pass

    def delecetFfmpegTxt(self, selectTsFileTv, showContext):
        selectPath = selectTsFileTv.cget("text")
        if selectPath=="" or selectPath is None:
            self.setTvContext(showContext, "========Waring========\n   请选择文件目录", TypeBgColor.waring)
            return

        path=selectPath+"/file_list.txt"
        if os.path.exists(path):
            listdir = os.listdir(selectPath)
            for item in listdir:
                # os.remove(selectPath+"/"+item)
                res = shell.SHFileOperation((0, shellcon.FO_DELETE, selectPath+"/"+item, None,
                                             shellcon.FOF_SILENT | shellcon.FOF_ALLOWUNDO | shellcon.FOF_NOCONFIRMATION,
                                             None, None))  # 删除文件到回收站
                if not res[1]:
                    os.system('del ' + selectPath+"/"+item)

            self.setTvContext(showContext, "========Success========\n   删除成功", TypeBgColor.Success)
        else:
            self.setTvContext(showContext, f"========Waring========\n   {path}删除文件不存在", TypeBgColor.waring)

        pass


from django.urls import path
from .views import LoginView, RegisterView, UpdateView, user_logout, gallery

urlpatterns = [
    path('signin/', LoginView.as_view(), name='login'),
    path('signup/', RegisterView.as_view(), name='register'),
    path('userupdate/', UpdateView.as_view(), name='update'),
    path('logout/', user_logout, name='logout'),
    path('gallery/', gallery)

]


import os

begining = "program_files +=  [('"
ending = ", 'DATA')]"
with open("imgScanSpec.txt", "w" ) as textFile:
	path = os.path.join(os.getcwd(), "img\\")
	pathForPrint = path.replace("\\", "\\\\")
	for file in os.listdir(path):
		if file.endswith(".png"):
			fullPath = pathForPrint + file
			textFile.write( begining + file + "', '" + fullPath + "'" + ending + "\n")


"""
Functions to convert pmfs to latex arrays.
"""

import contextlib
import os
import subprocess  # noqa: S404
import tempfile

from debtcollector import removals

import numpy as np
import numpy.core.arrayprint as arrayprint

from .context import cd, named_tempfile, tempdir
from .misc import default_opener

__all__ = (
    'printoptions',
    'to_latex',
    'to_pdf',
)

# http://stackoverflow.com/questions/2891790/pretty-printing-of-numpy-array
#
# Includes hack to prevent NumPy from removing trailing zeros.
#
@removals.remove(message="Please use np.core.arrayprint.printoptions",
                 version='1.2.3')
@contextlib.contextmanager
def printoptions(strip_zeros=True, **kwargs):
    if strip_zeros:
        kwargs['trim'] = 'k'
    origcall = arrayprint.FloatingFormat.__call__

    def __call__(self, x):
        return origcall.__call__(self, x)

    arrayprint.FloatingFormat.__call__ = __call__
    original = np.get_printoptions()
    np.set_printoptions(**kwargs)
    yield
    np.set_printoptions(**original)
    arrayprint.FloatingFormat.__call__ = origcall

def to_latex__numerical(a, decimals, tab):
    # The elements of each column are aligned on the decimal, if present.
    # Spacing is automatically adjusted depending on if `a` contains negative
    # numbers or not. For float data, trailing zeros are included so that
    # array output is uniform.

    array = np.atleast_2d(a)
    array = np.around(array, decimals)

    # Determine the number of digits left of the decimal.
    # Grab integral part, convert to string, and get maximum length.
    # This does not handle negative signs appropriately since -0.5 goes to 0.
    # So we make sure it never counts a negative sign by taking the abs().
    integral = np.abs(np.trunc(array.flat).astype(int))
    left_digits = max(map(len, map(str, integral)))

    # Adjust for negative signs.
    if np.any(array < 0):
        left_digits += 1

    # Set decimal digits to 0 if data are not floats.
    try:
        np.finfo(array.dtype)
    except ValueError:
        decimals = 0

    # Align the elements on the decimal, making room for largest element.
    coltype = r"\newcolumntype{{X}}{{D{{.}}{{.}}{{{0},{1}}}}}"
    coltype = coltype.format(left_digits, decimals)

    # Specify that we want all columns to have the same column type.
    nCols = array.shape[1]
    cols = r"*{{{nCols}}}{{X}}".format(nCols=nCols)

    # Build the lines in the array.
    #
    # In general, we could just use the rounded array and map(str, row),
    # but NumPy strips trailing zeros on floats (undesirably). So we make
    # use of the context manager to prevent that.
    options = {
        'precision': decimals,
        'suppress': True,
        'strip_zeros': False,
        'threshold': nCols + 1,
    }
    with printoptions(**options):
        lines = []
        for row in array:
            # Strip [ and ], remove newlines, and split on whitespace
            elements = row.__str__()[1:-1].replace('\n', '').split()
            # hack to fix trailing zeros, really the numpy stuff needs to be updated.
            try:
                elements = [element + '0' * (decimals - len(element.split('.')[1])) for element in elements]
            except:  # noqa; S110
                pass
            line = [tab, ' & '.join(elements), r' \\']
            lines.append(''.join(line))

    # Remove the \\ on the last line.
    lines[-1] = lines[-1][:-3]

    # Create the LaTeX code
    subs = {'coltype': coltype, 'cols': cols, 'lines': '\n'.join(lines)}
    template = r"""{coltype}
\begin{{array}}{{{cols}}}
{lines}
\end{{array}}"""

    return template.format(**subs)

def to_latex__exact(a, tol, tab):

    from dit.math import approximate_fraction

    array = np.atleast_2d(a)

    to_frac = lambda f: approximate_fraction(f, tol)
    fractions = np.array(list(map(to_frac, array.flat))).reshape(array.shape)

    # Specify that we want all columns to have the same column type.
    nCols = array.shape[1]
    cols = r"*{{{nCols}}}{{c}}".format(nCols=nCols)

    def to_frac(f):
        if f.denominator != 1:
            return r'\frac{{{}}}{{{}}}'.format(f.numerator, f.denominator)
        else:
            return str(f.numerator)

    # Build the lines in the array.
    lines = []
    for row in fractions:
        # Strip [ and ], remove newlines, and split on whitespace
        elements = map(to_frac, row)
        line = [tab, ' & '.join(elements), r' \\']
        lines.append(''.join(line))

    # Remove the \\ on the last line.
    lines[-1] = lines[-1][:-3]

    # Create the LaTeX code
    subs = {'cols': cols, 'lines': '\n'.join(lines)}
    template = r"""\begin{{array}}{{{cols}}}
{lines}
\end{{array}}"""

    return template.format(**subs)

def to_latex(a, exact=False, decimals=3, tab='  '):
    r"""
    Convert an array-like object into a LaTeX array.

    Parameters
    ----------
    a : array-like
        A list, tuple, NumPy array, etc. The elements are written into a
        LaTeX array environment.

    exact : bool
        When `exact` is False, the elements of each column are aligned on the
        decimal, if present. Spacing is automatically adjusted depending on if
        `a` contains negative numbers or not. For float data, trailing zeros
        are included so that array output is uniform.

        When `exact` is `True`, then each float is turned into a fraction and
        placed in a column. One may specify `exact` as a small float to be used
        as the tolerance while converting to a fraction.

    decimals : int
        The number of decimal places to round to before writing to LaTeX.

    tab : str
        The tab character to use for indentation within LaTeX.

    Examples
    --------
    >>> x = [1,2,-3]
    >>> print to_latex(x)
    \newcolumntype{X}{D{.}{.}{2,0}}
    \begin{array}{*{3}{X}}
      1 & 2 & -3
    \end{array}

    >>> import numpy as np
    >>> np.random.seed(0)
    >>> y = np.random.rand(12).reshape(4,3) - 0.5
    >>> print to_latex(y, decimals=2)
    \newcolumntype{X}{D{.}{.}{2,2}}
    \begin{array}{*{3}{X}}
      0.05 & 0.22 & 0.10 \\
      0.04 & -0.08 & 0.15 \\
      -0.06 & 0.39 & 0.46 \\
      -0.12 & 0.29 & 0.03
    \end{array}

    Notes
    -----
    The resultant array environment should be used within LaTeX's mathmode,
    and the following should appear in the preamble:

        \usepackage{array}
        \usepackage{dcolumn}

    """
    if exact:
        if exact is True:
            tol = 1e-6
        else:
            tol = exact
        return to_latex__exact(a, tol, tab)
    else:
        return to_latex__numerical(a, decimals, tab)

def to_pdf(a, exact=False,
              decimals=3,
              line="p(x) = \\left[\n{0}\n\\right]",
              show=True):
    """
    Converts a NumPy array to a LaTeX array, compiles and displays it.

    Examples
    --------
    >>> a = np.array([[.1, .23, -1.523],[2.1, .23, .523]])
    >>> to_pdf(a) # pragma: no cover

    """
    template = r"""\documentclass{{article}}
\usepackage{{amsmath}}
\usepackage{{array}}
\usepackage{{dcolumn}}
\pagestyle{{empty}}
\begin{{document}}
\begin{{displaymath}}
{0}
\end{{displaymath}}
\end{{document}}"""

    fline = line.format(to_latex(a, exact=exact, decimals=decimals))
    latex = template.format(fline)

    with contextlib.ExitStack() as stack:  # pragma: no cover
        EC = stack.enter_context
        tmpdir = EC(tempdir())
        EC(cd(tmpdir))
        latexfobj = EC(named_tempfile(dir=tmpdir, suffix='.tex'))

        # Write the latex file
        latexfobj.write(latex.encode('utf8'))
        latexfobj.close()

        # Compile to PDF
        args = ['pdflatex', '-interaction=batchmode', latexfobj.name]
        with open(os.devnull, 'w') as fp:
            subprocess.call(args, stdout=fp, stderr=fp)  # noqa: S603
            subprocess.call(args, stdout=fp, stderr=fp)  # noqa: S603

        # Create another tempfile which will not be deleted.
        pdffobj = tempfile.NamedTemporaryFile(suffix='_pmf.pdf', delete=False)
        pdffobj.close()

        # Crop the PDF and copy to persistent tempfile.
        pdfpath = latexfobj.name[:-3] + 'pdf'
        # Cannot add &>/dev/null to cmd, as Ghostscript is unable to find the
        # input file. This seems to be some weird interaction between
        # subprocess and pdfcrop. Also, we need to use shell=True since some
        # versions of pdfcrop rely on a hack to determine what perl interpreter
        # to call it with.
        cmd = r'pdfcrop --debug {} {}'.format(pdfpath, pdffobj.name)
        with open(os.devnull, 'w') as fp:
            subprocess.call(cmd, stdout=fp)  # noqa: S603

        # Open the PDF
        if show:
            default_opener(pdffobj.name)

        return pdffobj.name


from __future__ import print_function
import sys
import os
from pymongo import MongoClient
from pymongo.errors import InvalidDocument
import json

class factory(object):
    """ output class generator helper """
    def __init__(self, config, resconf, dry_run=False):
        outconf = config.getsection("outputdatabase")

        if 'db_engine' not in outconf and 'type' in outconf:
            # Older versions of the documentation recommended 'type' as the
            # configuration parameter use this if 'db_engine' is absent
            outconf['db_engine'] = outconf['type']

        if outconf['db_engine'].lower() == "mongodb":
            self._impl = MongoOutput(outconf, resconf) if not dry_run else NullOutput()
        elif outconf['db_engine'].lower() == "stdout":
            self._impl = StdoutOutput(outconf, resconf)
        elif outconf['db_engine'] == 'file':
            self._impl = FileOutput(outconf, resconf)
        else:
            raise Exception("Unsupported output mechanism {0}".format(outconf['db_engine']))

    def __enter__(self):
        return self._impl.__enter__()

    def __exit__(self, exception_type, exception_val, trace):
        return self._impl.__exit__(exception_type, exception_val, trace)

class FileOutput(object):
    """
    Dumps output into a file in one of two fashions
    1. Fragment - dumps snippets of json (currently one json object per job) into
                  a file as the process runs. This file will NOT be valid json, however
                  useful for pseudo interactive debugging purposes.
    2. Complete - dumps entire result as one large json array at end of process.
                  File will be empty during runtime, but will be valid json.
    """
    def __init__(self, outconf, resconf):
        self._resid = resconf['resource_id']

        jsonoption = outconf['json_format']
        if jsonoption == 'both':
            self._fragjson = True
            self._completejson = True
        elif jsonoption == 'fragment':
            self._fragjson = True
            self._completejson = False
        elif jsonoption == 'complete':
            self._fragjson = False
            self._completejson = True
        else:
            raise Exception("Not a valid json option {0}".format(jsonoption))

        if self._fragjson:
            self._fragpath = outconf['frag_file']
        if self._completejson:
            self._comppath = outconf['comp_file'].replace("%r", resconf["name"])

        if self._fragjson:
            self._fragfile = open(self._fragpath, 'w')
        if self._completejson:
            self._jsonarray = []

    def __enter__(self):
        return self

    def process(self, summary, mdata):
        """
        json print
        """
        if self._fragjson:
            print(self._resid, json.dumps(summary, indent=4, default=str), file=self._fragfile)
            print("MDATA: ", json.dumps(mdata, indent=4, default=str), file=self._fragfile)
        if self._completejson:
            self._jsonarray.append(summary)
            self._jsonarray.append(mdata)

    def __exit__(self, exception_type, exception_val, trace):
        if self._fragjson:
            self._fragfile.close()
        if self._completejson:
            with open(self._comppath, 'w') as f:
                json.dump(self._jsonarray, f, indent=4, default=str)

class MongoOutput(object):
    """ Support for mongodb output """
    def __init__(self, outconf, resconf):
        self._uri = outconf['uri']
        self._dname = outconf.get('dbname', outconf.get('db', 'supremm'))
        self._collection = "resource_" + str(resconf['resource_id'])
        self._timeseries = "timeseries-" + self._collection
        self._client = None
        self._outdb = None

    def __enter__(self):
        self._client = MongoClient(host=self._uri)
        self._outdb = self._client[self._dname]
        return self

    def process(self, summary, mdata):
        """ output the summary record """

        mongoid = str(summary['acct']['id']) + '-' + str(summary['acct']['end_time'])
        summary['_id'] = mongoid
        summary['summarization'].update(mdata)

        if 'timeseries' in summary:
            summary['timeseries']['_id'] = summary["_id"]
            self._outdb[self._timeseries].update({"_id": summary["_id"]}, summary['timeseries'], upsert=True)
            del summary['timeseries']

        self._outdb[self._collection].update({"_id": summary["_id"]}, summary, upsert=True)

    def __exit__(self, exception_type, exception_val, trace):
        if self._client != None:
            self._outdb = None
            self._client.close()
            self._client = None

class StdoutOutput(object):
    """
    Simple outputter that dumps the job summary to stdout. Intended for debug purposes.
    """
    def __init__(self, _, resconf):
        self._resid = resconf['resource_id']

    def __enter__(self):
        return self

    def process(self, summary, mdata):
        """
        json print
        """
        print(self._resid, json.dumps(summary, default=str, indent=4))
        print("MDATA: ", json.dumps(mdata, default=str, indent=4))

    def __exit__(self, exception_type, exception_val, trace):
        pass

class NullOutput(object):
    """
    Outputter used when configured outputter is a database (Mongo) but the dry-run option is set.
    Discards all data.
    """
    def __enter__(self):
        return self

    def process(self, summary, mdata):
        pass

    def __exit__(self, exception_type, exception_val, trace):
        pass