import time
from datetime import datetime, date, timedelta

from genshi.builder import tag

from trac.core import Component, implements
from trac.mimeview.api import Context
from trac.ticket.model import Ticket, Milestone
from trac.ticket.query import Query, QueryModule
from trac.ticket.web_ui import TicketModule
from trac.util.compat import any
from trac.util.datefmt import parse_date, utc
from trac.util.text import empty
from trac.web.api import IRequestHandler, IRequestFilter
from trac.web.chrome import (
    ITemplateProvider, INavigationContributor, Chrome, add_stylesheet,
    add_script, add_ctxtnav, add_script_data,
)
from trac.wiki.api import IWikiMacroProvider, parse_args

try:
    from babel import Locale
    from babel.core import LOCALE_ALIASES, UnknownLocaleError
    from babel.dates import get_day_names, get_month_names, format_date
except ImportError:
    LOCALE_ALIASES = {}
    def get_day_names(width=None, context=None, locale=None):
        names = dict()
        for day in range(0, 7):
            dt = datetime(2001, 1, day + 1)
            names[day] = dt.strftime('%a')
        return names
    def get_month_names(width=None, context=None, locale=None):
        names = dict()
        for m in range(1, 13):
            dt = datetime(2001, m, 1)
            names[m] = dt.strftime('%B')
        return names
    def format_date(date=None, format=None, locale=None):
        return str(date)

from ticketcalendar.api import (
    _, tag_, N_, gettext, add_domain, TEXTDOMAIN,
    Option, IntOption, ListOption,
)

# Utility functions

def _get_month_name(date, loc):
    m = date.month
    return get_month_names('wide', locale=loc)[m]

def _getdate(d, tz):
    try:
        if d.tzinfo is tz:
            return d.date()
        # Convert self to UTC, and attach the new time zone object.
        utc = (d - d.utcoffset()).replace(tzinfo=tz)
        # Convert from UTC to tz's local time.
        return tz.fromutc(utc).date()
    except:
        return None

def _parse_month_arg(string):
    if string:
        try:
            tt = time.strptime(string, '%Y-%m')
            return date(*tt[:3])
        except:
            pass
    return datetime.now().date().replace(day=1)

def _parse_duration_arg(arg):
    default = datetime.now(utc).date(), 7
    if not arg or '/P' not in arg:
        return default
    start, period = arg.split('/P', 1)
    try:
        start = parse_date(start, tzinfo=utc)
    except:
        start = None
    if not start:
        return default
    start = start.date()
    if period.endswith('D'):
        try:
            period = int(period[:-1])
        except:
            period = 0
        if period > 0:
            return start, period
    return default

"""
Output html data for Ticket calendar.
"""
class TicketCalendar(object):
    def __init__(self, env, req):
        self.env = env
        self.config = env.config
        self.log = env.log
        self.req = req
        self.mod = mod = TicketCalendarModule(env)
        self._priority_colors = self._prepare_list('priority',
                                                   mod.priority_colors)
        self._ticket_type_icons = self._prepare_list('ticket_type',
                                                     mod.ticket_type_icons)

    @classmethod
    def build_query_string(cls, constraints):
        args = []
        for idx, c in enumerate(constraints):
            if idx != 0:
                args.append(('or', empty))
            for key, val in c.iteritems():
                if isinstance(val, (list, tuple)):
                    val = '|'.join(val)
                args.append((key, val))
        def to_pair(key, val):
            if val is empty:
                return key
            return key + '=' + val
        return '&'.join(to_pair(key, val) for key, val in args)

    def get_priority_color(self, name):
        return self._priority_colors.get(name, '#fca89e')

    def get_ticket_type_icon(self, name):
        return self._ticket_type_icons.get(name, 'ui-icon-contact')

    def _prepare_list(self, type, values):
        values = filter(None, values)
        if not values:
            return {}
        db = self.env.get_read_db()
        cursor = db.cursor()
        cursor.execute('SELECT name FROM enum WHERE type=%s'
                       ' ORDER BY value',
                       (type,))
        names = [row[0] for row in cursor]
        cycle = []
        data = {}
        for idx, value in enumerate(values):
            value = value.strip()
            if not value:
                continue
            if ':' in value:
                name, value = value.split(':', 1)
                name = name.strip()
                value = value.strip()
                data[name] = value
                if name in names:
                    names.remove(name)
            else:
                cycle.append(value)
        data.update((name, cycle[idx % len(cycle)])
                    for idx, name in enumerate(names))
        return data

    def _get_href(self, name, query, kwargs):
        name = 'ticketcalendar-' + name
        args = []
        for idx, c in enumerate(query.constraints):
            if idx != 0:
                args.append(('or', empty))
            args.extend(c.iteritems())
        args.append(('order', query.order))
        if query.desc:
            args.append(('desc', 1))
        args.extend(kwargs.iteritems())
        return self.req.href(name, args)

    def get_list_href(self, query, start=None, period=7):
        kwargs = {}
        if start is not None:
            kwargs['_duration'] = '%s/P%dD' % (start.strftime('%Y-%m-%d'), period)
        return self._get_href('list', query, kwargs)

    def get_box_href(self, query, month=None):
        kwargs = {}
        if month:
            kwargs['_month'] = month.strftime('%Y-%m')
        return self._get_href('box', query, kwargs)

    @property
    def default_query(self):
        return self.config.get('query', 'default_query')

    @property
    def default_anonymous_query(self):
        return self.config.get('query', 'default_anonymous_query')

    def _create_milestone_item(self, milestone):
        icon = tag.span(tag.span('', class_='ui-icon %s' %
                                            self.mod.milestone_icon or ''),
                        class_='ui-icon-w')
        anchor = tag.a(milestone.name,
                       href=self.req.href('milestone', milestone.name))
        item = tag.li(icon, anchor,
                      style='background-color: %s;' %
                            self.mod.milestone_background_color,
                      class_='milestone')
        return item

    def _create_ticket_item(self, t):
        req = self.req
        style = None
        tktid = t.get('id')
        background = self.get_priority_color(t.get('priority'))
        if background:
            style = 'background-color: %s' % background
        item = tag.li(title='#%d %s' % (t.get('id'), t.get('summary')),
                      style=style,
                      data_href=req.href('ticketcalendar-ticket', id=tktid))

        for key, icon_class in (('_begin', 'arrowthick-1-e'),
                                ('_end',   'arrowthick-1-w')):
            if t.get(key):
                class_ = 'ui-icon ui-icon-' + icon_class
                item(tag.span(tag.span('', class_=class_), class_='ui-icon-w'))

        if t.get('_beginend'):
            item(tag.span(u'\u2666',
                          style='display: inline-block; margin: 0 0.4em;'))

        icon = self.get_ticket_type_icon(t.get('type'))
        if icon:
            item(tag.span(tag.span('', title=t.get('type'),
                                   class_ = 'ui-icon %s' % icon),
                          class_ = 'ui-icon-w'))
        else:
            item(tag.span(t.get('type'), class_='type'))

        item(tag.span(tag.a('#%d' % tktid,
                            href=req.href('ticket', tktid),
                            class_='open-ticketcalendar-ticket-detail %s' %
                                   t.get('status')),
                      u'\u00a0',
                      t.get('summary')))

        return item

    def _get_cols(self):
        cols = ['id','time','status','summary','priority',
                'milestone','owner','type','description']
        cols.extend(name
                    for name in set((self.mod.start_date_name,
                                     self.mod.due_date_name))
                    if name not in cols)
        return cols

    def _set_no_paginator(self, query):
        query.max = 0
        query.has_more_pages = False
        query.offset = 0
        query.group = None

    def get_query_from_string(self, qstring):
        query = Query.from_string(self.env, qstring)
        self._set_no_paginator(query)
        return query

    def get_constraints(self, arg_list=[]):
        return QueryModule(self.env)._get_constraints(self.req, arg_list)

    def get_query(self, constraints=None, args=None):
        req = self.req
        if not args:
            args = req.args
        if not constraints:
            # If no constraints are given in the URL, use the default ones.
            if req.authname and req.authname != 'anonymous':
                qstring = self.default_query
                user = req.authname
            else:
                email = req.session.get('email')
                name = req.session.get('name')
                qstring = self.default_anonymous_query
                user = email or name or None

            query = Query.from_string(self.env, qstring)
            args = {'order': query.order, 'col': query.cols, 'max': 0}
            constraints = query.constraints

            # Substitute $USER, or ensure no field constraints that depend
            # on $USER are used if we have no username.
            for clause in constraints:
                for field, vals in clause.items():
                    for (i, val) in enumerate(vals):
                        if user:
                            vals[i] = val.replace('$USER', user)
                        elif val.endswith('$USER'):
                            del clause[field]
                            break

        query = Query(self.env, args.get('report'), constraints,
                      self._get_cols(), args.get('order'), args.get('desc'))
        self._set_no_paginator(query)
        return query

    def template_data(self, req, query, kwargs=None):
        db = self.env.get_read_db()
        tickets = query.execute(req, db)
        context = Context.from_request(req, 'query')
        return query.template_data(context, tickets, None, datetime.now(utc),
                                   req)

    def gen_calendar(self, tickets, query, month, width=None, nav=True):
        milestones = self._get_milestones()

        req = self.req
        locale = self._get_locale()
        first_week_day = self._get_first_week_day(locale)
        start_date_format = self.mod.start_date_format
        due_date_format = self.mod.due_date_format

        if not month:
            month = datetime.now(req.tz)
            month = datetime(month.year, month.month, 1).date()

        # init data
        today = datetime.now(req.tz).date()

        # generate calendar data
        weeks = self._get_month_calendar(month.year, month.month,
                                         first_week_day, locale)
        days = sorted(sum(weeks, []))
        tickets = self._filter_tickets(days, tickets, req.tz)
        milestones = self._filter_milestones(days, milestones, req.tz)
        cal = [[{'date': day,
                 'tickets': tickets[day], 'milestones': milestones[day]}
                for day in week]
               for week in weeks]

        def genli(t):
            if isinstance(t, Milestone):
                return self._create_milestone_item(t)
            else:
                return self._create_ticket_item(t)

        def gentd(week_idx, day_info):
            day = day_info['date']
            tt = day_info['milestones'] + day_info['tickets']
            if len(tt) < 6:
                ttshow = tt
                ttall = []
            else:
                ttshow = tt[:4]
                ttall = tt

            tdclass = []
            if day == today:
                tdclass.append('today')
            if day.weekday() in (5, 6):
                tdclass.append('weekend')

            formatted_day = format_date(day, format='long', locale=locale)

            td = tag.td(
                class_=' '.join(tdclass),
                data_for_start_date=day.strftime(start_date_format),
                data_for_due_date=day.strftime(due_date_format),
                data_fdate=formatted_day)

            label = []
            if day == today:
                label.append(tag.span(_("Today"), class_='today'))
            label.append(tag.span(unicode(day.day),
                                  class_=('day normal', 'day')[day == today]))
            td(tag.div(label))
            if ttshow:
                td(tag.ul([genli(t) for t in ttshow]))

            if ttall:
                id = 'calendar-more-%s' % str(day)
                td(tag.a(_("%d more tickets...") % (len(ttall) - 4),
                         href='#' + id,
                         class_='show-all-list'),
                   tag.div(tag.h4(formatted_day),
                           tag.ul([genli(t) for t in ttall]),
                           class_='ticketcalendar-popup-list',
                           id=id,
                           data_title=format_date(day, locale=locale,
                                                  format='full')))

            return td

        day_names = get_day_names(locale=locale, width='abbreviated')
        day_names = [day_names[(idx + first_week_day) % 7]
                     for idx in xrange(7)]

        header = tag.div(class_='ticketcalendar-header')
        if nav:
            def nav_href(d):
                return self.get_box_href(query, d)

            def nav_pager(d):
                return tag.span(
                    tag.a(u'\u25c4', href=nav_href(d - timedelta(days=1))),
                    tag.a(_("Current month"), href=nav_href(date.today())),
                    tag.a(u'\u25ba', href=nav_href(d + timedelta(days=31))),
                    class_='ticketcalendar-pager')

            def nav_macro():
                macro = (
                    '[[TicketCalendar(type=box,month=%(month)s,'
                    'query=%(query)s,order=%(order)s%(extra)s)]]' %
                    (dict(month=month.strftime('%Y-%m'),
                          query=self.build_query_string(query.constraints),
                          order=query.order,
                          extra=query.desc and ',desc=1' or '')))
                text = tag.input(type='text', readonly='readonly', size='80',
                                 value=macro, style='width:0;display:none')
                return tag.span(_("Macro"), text,
                                class_='ticketcalendar-macro')

            header(tag.div(nav_macro(), nav_pager(month),
                           class_='ticketcalendar-nav'))

        header(tag.h4(_("%(month_name)s, %(year)s",
                        month_name=_get_month_name(month, locale),
                        year=month.year)))

        calendar = tag.table(
             tag.thead(tag.tr(tag.th(name) for name in day_names)),
             tag.tbody([tag.tr([gentd(idx, d) for d in w])
                               for idx, w in enumerate(cal)]),
             class_='calendar')

        can_create = 'TICKET_CREATE' in req.perm

        ticket_box = tag.div(
            tag.h4(tag.span('', class_='tc-today-date')),
            tag.ul(
                tag.li(tag.a(
                    tag_("New ticket with \"%(date)s\" as the start date",
                         date=tag.span('', data='start-date')),
                    data_href=req.href('newticket',
                                       [(self.mod.start_date_name, '')]),
                    class_='newticket-start-date')),
                tag.li(tag.a(
                    tag_("New ticket with \"%(date)s\" as the due date",
                         date=tag.span('', data='due-date')),
                    data_href=req.href('newticket',
                                       [(self.mod.due_date_name, '')]),
                    class_='newticket-due-date'))),
            title=_("Create new ticket"),
            class_='ticketcalendar-newticket-box',
            style='display:none',
            data_writable=(None, 'writable')[can_create])

        class_ = ('ticketcalendar',
                  'ticketcalendar ticketcalendar-can-create')[can_create]
        return tag.div(header, calendar, ticket_box, class_=class_,
                       style=width and ('width: %s' % width) or None)

    def get_list_events(self, tickets, start_date, period):
        milestones = self._get_milestones()

        req = self.req
        if not start_date or not period:
            start_date = date.today()
            period = 7
        locale = self._get_locale()

        days = [start_date + timedelta(days=i) for i in xrange(period)]
        tickets = self._filter_tickets(days, tickets, req.tz)
        milestones = self._filter_milestones(days, milestones, req.tz)

        events = [{'date': day,
                   'formatted_date': format_date(day, format='full',
                                                 locale=locale),
                   'tickets': tickets[day], 'milestones': milestones[day]}
                  for day in days]

        return events

    def render_list(self, tickets, start_date, period):
        events = self.get_list_events(tickets, start_date, period)
        data = {}
        data['ticketcalendar'] = {
            'events': events,
            'last_week': None,
            'next_week': None,
            'create_ticket': self._create_ticket_item,
            'create_milestone': self._create_milestone_item,
        }
        return Chrome(self.env).render_template(
            self.req, 'ticketcalendar_list.html', data, None, fragment=True)

    def _filter_tickets(self, days, tickets, tzinfo):
        get_start_date = self.mod.get_start_date
        get_due_date = self.mod.get_due_date
        result = dict((day, []) for day in days)

        for ticket in tickets:
            begin = get_start_date(ticket)
            if begin is None:
                begin = _getdate(ticket.get('time'), tzinfo)
            end = get_due_date(ticket)

            if begin == end:
                if begin in result:
                    copied = ticket.copy()
                    copied['_beginend'] = True
                    result[begin].append(copied)
            else:
                if begin in result:
                    copied = ticket.copy()
                    copied['_begin'] = True
                    result[begin].append(copied)
                if end in result:
                    copied = ticket.copy()
                    copied['_end'] = True
                    result[end].append(copied)

        return result

    def _filter_milestones(self, days, milestones, tzinfo):
        rv = dict((day, []) for day in days)
        for milestone in milestones:
            if not milestone.due:
                continue
            due = _getdate(milestone.due, tzinfo)
            if due in rv:
                rv[due].append(milestone)
        return rv

    def _get_locale(self):
        locale = self.req.locale
        if not locale:
            return locale

        if not locale.territory:
            # search first locale which has the same `langauge` and territory
            # in preferred languages
            for l in self.req.languages:
                l = l.replace('-', '_').lower()
                if l.startswith(locale.language.lower() + '_'):
                    try:
                        l = Locale.parse(l)
                        if l.territory:
                            locale = l
                            break
                    except UnknownLocaleError:
                        pass
            if not locale.territory and locale.language in LOCALE_ALIASES:
                locale = Locale.parse(LOCALE_ALIASES[locale.language])

        return locale or Locale('en', 'US')

    def _get_first_week_day(self, locale):
        first = self.mod.first_week_day
        if first not in xrange(0, 7):
            if locale:
                first = locale.first_week_day
            else:
                first = 0  # Monday
        return first

    def _get_month_calendar(self, year, month, first_week_day, locale):
        base = date(year, month, 1)
        base -= timedelta(days=(base.weekday() - first_week_day) % 7)
        days = []
        for week in xrange(6):
            first = base + timedelta(days=week * 7)
            last = first + timedelta(days=6)
            if first.month != month and last.month != month:
                break
            days.append([first + timedelta(days=i) for i in xrange(7)])
        return days

    def _get_milestones(self):
        return Milestone.select(self.env, include_completed=False)

class TicketCalendarModule(Component):

    implements(ITemplateProvider, IRequestHandler, IRequestFilter,
               INavigationContributor, IWikiMacroProvider)

    milestone_background_color = Option(
        'ticketcalendar', 'milestone.background-color', default='#c2c2c2',
        doc=N_("The background color for milestones"))

    milestone_icon = Option(
        'ticketcalendar', 'milestone.icon', default='ui-icon-flag',
        doc=N_("The icon name in jquery-ui for milestones. See http://jquery-"
               "ui.googlecode.com/svn/tags/1.8.21/tests/static/icons.html."))

    priority_colors = ListOption(
        'ticketcalendar', 'ticket.priority.color',
        default='#fca89e, #ffad46, #7bd148, #8db3f0, #cca6ac',
        doc=N_("""\
Comma-separated list of background colors to use for the ticket priorities. \
It can be specified explicitly pairs of priority name and background color. \
e.g. `blocker:#fa6653, critical:#ffad46, ...`."""))

    ticket_type_icons = ListOption(
        'ticketcalendar', 'ticket.type.icon',
        default='ui-icon-contact, ui-icon-lightbulb, ui-icon-check, '
                'ui-icon-gear, ui-icon-comment',
        doc=N_("""\
Comma-separated list of icon names in jquery-ui to use for ticket types.
See http://jquery-ui.googlecode.com/svn/tags/1.8.21/tests/static/icons.html.

It can be specified explicitly pairs of ticket type and icon name.
e.g. `defect:ui-icon-contact, task:ui-icon-lightbulb, ...`."""))

    first_week_day = IntOption(
        'ticketcalendar', 'first_week_day', default=-1,
        doc=N_("The first day of the week in calendar. If -1, use first week "
               "day in user's locale. Otherwise, use the specified number "
               "(0 is Monday as the first day of the week)."))

    start_date_name = Option(
        'ticketcalendar', 'ticket.start_date', default='start_date',
        doc=N_("The field name for start date of ticket"))

    _start_date_format = Option(
        'ticketcalendar', 'ticket.start_date.format', default='%Y/%m/%d',
        doc=N_("The format for start date of ticket"))

    due_date_name = Option(
        'ticketcalendar', 'ticket.due_date', default='due_date',
        doc=N_("The field name for due date of ticket"))

    _due_date_format = Option(
        'ticketcalendar', 'ticket.due_date.format', default='%Y/%m/%d',
        doc=N_("The format for due date of ticket"))

    def __init__(self):
        from pkg_resources import resource_filename, resource_isdir
        if resource_isdir(__name__, 'locale'):
            locale_dir = resource_filename(__name__, 'locale')
            add_domain(self.env.path, locale_dir)

    @property
    def start_date_format(self):
        return self._start_date_format.encode('utf-8')

    @property
    def due_date_format(self):
        return self._due_date_format.encode('utf-8')

    def get_start_date(self, ticket):
        return self._getdate(ticket.get(self.start_date_name),
                             self.start_date_format)

    def get_due_date(self, ticket):
        return self._getdate(
                ticket.get(self.due_date_name)
                , self.due_date_format)

    def _getdate(self, d, format='%d/%m/%Y'):
        if isinstance(d, int):
            return date.fromtimestamp(d).date()
        elif isinstance(d, basestring):
            try:
                tt = time.strptime(d, format)
                return date(*tt[0:3])
            except ValueError:
                return None
        else:
            return None

    # IRequestFilter methods

    def pre_process_request(self, req, handler):
        return handler

    def post_process_request(self, req, template, data, content_type):
        if (template and template.endswith('.html') and
            any(template.startswith(prefix)
                for prefix in ('wiki_', 'roadmap', 'milestone_'))):
            self._add_header_contents(req)
        return template, data, content_type

    # ITemplateProvider methods

    def get_htdocs_dirs(self):
        from pkg_resources import resource_filename
        return [('ticketcalendar', resource_filename(__name__, 'htdocs'))]

    def get_templates_dirs(self):
        from pkg_resources import resource_filename
        return [resource_filename(__name__, 'templates')]

    # INavigationContributor methods

    def get_active_navigation_item(self, req):
        return 'ticketcalendar-box'

    def get_navigation_items(self, req):
        if 'TICKET_VIEW' in req.perm:
            anchor = tag.a(_("Ticket Calendar"),
                           href=req.href('ticketcalendar-box'))
            yield ('mainnav', 'ticketcalendar-box', anchor)

    # IWikiMacroProvider methods

    def get_macros(self):
        return ['TicketCalendar']

    _macro_description = N_("""\
Display tickets and milestones in calendar or list view.

Usage:
{{{
[[TicketCalendar(type=box,query=status!=closed,month=2013-05)]]
[[TicketCalendar(type=list,query=status!=closed,duration=2013-05-16/P7D)]]
}}}

 * The `type` parameter determines how the tickets and milestones are presented:
   - `box`: display in calendar view (default)
   - `list`: display in list view
 * The `query` parameter is used to query the tickets to display.
 * The `month` parameter can be specified as `YYYY-MM` in only calendar view,
   The default is current month.
 * The `duration` parameter can be specified as `YYYY-MM-DD/PnD` in only list
   view. The default is today and 7 days.
""")

    def get_macro_description(self, name):
        from trac.wiki.macros import WikiMacroBase
        if hasattr(WikiMacroBase, '_domain'):
            return TEXTDOMAIN, self._macro_description
        else:
            return gettext(self._macro_description)

    def is_inline(self, content):
        return False

    def expand_macro(self, formatter, name, arguments):
        req = formatter.req
        calendar = TicketCalendar(self.env, req)
        args, kwargs = parse_args(arguments, strict=False)
        query = kwargs.get('query')
        if query:
            query = calendar.get_query_from_string(query)
        else:
            query = calendar.get_query(args=kwargs)
        data = calendar.template_data(req, query, kwargs)
        width = kwargs.get('width')

        if kwargs.get('type') == 'list':
            start, period = _parse_duration_arg(kwargs.get('duration'))
            return calendar.render_list(data['tickets'], start, period)
        else:
            month = _parse_month_arg(kwargs.get('month'))
            return calendar.gen_calendar(data['tickets'], query, month, width,
                                         nav=False)

    # IRequestHandler methods

    def match_request(self, req):
        return req.path_info in ('/ticketcalendar-box',
                                 '/ticketcalendar-list',
                                 '/ticketcalendar-ticket')

    def process_request(self, req):
        req.perm.assert_permission('TICKET_VIEW')
        req.send_header('X-UA-Compatible', 'IE=edge')

        if req.path_info == '/ticketcalendar-ticket':
            return self._process_ticket_detail(req)

        self._add_header_contents(req)

        calendar = TicketCalendar(self.env, req)
        query = calendar.get_query(calendar.get_constraints())
        data = calendar.template_data(req, query)
        data['query_form_filters'] = self._render_query_form_filters(req, data)
        properties = dict((name, dict((key, field[key])
                                      for key in ('type', 'label', 'options',
                                                  'optgroups')
                                      if key in field))
                          for name, field in data['fields'].iteritems())
        add_script_data(req, {'properties': properties,
                              'modes': data['modes']})
        redirect = 'update' in req.args

        if req.path_info == '/ticketcalendar-box':
            month = _parse_month_arg(req.args.get('_month'))
            if redirect:
                req.redirect(calendar.get_box_href(query, month))
            return self._process_box(req, query, data, month)

        if req.path_info == '/ticketcalendar-list':
            start, period = _parse_duration_arg(req.args.getfirst('_duration'))
            if redirect:
                req.redirect(calendar.get_list_href(query, start, period))
            return self._process_list(req, query, data, start, period)

    def _render_query_form_filters(self, req, data):
        if 'query_href' not in req.session:
            req.session['query_href'] = ''  # workaround crashing in batchmod
        fragment = Chrome(self.env).render_template(req, 'query.html', data,
                                                    None, fragment=True)
        return fragment.select('//fieldset[@id="filters"]')

    def _process_box(self, req, query, data, month):
        calendar = TicketCalendar(self.env, req)
        content = calendar.gen_calendar(data['tickets'], query, month)

        data['title'] = _('Ticket Calendar (Calendar view)')
        data['month'] = month.strftime('%Y-%m')
        data['content'] = content
        data['ticketcalendar'] = {'mode': 'calendar'}

        add_ctxtnav(req, _('Calendar view'))
        add_ctxtnav(req, _('List view'), calendar.get_list_href(query))

        return 'ticketcalendar_calendar.html', data, None

    def _process_list(self, req, query, data, start_date, period):
        calendar = TicketCalendar(self.env, req)
        events = calendar.get_list_events(data['tickets'], start_date, period)
        data['ticketcalendar'] = {
            'events': events,
            'last_week': calendar.get_list_href(query,
                                                start_date - timedelta(days=7),
                                                period + 7),
            'next_week': calendar.get_list_href(query, start_date, period + 7),
            'create_ticket': calendar._create_ticket_item,
            'create_milestone': calendar._create_milestone_item,
        }
        data['title'] = _('Ticket Calendar (List view)')
        data['macro'] = \
            '[[TicketCalendar(type=list,duration=%s,query=%s,order=%s%s)]]' % (
            '%s/P%dD' % (start_date.strftime('%Y-%m-%d'), period),
            calendar.build_query_string(query.constraints),
            query.order,
            query.desc and ',desc=1' or '')

        add_ctxtnav(req, _('Calendar view'),
                    calendar.get_box_href(query, date.today()))
        add_ctxtnav(req, _('List view'))

        return 'ticketcalendar_calendar.html', data, None

    def _process_ticket_detail(self, req):
        ticket = Ticket(self.env, int(req.args.get('id')))
        data = {'ticket': ticket,
                'fields': self._prepare_fields(req, ticket),
                'description_change': None,
                'can_append': None,
                'preview_mode': None,
                'writable': 'TICKET_CHGPROP' in req.perm or
                            'TICKET_APPEND' in req.perm or
                            'TICKET_MODIFY' in req.perm,
                }
        return 'ticketcalendar_ticket_detail.html', data, None

    def _add_header_contents(self, req):
        add_stylesheet(req, 'common/css/report.css')
        add_stylesheet(req, 'common/css/ticket.css')
        if hasattr(Chrome, 'add_jquery_ui'):
            Chrome(self.env).add_jquery_ui(req)
        else:
            add_stylesheet(req, 'ticketcalendar/css/jquery-ui.min.css')
        add_stylesheet(req, 'ticketcalendar/css/colorbox.css')
        add_stylesheet(req, 'ticketcalendar/css/ticketcalendar.css')
        add_script(req, 'common/js/folding.js')
        add_script(req, 'common/js/query.js')
        add_script(req, 'ticketcalendar/js/jquery.colorbox-min.js')
        add_script(req, 'ticketcalendar/js/jquery.balloon.min.js')
        add_script(req, 'ticketcalendar/js/ticketcalendar.js')

    def _prepare_fields(self, req, ticket):
        return TicketModule(self.env)._prepare_fields(req, ticket)


from math import log10
from noether.helpers import Real

from ...errors import IncompatibleUnitError
from ..Measure import Measure
from ..Unit import Unit

class LogarithmicUnit(Unit):
    "A logarithmic unit, such as the decibel."
    unit: Real | Measure
    units_per_ten: Real

    def __init__(
        self,
        unit: Real | Measure,
        units_per_ten: Real,
        names: list[str] | str | None = None,
        symbols: list[str] | str | None = None,
        info: str | None = None,
    ):
        object.__setattr__(self, 'unit', unit)
        object.__setattr__(self, 'units_per_ten', units_per_ten)
        # ensure unit == unit
        Unit.__init__(
            self, unit * 10 ** (1/units_per_ten),
            names=names, symbols=symbols, info=info)

    def __call__(self, value: Real | Unit):
        if isinstance(value, Unit):
            # ensure .unit is a Unit if possible
            unit = value if self.unit == 1 else self.unit * value
            return LogarithmicUnit(
                unit, self.units_per_ten,
                f'{self.name}({value.name})', f'{self.symbol}({value.symbol})')
        else:
            val = self.unit * 10 ** (value / self.units_per_ten)
            return val @ self

    def __and__(self, _: Unit):
        raise IncompatibleUnitError(
            'A logarithmic unit {self} cannot be composed with other units!')

    __rand__ = __and__

    def _repr_measure(self, measure: Real | Measure):
        V = Measure._extract_value
        m = V(measure)
        if m <= 0:
            raise ValueError(
                'LogarithmicUnits can only be used on positive units')
        return Unit._repr_measure(
            self, log10(m / V(self.unit)) * self.units_per_ten)

    def _json_extras(self):
        V = Measure._extract_value
        return {'value': V(self.unit), 'units_per_ten': self.units_per_ten}


#import pyperclip
import time
import requests
import subprocess
import json
import requests
import pyperclip
import pyautogui
import calendar
from datetime import date

#Varíavel que armazena o dia atual
todayName = date.today()
#Varíavel que armazena o nome do dia atual
todayNameShow = calendar.day_name[todayName.weekday()]
print(todayNameShow)

#Função de execução do Whatsapp
def openWhatsAppHome():

    # pyautogui.alert("O código vai começar. Não utilize nada do computador até o código finalizar!")
    #Tempo de espera do código
    time.sleep(2)
    #Abre o menu, como se o botão windows tivesse sido clicado
    pyautogui.press('winleft')
    #Tempo de espera do código
    time.sleep(3)
    pyautogui.PAUSE = 1.5
    #Escreve Whatsapp na busca já selecionada
    pyautogui.write('whatsapp')
    #Tempo de espera do código
    time.sleep(3)
    #Pressiona 'enter'
    pyautogui.press('enter')
    #Tempo de espera do código
    time.sleep(3)
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(1180,50)
    #Clique
    pyautogui.click()
    #Tempo de espera do código
    time.sleep(3)
    pyautogui.write('@Dúvidas diversas')
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(150,178)
    #Clique
    pyautogui.click()
    pyautogui.click()
    #Tempo de espera do código
    time.sleep(4)
    #Se o dia da semana estiver entre as posições abaixo, ele abre uma API
    if todayNameShow == 'Tuesday' or todayNameShow == 'Thursday':
        subprocess.call("C:/Albacete-automation/Automation-of-Spreadsheets/BOLETOS/support_files/translationApiValyrian.py", shell=True)

    #Caso contrário abre outro
    if todayNameShow == 'Monday' or todayNameShow == 'Wednesday' or todayNameShow == 'Friday':
        subprocess.call("C:/Albacete-automation/Automation-of-Spreadsheets/BOLETOS/support_files/translationApiDothrarki.py", shell=True)

    #Pressiona 'enter'
    pyautogui.press('enter')
    #Move o cursor do mouse para as coordenadas(x,y)
    pyautogui.moveTo(1255,15)
    #Clique
    pyautogui.click()

    '''

    pyautogui.press('enter')
    pyautogui.moveTo(150,178)
    pyautogui.click()
    time.sleep(3)
    text_with_special_chars = 'Teste 01 - Garoto de programa\n 17/10/2022'
    pyperclip.copy(text_with_special_chars)
    pyautogui.hotkey('ctrl', 'v')
    pyautogui.press('enter')
    time.sleep(5)
    pyautogui.moveTo(1000,15)
    pyautogui.click()

    #pyautogui.mouseDown()
    #pyautogui.moveTo(756,635)
'''
#Chama a função
openWhatsAppHome()


import os
import sys
import copy
import glob
import shutil
import skimage
import argparse
import scipy.ndimage as ndi
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from toolbox import BasicTools as bt
from toolbox import LIDCTools as lt
from toolbox import Evaluations as eva
from toolbox import CTViewer_Multiax as cvm

def mathcurves_paint():
        x = np.arange(-10.,10.,0.01)
        y = 1 / (1 + np.exp(-x))
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("sigmoid.png", format='png')
        plt.close()

        y = np.tanh(x)
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("tanh.png", format='png')
        plt.close()

        y = copy.copy(x)
        y[y<0] = 0
        plt.plot(x, y)
        plt.grid(True)
        plt.xlabel("x")
        plt.ylabel("y")
        plt.savefig("ReLU.png", format='png')
        plt.close()

def samples_replace(source_path="/data/fyl/data_samples/tianchild_cubes_overbound/npy_non_fine", target_path="/data/fyl/data_samples/tianchild_cubes_overbound/npy_non"):
	'''
	filelist = glob.glob(source_path+'/*.npy')
	for file in filelist:
		filename = os.path.basename(file)
		target_file = target_path + '/' + filename
		shutil.copyfile(file, target_file)
		print(target_file)
	'''
	bt.directory_arrange(source_path, target_path, mode='move')

def keras_generator_test():
	from keras.preprocessing.image import ImageDataGenerator
	image = skimage.io.imread("F:/lung_project2/lidc_cubes_64_overbound_roughspacing/LIDC-IDRI-1012_1_2.0_[2.       0.722656 0.722656]_oa_s3.png")
	#image = np.random.rand(50, 50, 3)
	plt.imshow(image)
	plt.savefig('temp.png')
	generator = ImageDataGenerator(shear_range=1., data_format='channels_last', fill_mode='constant')
	image_transformed = generator.random_transform(image)
	plt.imshow(image_transformed)
	plt.show()

def list_compare():
	path1 = "/data/fyl/data_samples/sph_overbound/slicewise"
	path2 = "/data/fyl/data_samples/tianchild_cubes_overbound2/npy"
	samples1 = os.listdir(path1)
	samples2 = os.listdir(path2)
	patients1 = []
	patients2 = []
	for sample in samples1:
		info = '_'.join(os.path.basename(sample).split('_')[:2])
		#info = os.path.basename(sample).split('_')[0]
		if info not in patients1:
			patients1.append(info)
	for sample in samples2:
		#info = '_'.join(os.path.basename(sample).split('_')[:2])
		info = os.path.basename(sample).split('_')[0]
		if info not in patients2:
			patients2.append(info)
	'''
	samples1 = os.listdir("/data/fyl/data_samples/sph2015_overbound/npy")
	samples1.extend(os.listdir("/data/fyl/data_samples/sph2015_overbound/npy_non"))
	statsdict1 = {}
	for sample in samples1:
		patient = sample.split('_')[0]
		if patient in statsdict1.keys():
			statsdict1[patient] += 1
		else:
			statsdict1[patient] = 1
	stats1 = []
	for statitem in statsdict1.items():
		stats1.append(statitem[0]+':'+str(statitem[1]))
	stats2 = bt.filelist_load("/home/fyl/programs/lung_project/data_samples/sph2015_overbound/nodstats.log")[:-1]
	'''
	set1 = set(patients1)
	set2 = set(patients2)
	set_diff1 = set2.difference(set1)
	set_diff2 = set1.difference(set2)
	print(set_diff1)
	print(set_diff2)

def detection_evaluation():
	annotation_file = "/data/fyl/datasets/LUNA16/csvfiles/annotations_corrected.csv"
	#result_file = "pytorch_project/experiments_dt/evaluations_test2/result.csv"
	evaluation_path = 'pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4'
	#eva.csv_evaluation(annotation_file, result_file, evaluation_path, exclude_file=None)
	#results = np.load('pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4/result.npy')
	results_csv = pd.read_csv('pytorch_project/experiments_dt/evaluations_sliccand_densecropnet_e11_fold4/result.csv')
	results = []
	for r in range(len(results_csv["seriesuid"].values)):
		uid = results_csv["seriesuid"].values[r]
		coordX = results_csv["coordX"].values[r]
		coordY = results_csv["coordY"].values[r]
		coordZ = results_csv["coordZ"].values[r]
		prob = results_csv["probability"].values[r]
		results.append([uid, coordX, coordY, coordZ, prob])
	#result_order = results[:,-1].argsort()[::-1]
	#for oi in result_order:
	#	print(results[oi])
	assessment = eva.detection_assessment(results, annotation_file, label=1)
	eva.evaluation_vision([], assessment['num_scans'], assessment['FROC'][0], assessment['FROC'][1], assessment['CPM'], assessment['detection_cites'], output_path=evaluation_path)

def filename_arrange(source_path="sph_cubes_64_overbound/npy", target_path="lung adenocarcinoma samples of the private dataset"):
	filenames = os.listdir(source_path)
	ids = []
	for filename in filenames:
		fnsplit = os.path.splitext(filename)[0].split('_')
		if len(fnsplit)>1:
			if (fnsplit[0], fnsplit[1]) not in ids: ids.append((fnsplit[0], fnsplit[1]))
			idx = ids.index((fnsplit[0], fnsplit[1])) + 1
			label = fnsplit[2]
			#pidx = fnsplit[-1][1:]
			#newname = "sample%d_%s_patch%s.png" %(idx, label, pidx)
			newname = "sample%d_%s.npy" %(idx, label)
			newpath = target_path+'/'+label
			if not os.path.exists(newpath):
				os.makedirs(newpath)
			shutil.copy(source_path+'/'+filename, newpath+'/'+newname)

def featuremap_visualize(files, names = [], num_channels = 20):
	fig = plt.figure()
	#fig = plt.figure(figsize=(12, 3.5))
	#plt.title('channels')
	#fig, ax = plt.subplots(len(files), 1)
	fms = []
	axs = []
	for f in range(len(files)):
		if isinstance(files[f], str):
			featuremap = np.load(files[f])
			ax = fig.add_subplot(len(files), 1, f+1)
			fm = ax.imshow(featuremap[0], vmin=0, vmax=1)
			ax.set_xticks(np.append(np.arange(0, featuremap.shape[2], featuremap.shape[2]//num_channels), featuremap.shape[2]-1))
			ax.set_xticklabels(np.arange(0, num_channels+1))
			ax.set_yticks([])
			#ax.set_yticks([0, featuremap.shape[1]-1])
			#ax.set_yticklabels([0, 1])
			#ax.set_title('channels')
			if f==0:
				ax.set_xlabel('channels')
				ax.xaxis.set_label_position('top')
			ax.set_ylabel('layer %d' %(f + 1))
			axs.append(ax)
			fms.append(fm)
		else:
			aaxs = []
			file_nums = [len(file_approach) for file_approach in files]
			for l in range(len(files[f])):
				featuremap = np.load(files[f][l])
				ax = fig.add_subplot(sum(file_nums), 1, sum(file_nums[:f])+l+1)
				fm = ax.imshow(featuremap[0], vmin=0, vmax=1)
				ax.set_xticks(np.append(np.arange(0, featuremap.shape[2], featuremap.shape[2]//num_channels), featuremap.shape[2]-1))
				ax.set_xticklabels(np.arange(0, num_channels+1))
				ax.set_yticks([])
				ax.set_ylabel('layer %d'%(l*3+1), rotation=0)
				aaxs.append(ax)
				fms.append(fm)
			aaxs[-1].set_xlabel(names[f])
			#aaxs[0].set_title('channels')
			axs.extend(aaxs)
	#plt.subplots_adjust(left=-1, hspace=5)
	fig.colorbar(fms[0], ax=axs)
	plt.show()
	plt.close()

featuremap_visualize(files=[['fmn1l1.npy', 'fmn1l4.npy'], ['fmtr1l1.npy', 'fmtr1l4.npy'], ['fmt1l1.npy', 'fmt1l4.npy'], ['fms1l1.npy', 'fms1l4.npy']], names=['DNC learning simply on LIDC-IDRI', 'transfer learning', 'multitask learning', 'bias-undoing learning'])
'''
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('integers', metavar='N', type=int, nargs='+',
                    help='an integer for the accumulator')
parser.add_argument('--sum', dest='accumulate', action='store_const',
		    const=sum, default=max, help='sum the integers (default: find the max)')
args = parser.parse_args()
print(args.accumulate(args.integers))
'''
#detection_evaluation()


"""Remove unused tables

Revision ID: 46597280a7c
Revises: 25f9a557214
Create Date: 2015-07-03 17:04:54.413092

"""

# revision identifiers, used by Alembic.
revision = '46597280a7c'
down_revision = '25f9a557214'
branch_labels = None
depends_on = None

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade(engine_name):
    globals()["upgrade_%s" % engine_name]()

def downgrade(engine_name):
    globals()["downgrade_%s" % engine_name]()

def upgrade_account():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('sales_order')
    op.drop_table('billing_rate')
    op.drop_constraint('service_usage_ibfk_1', 'service_usage', type_='foreignkey')
    ### end Alembic commands ###

def downgrade_account():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_foreign_key('service_usage_ibfk_1', 'service_usage', 'tenant', ['tenant_id'], ['tenant_id'])
    op.create_table('billing_rate',
    sa.Column('id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('name', mysql.VARCHAR(length=64), nullable=True),
    sa.Column('unit', mysql.VARCHAR(length=32), nullable=True),
    sa.Column('rate', mysql.FLOAT(), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('sales_order',
    sa.Column('sale_order_id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('tenant_id', mysql.VARCHAR(length=100), nullable=False),
    sa.Column('start', mysql.DATETIME(), nullable=False),
    sa.Column('end', mysql.DATETIME(), nullable=False),
    sa.ForeignKeyConstraint(['tenant_id'], ['tenant.tenant_id'], name='sales_order_ibfk_1'),
    sa.PrimaryKeyConstraint('sale_order_id', 'tenant_id', 'start', 'end'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    ### end Alembic commands ###

def upgrade_fitter():
    pass

def downgrade_fitter():
    pass


from .setup import *

class ModuleFP(PluginModuleBase):

    def __init__(self, P):
        super(ModuleFP, self).__init__(P, name='fp', first_menu='setting')
        self.db_default = {
            f'{self.name}_db_version' : '3',
            f'{self.name}_use_plexscan' : 'False',
            f'{self.name}_ignore_rule' : '',
            f'{self.name}_change_rule' : '',
            f'fp_item_last_list_option' : '',
        }
        self.web_list_model = ModelFPItem

    def process_discord_data(self, data):
        #P.logger.warning(d(data))
        try:
            db_item = ModelFPItem.process_discord_data(data)

            if P.ModelSetting.get_bool(f'{self.name}_use_plexscan') == False:
                return
            ignore_list = P.ModelSetting.get_list(f'{self.name}_ignore_rule')
            for ignore in ignore_list:
                if ignore in db_item.gds_path:
                    db_item.status = 'IGNORE'
                    db_item.save()
                    return
            change_list = P.ModelSetting.get_list(f'{self.name}_change_rule')
            ret = db_item.gds_path
            if ret == None: return
            for rule in change_list:
                tmp = rule.split('|')
                ret = ret.replace(tmp[0].strip(), tmp[1].strip())
            if ret[0] != '/':
                ret = ret.replace('/', '\\')

            db_item.local_path = ret

            PP = F.PluginManager.get_plugin_instance('plex_mate')
            PP.ModelScanItem(ret, mode="ADD", callback_id=f"gds_tool_{db_item.id}", callback_url=ToolUtil.make_apikey_url("/gds_tool/normal/fp/scan_completed")).save()
            db_item.status = "SCAN_REQUEST"
            db_item.save()
        except Exception as e:
            P.logger.error(f"Exception:{str(e)}")
            P.logger.error(traceback.format_exc())

    def process_normal(self, sub, req):
        #P.logger.error(sub)
        #P.logger.error(d(req.form))
        try:
            if sub == 'scan_completed':
                callback_id = req.form['callback_id']
                db_item = ModelFPItem.get_by_id(callback_id.split('_')[-1])
                db_item.status = req.form['status']
                db_item.save()
                return jsonify({'ret':'success'})
        except Exception as e:
            P.logger.error(f"Exception:{str(e)}")
            P.logger.error(traceback.format_exc())

class ModelFPItem(ModelBase):
    P = P
    __tablename__ = 'fp_item'
    __table_args__ = {'mysql_collate': 'utf8_general_ci'}
    __bind_key__ = P.package_name

    id = db.Column(db.Integer, primary_key=True)
    created_time = db.Column(db.DateTime)

    mode = db.Column(db.String)
    data = db.Column(db.JSON)
    status = db.Column(db.String)
    gds_path = db.Column(db.String)
    local_path = db.Column(db.String)

    def __init__(self):
        self.created_time = datetime.now()
        self.status = 'READY'

    @classmethod
    def make_query(cls, req, order='desc', search='', option1='all', option2='all'):
        with F.app.app_context():
            query = db.session.query(cls)
            query = cls.make_query_search(F.db.session.query(cls), search, cls.gds_path)

            if option1 != 'all':
                query = query.filter(cls.mode == option1)
            if option2 != 'all':
                query = query.filter(cls.status.like(option2 + '%'))
            query = query.order_by(desc(cls.id)) if order == 'desc' else query.order_by(cls.id)
            return query

    @classmethod
    def process_discord_data(cls, data):
        try:
            P.logger.debug(data)
            db_item = ModelFPItem()
            if data['ch'] == 'bot_gds_vod':
                db_item.mode = 'VOD'
                db_item.data = data
                db_item.gds_path = data['msg']['data']['r_fold'] + '/' + data['msg']['data']['r_file']
            elif data['ch'] == 'bot_gds_movie':
                db_item.mode = 'MOVIE'
                db_item.data = data
                db_item.gds_path = data['msg']['data']['gds_path']
            db_item.save()
            return db_item
        except Exception as e:
            P.logger.error(f"Exception:{str(e)}")
            P.logger.error(traceback.format_exc())


import torch as torch
import copy
import os
import yaml
import numpy as np
from tqdm.autonotebook import tqdm
from .layers.coupling_cells import  AffineCoupling, PWLin, PWQuad
from .layers.layers import *
from .misc import tqdm_recycled
from statistics import mean
import time
import gc
import datetime
import math

TF_CPP_MIN_LOG_LEVEL="2"

def get_bin(x, n=0):
    """
    Get the binary representation of x.

    Parameters
    ----------
    x : int
    n : int
        Minimum number of digits. If x needs less digits in binary, the rest
        is filled with zeros.

    Returns
    -------
    list of binary digits
    """
    y=format(x, 'b').zfill(n)
    return [int(i) for i in str(y)]

def normal(x,mu, sigma, n_flow):
    return (torch.exp(-torch.sum((x-mu)**2/(2*sigma**2),-1)))/(sigma*np.sqrt((2*np.pi)**n_flow))

class ModelAPI():

    @property
    def model(self):
        if self._model is not None:
            return self._model
        else:
            raise AttributeError("No model was instantiated")

class BasicManager(ModelAPI):
    """Basic training methods"""

    def __init__(self,
                  n_flow=2,*args):

        self.n_flow = n_flow
        self._model = None
        self._inverse_model = None

        self.optimizer_object = None

    def _train_variance_forward_seq(self, f, optimizer_object, log=True, logdir=None, batch_size = 10000, epochs=10,
                                    epoch_start=0,
                               pretty_progressbar=True,
                                    save_best=True, run=None,dev=0,mini_batch_size=2000, integrate=False,
                                    preburn_time=75, kill_counter=7, impr_ratio=1e-2, loss_mode="var"):
        """Train the model using the integrand variance as loss and compute the Jacobian in the forward pass
        (fixed latent space sample mapped to a phase space sample)

        """

        dev = torch.device("cuda:"+str(dev)) if torch.cuda.is_available() else torch.device("cpu")

        if preburn_time>10:
            check_time=preburn_time
        else:
            check_time=50

        if(mini_batch_size>batch_size):
            mini_batch_size=batch_size

        n_minibatches = int(batch_size/mini_batch_size)
        batch_size=batch_size-(batch_size%mini_batch_size)

        if(run!=None and log):
            filename=logdir+"/"+str(run._id)+"/torch"
            filename2=filename+"_int"
            if not os.path.exists(logdir+"/"+str(run._id)):
                os.mkdir(logdir+"/"+str(run._id))

        else:
            filename=logdir+"/torch"
            filename2=filename+"_int"
            if not os.path.exists(logdir):
                os.makedirs(logdir)

        try:
            if(log):
                file= open(filename,"w+")
                file2=open(filename2,"w+")
                file2.close()
                file.close()
                torch.save({
                    'model_state_dict': self.best_model.state_dict()
                    },filename2)

        except:
            print("Torch save not possible")

        integ=torch.zeros((epochs+1,),device=dev)
        err=torch.zeros((epochs+1,),device=dev)

        # Instantiate a pretty progress bar if needed
        if pretty_progressbar:
            epoch_progress = tqdm(range(epoch_start,epoch_start+epochs), leave=False, desc="Loss: {0:.3e} | Epoch".format(0.))
            # Instantiate a pretty progress bar for the minibatch loop if it is not trivial
            if n_minibatches>1:
                minibatch_progress = tqdm_recycled(range(n_minibatches), leave=False, desc="Step")
            else:
                minibatch_progress = range(n_minibatches)

        else:
            epoch_progress = range(epoch_start, epoch_start+epochs)
            minibatch_progress = range(n_minibatches)

        i=0
        self.model.to(dev)
        self.best_loss=0

        self.best_var=0
        maxf=0

        while (i<self.n_flow):
            w = torch.empty(2*mini_batch_size, self.n_flow).to(torch.double)

            torch.nn.init.uniform_(w).to(dev)  # Generate a batch of points in latent space

            dkl=torch.nn.KLDivLoss(reduction='batchmean')
            fres=f(w)
            integ[0]+=torch.sum(fres)/(self.n_flow*2*mini_batch_size)
            err[0]+=torch.var(fres)/(self.n_flow)
            if torch.max(fres)>maxf:
                maxf=torch.max(fres)

            if(loss_mode=="var"):
                self.best_loss+=torch.var(fres/maxf).detach()/self.n_flow
            elif(loss_mode=="est" ):
                self.best_loss+=torch.mean((fres)**2).detach()/self.n_flow
            else:
                print("Unknown loss function")
                return

            i=i+1

            self.best_var+=float((torch.var((fres/maxf)**2)/2*mini_batch_size).detach())

        if save_best or log:

            XJ = self.model(  # Pass through the model
                self.format_input(  # Append a unit Jacobian to each point
                    w,dev
                )
              )
            X=XJ[:,:-1]
            J = XJ[:,-1]
            self.varJ=torch.mean(J**2).detach()

            self.DKL=dkl(torch.log(X+torch.ones_like(X).fill_(1e-45)).to(dev),w.to(dev)).detach()

            self.best_model=copy.deepcopy(self.model)

            self.best_epoch=0
            self.best_time=0
            self.best_loss_rel=torch.ones_like(self.best_loss)
            self.best_func_count=2*mini_batch_size*self.n_flow/(batch_size/mini_batch_size)
            self.best_func_count=2*batch_size*self.n_flow
            self.history=[]

            del XJ,X,J
        if(run!=None and log):
            run.log_scalar("training.int_loss", self.best_loss.tolist(), 0)

        self.int_loss=self.best_loss

        stale_save=1000

        preburner=(preburn_time>0)

        counter=0
        last_loss=1000

        for i in epoch_progress:

            loss = 0

            var = 0
            optimizer_object.zero_grad()

            for j in minibatch_progress:

                w = torch.empty(mini_batch_size, self.n_flow).to(dev).to(torch.double)
                torch.nn.init.uniform_(w)

                XJ = self.model(                                            # Pass through the model
                    self.format_input(                                      # Append a unit Jacobian to each point
                        w,dev
                    )
                )

                # Separate the points and their Jacobians:
                # This sample is fixed, we optimize the Jacobian
                X = (XJ[:, :-1]).detach()

                if(preburner):
                    fres=f(w)
                    fXJ=torch.mul(fres, XJ[:, -1])/maxf

                    integ[i+1]+=torch.mean(fres)/n_minibatches
                    err[i+1]+=torch.var(fres)/n_minibatches

                else:
                    fres=torch.mul(f(X), XJ[:, -1])
                    fXJ = fres/maxf

                    integ[i+1]+=torch.mean(fres.detach())/n_minibatches
                    err[i+1]+=torch.var(fres.detach())/n_minibatches

                if(loss_mode=="var"):
                    loss+=torch.var(fXJ)

                elif(loss_mode=="est"):
                    loss+=torch.mean((fXJ*maxf)**2)

                # The Monte Carlo integrand is fXJ: we minimize its variance up to the constant term
                #in newer versions, the variance estimator is used in order to increase
                #stability for flat distributions

                var+=float(torch.var(fXJ**2)/mini_batch_size) # variance of the mean is variance/N
                del X, fXJ, XJ
                gc.collect()

            loss=loss/n_minibatches

            loss.backward()

            optimizer_object.step()

            self.history.append(loss.detach())
            # Update the progress bar
            if pretty_progressbar:
                epoch_progress.set_description("Loss: {0:.3e} | Epoch".format(loss))
            if(run!=None and log):
                run.log_scalar("training.loss", loss.tolist(), i)
                run.log_scalar("training.loss_rel",(loss/self.int_loss).tolist(),i)

            if (save_best or log):
                self.best_func_count=self.best_func_count+batch_size
            if (save_best or log) and (loss < self.best_loss) and not preburner:
                self.best_loss = loss
                self.best_var=var
                self.best_loss_rel=loss/self.int_loss
                self.best_model=copy.deepcopy(self.model)
                self.best_epoch=i

                if(run!=None):
                    self.best_time=(datetime.datetime.utcnow()-run.start_time).total_seconds()
                else:
                    self.best_time=0

            if loss<last_loss:
                counter=0
            else:
                counter+=1
                if counter>kill_counter and preburner:
                    counter=0
                    preburner=False
                elif counter>kill_counter:
                    break
            last_loss=loss
            if ( i%check_time==0) and i>(preburn_time+1) and float(self.best_loss/stale_save)>(1-impr_ratio) and not preburner:
                break
            elif  i%check_time==0 and not preburner and (self.best_loss<self.int_loss or i>300):

                stale_save=self.best_loss

            if preburner and ((loss<0.25*self.best_loss) or i>preburn_time):

                preburner=False

        endpoint=i+1
        with torch.no_grad():
            if(integrate and endpoint<epochs-1):
                model=self.best_model.eval()
                for s in range(endpoint,epochs):
                    for t in range(n_minibatches):
                        w = torch.empty(mini_batch_size, self.n_flow).to(dev).to(torch.double)
                        torch.nn.init.uniform_(w)

                        XJ = model(self.format_input(w,dev)).detach()
                        fres=torch.mul(f(XJ[:,:-1]), XJ[:, -1])
                        integ[s+1]+=torch.mean(fres)/(n_minibatches*math.sqrt(mini_batch_size))
                        err[s+1]+=torch.std(fres)/n_minibatches

                    self.best_func_count=self.best_func_count+batch_size

        self.integ_tot=torch.sum(integ/err)/torch.sum(1/err)
        self.err_tot=torch.sqrt(1/torch.sum(1/err))

        if(run!=None and integrate):
            run.log_scalar("training.integ", self.integ_tot.tolist(), 0)
            run.log_scalar("training.err", self.err_tot.tolist(), 0)

        try:
            if (log):
                torch.save({
                    'best_epoch': self.best_epoch,
                    'best_loss': self.best_loss,
                    'int_loss': self.int_loss,
                    'best_loss_rel' : self.best_loss_rel,
                    'best_func_count' : self.best_func_count,
                    'model_state_dict': self.best_model.state_dict(),
                    'integ' : self.integ_tot,
                    'err' : self.err_tot
                    },filename)
        except:
            print("Torch save not possible")

        del loss, var, w, optimizer_object

        if(integrate):
            return (self.integ_tot.detach().tolist(),self.err_tot.detach().tolist())
        else:
            return (0,0)

    def integrate(self, f, nitn, neval, dev=None):
        if(self.best_model==None):
            print("No model has been trained")
            return (0,0)
        if(dev==None or not torch.cuda.is_available()):
            dev=torch.device("cpu")
        else:
            dev=dev = torch.device("cuda:"+str(dev))
        neval=int(neval)
        nitn=int(nitn)
        w = torch.empty(neval, self.n_flow).to(dev)
        var=torch.zeros((nitn,))
        mean=torch.zeros((nitn,))
        for i in range(nitn):

            torch.nn.init.uniform_(w)
            Y=self.format_input(w,dev)
            X=self.best_model(Y).detach()
            fres=f(X[:,:-1])*X[:,-1]
            var[i]=torch.var(fres)
            mean[i]=torch.mean(fres)

        sig=torch.sum(mean/var)/torch.sum(1/var)
        sig_err=torch.sqrt(1/torch.sum(1/var))/np.sqrt(neval*nitn)

        return (sig, sig_err)

class AffineManager(BasicManager):

    """A manager for normalizing flows with affine coupling cells interleaved with rolling layers that
    apply cyclic permutations on the variables. All cells have the same number of pass through variables and the
    same step size in the cyclic permutation.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_pass_through
    - n_bins
    - nn_width
    - nn_depth
    - roll_step
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_pass_through,
                     n_cells,
                     NN,
                     roll_step
                     ):

        dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()

        for i_cell in range(n_cells):#create coupling cells
            self._model.add_module(str(i_cell),
                AffineCoupling(flow_size=self.n_flow, pass_through_size=n_pass_through,
                                                NN_layers=NN)
            )
            self._model.add_module("roll",RollLayer(roll_step)) #add roll layer

        w = torch.empty(10, self.n_flow).to(dev)
        torch.nn.init.uniform_(w)
        # Do one pass forward:
        self._model(self.format_input(w, dev))

class PWLinManager(BasicManager):

    """A manager for normalizing flows with piecewise-linear coupling cells interleaved with rolling layers that
    apply cyclic permutations on the variables. All cells have the same number of pass through variables and the
    same step size in the cyclic permutation.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_pass_through
    - n_bins
    - nn_width
    - nn_depth
    - roll_step
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_pass_through,
                     n_cells,
                     n_bins,
                     NN,
                     roll_step
                     ):

        dev = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()

        for i_cell in range(n_cells):#create coupling cells
            self._model.add_module(str(i_cell),
                PWLin(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                NN_layers=NN)
            )
            self._model.add_module("roll",RollLayer(roll_step)) #add roll layer
        self._model.to(dev)
        self.best_model=self.model

        w = torch.empty(5, self.n_flow).to(torch.double)
        torch.nn.init.uniform_(w).to(dev)
        # Do one pass forward:
        self._model(self.format_input(w,dev))

class PWQuadManager(BasicManager):

    """A manager for normalizing flows with piecewise-quadratic coupling cells interleaved with masking layers that
    mask subsets of the varialbes. The cells have different numbers of pass through variables.
    Each coupling cell has a fully connected NN with a fixed number of layers (depth) of fixed size (width)

    Hyperparameters:
    - n_bins
    - nn_width
    - nn_depth
    - n_cells
    """

    format_input = AddJacobian()

    def create_model(self,
                     n_cells,
                     n_bins,
                     NN,
                     dev=0
                     ):

        if(n_cells<2*np.ceil(np.log2(self.n_flow)) and n_cells<self.n_flow):

            if(self.n_flow<=6):
                n_cells=self.n_flow
            elif self.n_flow==7:
                n_cells=6
            else:
                n_cells=int(2*np.ceil(np.log2(self.n_flow)))
            print("Adjusted # coupling cells to "+str(n_cells))
        dev = torch.device("cuda:"+str(dev)) if torch.cuda.is_available() else torch.device("cpu")

        self._model = torch.nn.Sequential()
        if(self.n_flow<=7):
            if(self.n_flow<=6):
                roll_step=1
                n_pass_through=1
            elif (self.n_flow==7):
                roll_step=1
                n_pass_through=2

            for i_cell in range(n_cells):#create coupling cells
                self._model.add_module(str(i_cell),
                    PWQuad(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )
                if(i_cell<n_cells-1):
                    self._model.add_module("roll"+str(i_cell),RollLayer(roll_step)) #add roll layer
                else:
                     self._model.add_module("roll"+str(i_cell),RollLayer(self.n_flow-((n_cells-1)%self.n_flow)))

        else:
            roll_step=1
            n_pass_through=int(self.n_flow/2)
            dims=torch.arange(self.n_flow)
            n=len(get_bin(self.n_flow-1,0))
            dims_bin=torch.ones(self.n_flow,n)

            dims_bin=torch.IntTensor(list(map(get_bin, dims,[n]*self.n_flow))).to(dev)

            for i_cells in range(2*n):#create coupling cells
                masker=MaskLayer(dims_bin,i_cells,dev)
                self._model.add_module("mask"+str(i_cells),masker)
                self._model.add_module(str(i_cells),
                    PWQuad(flow_size=self.n_flow, pass_through_size=masker.pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )

                self._model.add_module("demask"+str(i_cells),DeMaskLayer(masker.feeder,masker.trafoer))
            for i_cells in range(n_cells-2*n):
                self._model.add_module(str(i_cells+2*n),
                    PWQuad(flow_size=self.n_flow, pass_through_size=n_pass_through,n_bins=n_bins,
                                                    NN_layers=NN)
                )
                if(i_cells<n_cells-2*n-1):
                    self._model.add_module("roll"+str(i_cells+2*n),RollLayer(roll_step)) #add roll layer
                else:
                    self._model.add_module("roll"+str(i_cells+2*n),RollLayer(self.n_flow-((n_cells-2*n-1)%self.n_flow)))

        self._model.to(dev).to(torch.double)
        self.best_model=self.model

        w = torch.empty(5, self.n_flow).to(torch.double)
        torch.nn.init.uniform_(w)

        # Do one pass forward:

        self._model(self.format_input(w,dev))

        return


#encoding=utf-8
import logging
import os
from datetime import datetime
class RecordLog(object):
    def __init__(self,logger):
        self.logger = logging.getLogger(logger)
        self.logger.setLevel(logging.DEBUG)

        # 2.将log信息输出到log文件中
        # 2.1 先定位看将log文件输出到哪里去
        current_dir = os.path.dirname(os.path.abspath(__file__))
        print(current_dir)  # D:\MySpace\Python\WebTesting\util
        log_dir = os.path.join('../Result')
        # 日志名称构建
        log_file_name = datetime.now().strftime("%Y-%m-%d") + '.log'
        log_file_path = log_dir + '/' + log_file_name
        print(log_file_path)

        # 2.2 好的，将日志写进log文件中
        self.file_handle = logging.FileHandler(log_file_path, 'a', encoding='utf-8')
        self.file_handle.setLevel(logging.INFO)
        formatter = logging.Formatter(
            '%(asctime)s %(name)s %(levelname)s ---> %(message)s')
        self.file_handle.setFormatter(formatter)
        self.logger.addHandler(self.file_handle)

    def get_log(self):
        return self.logger

if __name__ == "__main__":
    rl = RecordLog()
    log_info = rl.get_log()
    log_info.debug('输出到文件中去')
    rl.close_handle()


#知识储备__call__方法
# class Foo:
#     def __call__(self, *args, **kwargs):
#         print(self)
#         print(args)
#         print(kwargs)
#
#
# obj=Foo()
#
# obj(1,2,3,a=1,b=2,c=3) #obj.__call__(obj,1,2,3,a=1,b=2,c=3)
#
#
# #元类内部也应有有一个__call__方法，会在调用Foo时触发执行
# #Foo(1,2,x=1)  #Foo.__call__(Foo,1,2,x=1)

class Mymeta(type):
    def __init__(self,class_name,class_bases,class_dic):
        if not class_name.istitle():
            raise TypeError('类名的首字母必须大写')

        if '__doc__' not in class_dic or not class_dic['__doc__'].strip():
            raise TypeError('必须有注释，且注释不能为空')

        super(Mymeta,self).__init__(class_name,class_bases,class_dic)

    def __call__(self, *args, **kwargs): #obj=Chinese('egon',age=18)
        # print(self) #self=Chinese
        # print(args) #args=('egon',)
        # print(kwargs) #kwargs={'age': 18}

        #第一件事：先造一个空对象obj
        obj=object.__new__(self)
        #第二件事：初始化obj
        self.__init__(obj,*args,**kwargs)
        #第三件事：返回obj
        return obj

class Chinese(object,metaclass=Mymeta):
    '''
    中文人的类
    '''
    country='China'

    def __init__(self,namem,age):
        self.name=namem
        self.age=age

    def talk(self):
        print('%s is talking' %self.name)

obj=Chinese('egon',age=18) #Chinese.__call__(Chinese,'egon',18)

print(obj.__dict__)

3面向对象编程/面向对象&网络编程基础/网络编程入门&FTP服务开发实战/第一章面向对象/27 自定义元类控制类的实例化行为.py

27 自定义元类控制类的实例化行为.py


from base import BaseTestCase
from gaedocstore import GDSDocument, GDSJson
from google.appengine.ext import ndb
import json
import gaedocstore
class testDictGDSDocumentConversions(BaseTestCase):

    def test0(self):
        x = ndb.Expando() #GDSDocument()
#        x.fred = [GDSDocument(), 5]
        x.fred = ndb.Expando()
        x.fred.george = ndb.Expando()
        x.fred.george.ringo = "wut"

        x.put()

        y = x.key.get()

        self.assertEqual(y.fred.george.ringo, "wut")

    def test1(self):
        x = GDSDocument()
#        x.fred = [GDSDocument(), 5]
        x.fred = GDSDocument()
        x.fred.george = GDSDocument()
        x.fred.george.ringo = "wut"

        x.put()

        gp = ndb.GenericProperty()
        gp._name = "fred.george.ringo"
        y = GDSDocument.query(gp == "wut").get()

        self.assertIsNotNone(y)
        self.assertEqual(y.fred.george.ringo, "wut")

    def test2(self):
        x = GDSDocument()
#        x.fred = [GDSDocument(), 5]
        x.fred = GDSDocument()
        x.fred.george = GDSDocument()
        x.fred.george.ringo = "wut"
        x.fred.george.ttt = GDSJson()
        x.fred.george.ttt.json = [3, 4, {'x':'thingo'}]
#        x.fred.george.ttt = ndb.JsonProperty()
#        x.fred.george.ttt = {"item":[3, 4, {'x':'thingo'}]}

        x.put()

        gp = ndb.GenericProperty()
        gp._name = "fred.george.ringo"
        y = GDSDocument.query(gp == "wut").get()

        self.assertIsNotNone(y)
        self.assertEqual(y.fred.george.ringo, "wut")

        z = y.to_dict()
#        s = FixToDict(z)
        json.dumps(z, indent=True)
        #logging.warning(s)

    def convertToGDSAndBack(self, aDict):
        lgdsDocument = gaedocstore.DictToGDSDocument(aDict)

        self.assertIsNotNone(lgdsDocument, "lgdsDocument should not be None")

        ldict2 = lgdsDocument.to_dict()

        self.assertIsNotNone(ldict2, "ldict2 should not be None")

        self.assertDictEqual(aDict, ldict2)

    def convertToGDSAndQuery(self, aDict, aQueryField, aQueryValue):
        lgdsDocument = gaedocstore.DictToGDSDocument(aDict)
        lgdsDocument.put()

        self.assertIsNotNone(lgdsDocument, "lgdsDocument should not be None")

        lgp = ndb.GenericProperty()
        lgp._name = aQueryField
        lgdsDocument = GDSDocument.query(lgp == aQueryValue).get()

        self.assertIsNotNone(lgdsDocument)

        ldict2 = lgdsDocument.to_dict()

        self.assertIsNotNone(ldict2, "ldict2 should not be None")

        self.assertDictEqual(aDict, ldict2)

    def test3(self):
        ldict = {"fred":"george", "bill": True}

        self.convertToGDSAndBack(ldict)

    def test4(self):
        ldict = {"fred":"george", "bill": True, "harry": [1, 2, 3]}

        self.convertToGDSAndBack(ldict)

    def test5(self):
        ldict = {"fred":"george", "larry":{"gertrude": 47}}

        self.convertToGDSAndBack(ldict)

    def test6(self):
        # need a key because we're actually saving the object here, it'll come back to us with an unexpected generated key otherwise.
        ldict = {"key": "thing", "fred":"george", "larry":{"gertrude": 47}}

        self.convertToGDSAndQuery(ldict, "fred", "george")

    def test7(self):
        # need a key because we're actually saving the object here, it'll come back to us with an unexpected generated key otherwise.
        ldict = {"key": "thing", "fred":"george", "larry":{"gertrude": 47}}

        self.convertToGDSAndQuery(ldict, "larry.gertrude", 47)

    def test8(self):
        # need a key because we're actually saving the object here, it'll come back to us with an unexpected generated key otherwise.
        ldict = {"key": "thing", "fred":"george", "gertrude": [1, 3, {"harry": "thing"}], "larry":{"gertrude": 47}}

        self.convertToGDSAndQuery(ldict, "larry.gertrude", 47)

    def test9(self):
        ldict = {"fred":"george", "bill": [1, 2], "harry": []}

        self.convertToGDSAndBack(ldict)


from selenium import webdriver
import scrapy
from scrapy.selector import Selector
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from shutil import which
from fake_useragent import UserAgent
import time
from selenium.webdriver.common.action_chains import ActionChains
import csv
from pandas import DataFrame

path = which("chromedriver")
options = Options()
options.add_experimental_option("detach", True)
options.add_argument("--headless")
ua = UserAgent()
userAgent = ua.random
options.add_argument(f'user-agent={userAgent}')
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(executable_path=path,options=options)
driver.get("https://www.fashiontiy.com/men-swimwear.html")
time.sleep(2)

def aloo():
    color,siz,images,qity_pri=[],[],[],[]
    try:
        coli_item = driver.find_elements_by_xpath("//div[@class='color_colors']/div")
        for i in range(0,len(coli_item)):
            btn = driver.find_elements_by_xpath("//div[@class='color_colors']/div")[i]
            driver.execute_script("arguments[0].click();", btn)
            time.sleep(2)
            col = driver.find_element_by_xpath("//div[@class='base_key_value size_title_left']/span[@class='notranslate']").text
            color.append(col)
            img = driver.find_elements_by_xpath("//div[@class='detail_left']/div[@class='left_thums']/img")
            for im in range(0,len(img)):
                img = driver.find_elements_by_xpath("//div[@class='detail_left']/div[@class='left_thums']/img")[im].get_attribute("src")
                images.append(img)

            sizes = driver.find_elements_by_xpath("//div[@class='size_list notranslate']/div/span[@class='size_item_name ']")
            for j in range(0,len(sizes)):
                sizes_avaiable = driver.find_elements_by_xpath("//div[@class='size_list notranslate']/div/span[@class='size_item_name ']")[j].text
                siz.append(sizes_avaiable)

        g1 =  ', '.join(images)
        g2 =  '|'.join(siz)
        g3 = ', '.join(color)
    except:
        print("There is nothing to scrape.")

    html = driver.page_source
    resp = Selector(text=html)
    for xlo in resp.xpath("//div[@class='right_price_parent']/div"):
        Quantity_price = xlo.xpath(".//p/text()").getall()
        qity_pri.append(Quantity_price)

    absolute_id = ""
    Id = resp.xpath("//div[@class='right_title notranslate']/text()").get()
    absolute_id = Id
    absolute_id = absolute_id.replace("SKU:","")
    absolute_id = absolute_id.lstrip()
    Name = resp.xpath("//div[@class='right_desc']/span/text()").get()
    Min_order_req = resp.xpath("//div[@class='right_moq base_key_value']/span[@class='notranslate']/text()").get()
    Gross_Weight = resp.xpath("(//div[@class='base_key_value']/span[not(contains(@class,'key'))])[1]/text()").get()
    Material = resp.xpath("(//div[@class='base_key_value']/span[not(contains(@class,'key'))])[2]/text()").get()

    with open("mens_swimwear.csv", 'a',newline='',encoding="utf-8") as file:
        writer = csv.writer(file)
        writer.writerow([absolute_id,Name,Min_order_req,g3,qity_pri,g2,g1,Gross_Weight,Material])
        print("Data Saved in CSV :")
def moli():

    links = driver.find_elements_by_xpath("//div[@class='list_products']/div//a[@class='product_infos_2 notranslate']")
    for i in range(0,len(links)):
        link = driver.find_elements_by_xpath("//div[@class='list_products']/div//a[@class='product_infos_2 notranslate']")[i].get_attribute("href")
        driver.get(link)
        time.sleep(2)
        aloo()

        driver.back()
        time.sleep(5)
    next_page = driver.find_element_by_xpath("//div[contains(text(),'NEXT')]")
    if next_page:
        next_page.click()
        time.sleep(4)
        moli()

with open("mens_swimwear.csv", 'w', newline='') as file:
    writer = csv.writer(file)
    writer.writerow(["ID","Name","Minimum Order requirenment","color","Quantity and price","Size","images_url","Gross Weight","Material"])
moli()


# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
import sys
sys.path.append('/usr/local//lib/python3.8/dist-packages/sympy')
from sympy import *
import numpy
import random

debug = False

# maximum error: if |field| is 28

def ord2(char):
    if ord(char) - ord('0') > 10:
        return ord(char) - ord('a') + 10
    return ord(char) - ord('0')

def encode_one_letter(word, val):
    start_ = 0
    sum = 0
    for i in range(0, len(word)):
        sum += val ** start_ * ord2(word[i])
        start_ += 1
    return sum

def make_noise(X_arr, Y_arr, num_of_noise):
    num_of_curr_noise = 0
    history = []
    if num_of_noise == 0:
        return
    while true:
        curr = random.randint(0, len(X_arr) - 1)
        if history.count(curr) == 0:
            history.append(curr)
            X_arr[curr] = random.randint(0, 100)
            Y_arr[curr] = random.randint(0, 100)
        if len(history) == num_of_noise:
            break

# inputs:
encode = "00100"
print("Sending: ", encode)
print('Number of errors that will occur when sending the message:')
error = int(input())
x_arr = []
y_arr = []
field = 10
# Change the 28 to change the field size!
for i in range(0, field):
    y_arr.append(encode_one_letter(encode, i))
    x_arr.append(i)
print('The message was encoded to: ', [[x_arr[i],y_arr[i]] for i in range(0,field)])
make_noise(x_arr, y_arr, error)
print('The message after the noise tunnel: ', [[x_arr[i],y_arr[i]] for i in range(0,field)])
n = len(x_arr)
k = len(encode) - 1
t = int(sqrt(2 * k * n)) - 1
print('n is: ', n, '\nD is:', t, '\nt is: ', t + 1, '\ne is ', error, '\nk is: ', k)

for i in range(0, list.__len__(x_arr)):
    x_arr[i] = x_arr[i]

def double_sigma_algo1(D, k, X, Y, Z_arr):  # FOR BUILDING THE (1,k) DEGREE POLY
    for x in range(0, list.__len__(x_arr)):  # fix with +1
        for y in range(0, list.__len__(x_arr)):
            if x + k * y <= D:
                list.append(Z_arr, ((pow(X, x)) * (pow(Y, y))))

def double_sigma_algo1_last_step(D, k, co_acc, Z_arr):  # FOR BUILDING THE (1,k) DEGREE POLY WITH FEW ADDITIONS
    i = 0
    for x in range(0, list.__len__(x_arr)):  # fix with +1
        for y in range(0, list.__len__(x_arr)):
            if x + k * y <= D:
                list.append(Z_arr, ((co_acc[i]) * Symbol('x') ** (x) * Symbol('y') ** y))
                i = i + 1

def can_be_poli_div(p):
    # TESTING IF THE MULTIVARIATE POLYNOMIAL IS IN THE FORM OF (Y-P(X)) WITH FEW ADDITIONS FOR THE RETURN VALUE
    args = p.args
    count = 0
    legal = 0
    e = 1
    for arg in args:
        if (arg.replace(Symbol('y'), 1)*2 == arg.replace(Symbol('y'),2)) and arg.replace(Symbol('x'),0) == arg:
            legal = 1
            count = count + 1
            if (arg.replace(Symbol('y'), 1).replace(Symbol('x'), 1) < 0):
                e = -1
        if (arg.replace(Symbol('y'), 1) * 2 == arg.replace(Symbol('y'), 2)) and arg.replace(Symbol('x'), 0) != arg:
            return 'null'
    if (count == 1) and legal == 1:
        return e
    return 'null'

#########################################################BUILDING THE Q(X,Y)#######################################################
z_acc = []
z_arr = []
for i in range(0, list.__len__(x_arr)):
    double_sigma_algo1(t, k, x_arr[i], y_arr[i], z_arr)
    list.append(z_acc, z_arr)
    z_arr = []
z_acc = numpy.fliplr(z_acc)
z_acc = numpy.fliplr(z_acc)
z_soof = []
for i in range(0, int(numpy.size(z_acc) / numpy.size(z_acc[0]))):
    row = []
    for j in range(0, numpy.size(z_acc[0])):
        if z_acc[i][j] != 0:
            if numpy.size(row) == 0:
                row = [(z_acc[i][j]) * Symbol('a' + str(j))]
            else:
                row[0] = row[0] + z_acc[i][j] * Symbol('a' + str(j))
    list.append(z_soof, row[0])

#############################################################SOLVING THE Q(X,Y)#######################################################################

# now we have the full linear equation ready to solve,
# just need to prepare some things for the final step of building the polynomial.
poly_coeff = []
solved = solve(z_soof)
for i in range(0, numpy.size(z_acc[0])):
    poly_coeff.append('null')
for i in range(0, numpy.size(z_acc[0])):
    if not solved.keys().__contains__(Symbol('a' + str(i))):
        poly_coeff[i] = 1
while True:
    if list.count(poly_coeff, 'null') == 0:  # all coefficients found
        break
    for i in range(0, numpy.size(z_acc[0])):  # going through the coefficients to try and replace them
        if poly_coeff[i] != 'null':
            for x in range(0, list.__len__(z_soof)):
                z_soof[x] = z_soof[x].replace(Symbol('a' + str(i)), poly_coeff[i])
    solved = solve(z_soof)
    for s in solved:  # going through the equations to see if one of them can be turned into a number!
        if solved[s].is_Number:
            poly_coeff[int(str(s)[1:])] = (solved[s])
ff_arr = []
double_sigma_algo1_last_step(t, k, poly_coeff, ff_arr)
ff_arr_flat = 0
for line in ff_arr:
    ff_arr_flat = ff_arr_flat + line
# testing if the polynomial is legit:
for i in range(0, list.__len__(x_arr)):
    if not int(ff_arr_flat.replace(Symbol('x'), x_arr[i]).replace(Symbol('y'), y_arr[i])) == 0:
        print('oops ', i)
        print((ff_arr_flat.replace(Symbol('x'), x_arr[i]).replace(Symbol('y'), y_arr[i])))

#########################################################################ANALYZING THE OUTPUT########################################################################
print('-------')
f = False

# Here we split the Q(X,Y) into smaller polynomials, we study each one of them and see
# if it is in the form of "Y-P(X)", and see if it agrees with atleast 't' coordinates
# and it is a poly with at most 'k' degree.
if ff_arr_flat.__eq__(factor(ff_arr_flat)):
    ff_arr_flat = (ff_arr_flat)
print('Q(X,Y): ', ff_arr_flat)
print('Factorized Q(X,Y): ', factor(ff_arr_flat))
c = 0
skip = 0
for x in factor(ff_arr_flat).args:
    c = 0
    p = can_be_poli_div(x)
    if p != 'null':
        for i in range(0, list.__len__(x_arr)):
            if int(x.replace(Symbol('x'), x_arr[i]).replace(Symbol('y'), y_arr[i])) == 0:
                c = c + 1
        if c > t:
            if not x.is_Number:
                print('x is ',c)
                x = x.replace(Symbol('y'), 0)
                x_ = 0 - (x * p)
                ret = ""
                count = 0
                index = 0
                map = {}
                while index < len(x_.args) and x_.is_Add:
                    if x_.args[index].is_Number:
                        map[0] = abs(x_.args[index])
                    elif len(x_.args[index].args) == 0:
                        map[1] = abs(x_.args[index].replace(Symbol('x'), 1))
                    elif len(x_.args[index].args[1].args) < 2 and x_.args[index].args[0].is_Number:
                        map[1] = abs(x_.args[index].args[0])
                    elif x_.args[index].args[1].is_Number:
                        map[x_.args[index].args[1]] = 1
                    else:
                        map[x_.args[index].args[1].args[1]] = x_.args[index].args[0]
                    index += 1
                if x_.is_Number:
                    map[0]=x_
                if x_.is_Symbol:
                    map[1] = 1
                if x_.args[0].is_Number:
                    map[x_.args[1].args[1]] = abs(x_.args[0])
                if x_.args[1].is_Number:
                    map[x_.args[1]] = 1
                for i in range(0, k + 1):
                    if map.__contains__(i):
                        if map[i] < 10:
                            ret += chr(map[i] + ord('0'))
                        else:
                            ret += chr(map[i] - 10 + ord('a'))
                    else:
                        ret += '0'
                print("Decoded into: ", ret)
                f = True
                break
        c = 0
if not f:
    print('could not find a polynomial that satisfies both \'k\' and \'t\' restrictions ')


print('Input 10 angka (dipisah dengan enter): ');

# simpan setiap angka yang diinput ke dalam list
x = []
for i in range(10):
  print('Angka ke-',i+1,': ',sep='',end='')
  x.append(int(input()))

print()

angka = int(input('Input angka yang akan dicari: '))

# proses pencarian element list
for i in range(10):
  if(x[i]==angka):
    print('Angka ditemukan pada urutan ke',i+1)
    break

if(i+1 == 10):
  print('Angka tidak ditemukan')


from random import randrange
from get_users_data import ApiVk
from find_peoples import VkFinder
from get_user_photos import top_three_photos
from create_database import get_user_id, get_result, get_photo, get_result_photo, get_all_users, \
    get_user_result, get_all_domains, get_exist_pairs

from settings.settings import GROUPS_TOKEN

import vk_api
from vk_api.longpoll import VkLongPoll, VkEventType

import random

token = GROUPS_TOKEN

vk = vk_api.VkApi(token=token)
longpoll = VkLongPoll(vk)

def write_msg(user_id, message):
    vk.method('messages.send', {'user_id': user_id, 'message': message,  'random_id': randrange(10 ** 7),})

for event in longpoll.listen():
    if event.type == VkEventType.MESSAGE_NEW:

        if event.to_me:
            # Получаем id пользователя
            request = event.text
            if (str(request),) not in get_all_users():
                get_user_id(request)

            api_vk = ApiVk(str(request))
            finders_data = api_vk.get_users_data()
            # Уточняем данные пользователя ,если их не хватает
            for key, value in finders_data.items():
                if key == 'age' and value == None:
                    if finders_data['age'] == None:
                        write_msg(event.user_id, "Укажите ваш возраст")
                        for event in longpoll.listen():
                            if event.type == VkEventType.MESSAGE_NEW:
                                if event.to_me:
                                    age = event.text
                                    finders_data['age'] = age
                                    break
                if key == 'city' and value == None:
                    if finders_data['city'] == None:
                        write_msg(event.user_id, "Укажите ваш город")
                        for event in longpoll.listen():
                            if event.type == VkEventType.MESSAGE_NEW:
                                if event.to_me:
                                    city = event.text
                                    finders_data['city'] = city
                                    break

            vkfinder = VkFinder(finders_data['city'], finders_data['sex'], finders_data['age'],
                                    finders_data['age'])
            congruence = vkfinder.get_users()
            single = random.choice(congruence)

            link = single[
                'domain']  # Получаем ссылку на найденного пользователя
            # Проверям была ли такая пара
            while (request, link) in get_exist_pairs():
                single = random.choice(congruence)
                link = single['domain']

            # Проверяем есть ли найденный пользователь в базе данных
            if (link,) in get_all_domains():
                (get_user_result(request, link))
                write_msg(event.user_id, link)
                for photo in top_three_photos(str(single['id'])):
                    ph = photo['sizes'][-1]['url']  # Получаем фото
                    get_result_photo(link, ph)
                    write_msg(event.user_id, ph)

            else:
                get_result(link)
                get_user_result(request, link)
                write_msg(event.user_id, link)
                for photo in top_three_photos(str(single['id'])):
                    ph = photo['sizes'][-1]['url']  # Получаем фото
                    get_photo(ph)
                    get_result_photo(link, ph)
                    write_msg(event.user_id, ph)

            print("Поиск завершен")


import logging
import os
import datetime as dt
import json
import azure.functions as func
from azure.storage.blob import BlobClient, ContainerClient, ContentSettings
from .tablestorage import tablestorage
from .event import event

def uploadblob(json_in, blobname, conn_str, lin_container):

    container_client = ContainerClient.from_connection_string(conn_str, container_name=lin_container)
    blob = BlobClient.from_connection_string(conn_str, container_name= lin_container, blob_name=blobname)
    BlobClient
    blob.upload_blob(json_in, overwrite=True)

def main(req: func.HttpRequest) -> func.HttpResponse:

    logging.info("http trigger function kicked off")

    lineageContainerStr = os.environ["LINEAGE_STORAGE_CONN_STR"]
    lineageContainer = os.environ["LINEAGE_CONTAINER"]

    data = req.get_json()

    currenttimestamp = dt.datetime.utcnow().strftime("%Y%m%d%H%M%S")
    eventType = data["eventType"]
    className = data["run"]["facets"]["spark.logicalPlan"]["plan"][0]["class"]
    runId = data["run"]["runId"]
    notebookName = data["job"]["name"]
    notebookName = notebookName[0 : notebookName.index('.')]

    fileName = runId + '_' + notebookName + '_' + currenttimestamp + '.json'
    filePath = lineageContainer + '/' + fileName

    predefined_class_list = ["org.apache.spark.sql.execution.datasources.CreateTable",
                "org.apache.spark.sql.catalyst.plans.logical.CreateViewStatement",
                "org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelectStatement",
                "org.apache.spark.sql.catalyst.plans.logical.InsertIntoStatement",
                "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand"]

    if eventType is not None and eventType == "COMPLETE" and className in predefined_class_list :
        # upload file as json into blob storage
        uploadblob(json.dumps(data), fileName, lineageContainerStr, lineageContainer)

        #code to add new unprocessed row for same uploaded json into azure table
        eventrow = event('HRSI', fileName)
        eventrow.Status = 'Unprocessed'
        eventrow.RetryCount = 3
        eventrow.FilepPath = filePath
        eventrow.isArchived = False
        eventrow.Message = ''

        tableStorage = tablestorage()
        tableStorage.insertEventMetadata(eventrow.__dict__)

        return func.HttpResponse(f"Func App successfully processed http request")

    else:

        return func.HttpResponse(f"Event Type is not COMPLETE Or ClassName Not Matched")


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode, carrier=0) -> ListNode:
        val = l1.val + l2.val + carrier
        carrier = val // 10
        result = ListNode(val % 10)
        while l1.next is not None or l2.next is not None or carrier != 0:
            if l1.next is None:
                l1.next = ListNode(0)
            if l2.next is None:
                l2.next = ListNode(0)
            result.next = self.addTwoNumbers(l1.next, l2.next, carrier)
            return result
        return result


import sys
import os
import numpy as np
import csv
import random

def readAnnotations(name):
    # camid: 2
    # filename: 3
    # spring, summer, fall, winter: 4-7
    # night, twilight, sunrise, sunset, day, fullday: 8-13

    season = {}
    season['spring'] = []
    season['summer'] = []
    season['fall'] = []
    season['winter'] = []

    tod = {}
    tod['night'] = []
    tod['twilight'] = []
    tod['sunrise'] = []
    tod['sunset'] = []
    tod['day'] = []
    tod['fullday'] = []

    with open(name) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        header = True
        for row in reader:
            if header:
                header=False
                continue

            imgfn = row[2] + '/' + row[3]

            if int(row[4])>0: season['spring'].append(imgfn)
            if int(row[5])>0: season['summer'].append(imgfn)
            if int(row[6])>0: season['fall'].append(imgfn)
            if int(row[7])>0: season['winter'].append(imgfn)

            if int(row[8])>0: tod['night'].append(imgfn)
            if int(row[9])>0: tod['twilight'].append(imgfn)
            if int(row[10])>0: tod['sunrise'].append(imgfn)
            if int(row[11])>0: tod['sunset'].append(imgfn)
            if int(row[12])>0: tod['day'].append(imgfn)
            if int(row[13])>0: tod['fullday'].append(imgfn)

    return (season,tod)

# main

if len(sys.argv)<5:
    print("usage: create_timm_skyfinder.py task annotationfile outputdir sourcedir valshare")
    print("       task: season or tod")
    exit()

task = sys.argv[1]
season,tod = readAnnotations(sys.argv[2])
outdir = sys.argv[3]
srcdir = sys.argv[4]
valshare = float(sys.argv[5])

# keep only full day, not day
del tod['day']

if task=='season': mydict = season
elif task=='tod': mydict = tod
else:
    print('unknown task '+task)
    exit()

# create directories
traindir = outdir + '/train'
valdir = outdir + '/val'
os.makedirs(traindir, exist_ok=True)
os.makedirs(valdir, exist_ok=True)

for c in mydict.keys():
    os.makedirs(traindir + '/'+ c, exist_ok=True)
    os.makedirs(valdir + '/'+ c, exist_ok=True)

for c in mydict.keys():
    items = mydict[c]
    random.shuffle(items)

    firstval = int(len(items)*(1-valshare))

    for i in range(firstval):
        trgname = items[i].replace('/','_')
        if os.path.exists(srcdir+'/'+items[i]):
            os.symlink(srcdir+'/'+items[i],traindir+'/'+c+'/'+trgname)
        else:
            print(imgsourcedir+'/'+items[i] + 'does not exist')
    for i in range(firstval,len(items)):
        trgname = items[i].replace('/','_')
        if os.path.exists(srcdir+'/'+items[i]):
            os.symlink(srcdir+'/'+items[i],valdir+'/'+c+'/'+trgname)
        else:
            print(imgsourcedir+'/'+items[i] + 'does not exist')