text:
namespace {

enum MessageLoopProblems {
  MESSAGE_POST_ERROR,
  COMPLETION_POST_ERROR,
  SET_TIMER_ERROR,
  MESSAGE_LOOP_PROBLEM_MAX,
};

}  // namespace

namespace base {

static const wchar_t kWndClass[] = L"Chrome_MessagePumpWindow";

// Message sent to get an additional time slice for pumping (processing) another
// task (a series of such messages creates a continuous task pump).
static const int kMsgHaveWork = WM_USER + 1;

//-----------------------------------------------------------------------------
// MessagePumpWin public:

void MessagePumpWin::AddObserver(MessagePumpObserver* observer) {
  observers_.AddObserver(observer);
}

void MessagePumpWin::RemoveObserver(MessagePumpObserver* observer) {
  observers_.RemoveObserver(observer);
}

void MessagePumpWin::WillProcessMessage(const MSG& msg) {
  FOR_EACH_OBSERVER(MessagePumpObserver, observers_, WillProcessEvent(msg));
}

void MessagePumpWin::DidProcessMessage(const MSG& msg) {
  FOR_EACH_OBSERVER(MessagePumpObserver, observers_, DidProcessEvent(msg));
}

void MessagePumpWin::RunWithDispatcher(
    Delegate* delegate, MessagePumpDispatcher* dispatcher) {
  RunState s;
  s.delegate = delegate;
  s.dispatcher = dispatcher;
  s.should_quit = false;
  s.run_depth = state_ ? state_->run_depth + 1 : 1;

  RunState* previous_state = state_;
  state_ = &s;

  DoRunLoop();

  state_ = previous_state;
}

void MessagePumpWin::Quit() {
  DCHECK(state_);
  state_->should_quit = true;
}

//-----------------------------------------------------------------------------
// MessagePumpWin protected:

int MessagePumpWin::GetCurrentDelay() const {
  if (delayed_work_time_.is_null())
    return -1;

  // Be careful here.  TimeDelta has a precision of microseconds, but we want a
  // value in milliseconds.  If there are 5.5ms left, should the delay be 5 or
  // 6?  It should be 6 to avoid executing delayed work too early.
  double timeout =
      ceil((delayed_work_time_ - TimeTicks::Now()).InMillisecondsF());

  // If this value is negative, then we need to run delayed work soon.
  int delay = static_cast<int>(timeout);
  if (delay < 0)
    delay = 0;

  return delay;
}

//-----------------------------------------------------------------------------
// MessagePumpForUI public:

MessagePumpForUI::MessagePumpForUI() : instance_(NULL) {
  InitMessageWnd();
}

MessagePumpForUI::~MessagePumpForUI() {
  DestroyWindow(message_hwnd_);
  UnregisterClass(kWndClass, instance_);
}

void MessagePumpForUI::ScheduleWork() {
  if (InterlockedExchange(&have_work_, 1))
    return;  // Someone else continued the pumping.

  // Make sure the MessagePump does some work for us.
  BOOL ret = PostMessage(message_hwnd_, kMsgHaveWork,
                         reinterpret_cast<WPARAM>(this), 0);
  if (ret)
    return;  // There was room in the Window Message queue.

  // We have failed to insert a have-work message, so there is a chance that we
  // will starve tasks/timers while sitting in a nested message loop.  Nested
  // loops only look at Windows Message queues, and don't look at *our* task
  // queues, etc., so we might not get a time slice in such. :-(
  // We could abort here, but the fear is that this failure mode is plausibly
  // common (queue is full, of about 2000 messages), so we'll do a near-graceful
  // recovery.  Nested loops are pretty transient (we think), so this will
  // probably be recoverable.
  InterlockedExchange(&have_work_, 0);  // Clarify that we didn't really insert.
  UMA_HISTOGRAM_ENUMERATION("Chrome.MessageLoopProblem", MESSAGE_POST_ERROR,
                            MESSAGE_LOOP_PROBLEM_MAX);
}

void MessagePumpForUI::ScheduleDelayedWork(const TimeTicks& delayed_work_time) {
  //
  // We would *like* to provide high resolution timers.  Windows timers using
  // SetTimer() have a 10ms granularity.  We have to use WM_TIMER as a wakeup
  // mechanism because the application can enter modal windows loops where it
  // is not running our MessageLoop; the only way to have our timers fire in
  // these cases is to post messages there.
  //
  // To provide sub-10ms timers, we process timers directly from our run loop.
  // For the common case, timers will be processed there as the run loop does
  // its normal work.  However, we *also* set the system timer so that WM_TIMER
  // events fire.  This mops up the case of timers not being able to work in
  // modal message loops.  It is possible for the SetTimer to pop and have no
  // pending timers, because they could have already been processed by the
  // run loop itself.
  //
  // We use a single SetTimer corresponding to the timer that will expire
  // soonest.  As new timers are created and destroyed, we update SetTimer.
  // Getting a spurrious SetTimer event firing is benign, as we'll just be
  // processing an empty timer queue.
  //
  delayed_work_time_ = delayed_work_time;

  int delay_msec = GetCurrentDelay();
  DCHECK_GE(delay_msec, 0);
  if (delay_msec < USER_TIMER_MINIMUM)
    delay_msec = USER_TIMER_MINIMUM;

  // Create a WM_TIMER event that will wake us up to check for any pending
  // timers (in case we are running within a nested, external sub-pump).
  BOOL ret = SetTimer(message_hwnd_, reinterpret_cast<UINT_PTR>(this),
                      delay_msec, NULL);
  if (ret)
    return;
  // If we can't set timers, we are in big trouble... but cross our fingers for
  // now.
  // TODO(jar): If we don't see this error, use a CHECK() here instead.
  UMA_HISTOGRAM_ENUMERATION("Chrome.MessageLoopProblem", SET_TIMER_ERROR,
                            MESSAGE_LOOP_PROBLEM_MAX);
}

void MessagePumpForUI::PumpOutPendingPaintMessages() {
  // If we are being called outside of the context of Run, then don't try to do
  // any work.
  if (!state_)
    return;

  // Create a mini-message-pump to force immediate processing of only Windows
  // WM_PAINT messages.  Don't provide an infinite loop, but do enough peeking
  // to get the job done.  Actual common max is 4 peeks, but we'll be a little
  // safe here.
  const int kMaxPeekCount = 20;
  int peek_count;
  for (peek_count = 0; peek_count < kMaxPeekCount; ++peek_count) {
    MSG msg;
    if (!PeekMessage(&msg, NULL, 0, 0, PM_REMOVE | PM_QS_PAINT))
      break;
    ProcessMessageHelper(msg);
    if (state_->should_quit)  // Handle WM_QUIT.
      break;
  }
  // Histogram what was really being used, to help to adjust kMaxPeekCount.
  DHISTOGRAM_COUNTS("Loop.PumpOutPendingPaintMessages Peeks", peek_count);
}

//-----------------------------------------------------------------------------
// MessagePumpForUI private:

// static
LRESULT CALLBACK MessagePumpForUI::WndProcThunk(
    HWND hwnd, UINT message, WPARAM wparam, LPARAM lparam) {
  switch (message) {
    case kMsgHaveWork:
      reinterpret_cast<MessagePumpForUI*>(wparam)->HandleWorkMessage();
      break;
    case WM_TIMER:
      reinterpret_cast<MessagePumpForUI*>(wparam)->HandleTimerMessage();
      break;
  }
  return DefWindowProc(hwnd, message, wparam, lparam);
}

void MessagePumpForUI::DoRunLoop() {
  // IF this was just a simple PeekMessage() loop (servicing all possible work
  // queues), then Windows would try to achieve the following order according
  // to MSDN documentation about PeekMessage with no filter):
  //    * Sent messages
  //    * Posted messages
  //    * Sent messages (again)
  //    * WM_PAINT messages
  //    * WM_TIMER messages
  //
  // Summary: none of the above classes is starved, and sent messages has twice
  // the chance of being processed (i.e., reduced service time).

  for (;;) {
    // If we do any work, we may create more messages etc., and more work may
    // possibly be waiting in another task group.  When we (for example)
    // ProcessNextWindowsMessage(), there is a good chance there are still more
    // messages waiting.  On the other hand, when any of these methods return
    // having done no work, then it is pretty unlikely that calling them again
    // quickly will find any work to do.  Finally, if they all say they had no
    // work, then it is a good time to consider sleeping (waiting) for more
    // work.

    bool more_work_is_plausible = ProcessNextWindowsMessage();
    if (state_->should_quit)
      break;

    more_work_is_plausible |= state_->delegate->DoWork();
    if (state_->should_quit)
      break;

    more_work_is_plausible |=
        state_->delegate->DoDelayedWork(&delayed_work_time_);
    // If we did not process any delayed work, then we can assume that our
    // existing WM_TIMER if any will fire when delayed work should run.  We
    // don't want to disturb that timer if it is already in flight.  However,
    // if we did do all remaining delayed work, then lets kill the WM_TIMER.
    if (more_work_is_plausible && delayed_work_time_.is_null())
      KillTimer(message_hwnd_, reinterpret_cast<UINT_PTR>(this));
    if (state_->should_quit)
      break;

    if (more_work_is_plausible)
      continue;

    more_work_is_plausible = state_->delegate->DoIdleWork();
    if (state_->should_quit)
      break;

    if (more_work_is_plausible)
      continue;

    WaitForWork();  // Wait (sleep) until we have work to do again.
  }
}

void MessagePumpForUI::InitMessageWnd() {
  WNDCLASSEX wc = {0};
  wc.cbSize = sizeof(wc);
  wc.lpfnWndProc = base::win::WrappedWindowProc<WndProcThunk>;
  wc.hInstance = base::GetModuleFromAddress(wc.lpfnWndProc);
  wc.lpszClassName = kWndClass;
  instance_ = wc.hInstance;
  RegisterClassEx(&wc);

  message_hwnd_ =
      CreateWindow(kWndClass, 0, 0, 0, 0, 0, 0, HWND_MESSAGE, 0, instance_, 0);
  DCHECK(message_hwnd_);
}

void MessagePumpForUI::WaitForWork() {
  // Wait until a message is available, up to the time needed by the timer
  // manager to fire the next set of timers.
  int delay = GetCurrentDelay();
  if (delay < 0)  // Negative value means no timers waiting.
    delay = INFINITE;

  DWORD result;
  result = MsgWaitForMultipleObjectsEx(0, NULL, delay, QS_ALLINPUT,
                                       MWMO_INPUTAVAILABLE);

  if (WAIT_OBJECT_0 == result) {
    // A WM_* message is available.
    // If a parent child relationship exists between windows across threads
    // then their thread inputs are implicitly attached.
    // This causes the MsgWaitForMultipleObjectsEx API to return indicating
    // that messages are ready for processing (specifically mouse messages
    // intended for the child window. Occurs if the child window has capture)
    // The subsequent PeekMessages call fails to return any messages thus
    // causing us to enter a tight loop at times.
    // The WaitMessage call below is a workaround to give the child window
    // sometime to process its input messages.
    MSG msg = {0};
    DWORD queue_status = GetQueueStatus(QS_MOUSE);
    if (HIWORD(queue_status) & QS_MOUSE &&
       !PeekMessage(&msg, NULL, WM_MOUSEFIRST, WM_MOUSELAST, PM_NOREMOVE)) {
      WaitMessage();
    }
    return;
  }

  DCHECK_NE(WAIT_FAILED, result) << GetLastError();
}

void MessagePumpForUI::HandleWorkMessage() {
  // If we are being called outside of the context of Run, then don't try to do
  // any work.  This could correspond to a MessageBox call or something of that
  // sort.
  if (!state_) {
    // Since we handled a kMsgHaveWork message, we must still update this flag.
    InterlockedExchange(&have_work_, 0);
    return;
  }

  // Let whatever would have run had we not been putting messages in the queue
  // run now.  This is an attempt to make our dummy message not starve other
  // messages that may be in the Windows message queue.
  ProcessPumpReplacementMessage();

  // Now give the delegate a chance to do some work.  He'll let us know if he
  // needs to do more work.
  if (state_->delegate->DoWork())
    ScheduleWork();
}

void MessagePumpForUI::HandleTimerMessage() {
  KillTimer(message_hwnd_, reinterpret_cast<UINT_PTR>(this));

  // If we are being called outside of the context of Run, then don't do
  // anything.  This could correspond to a MessageBox call or something of
  // that sort.
  if (!state_)
    return;

  state_->delegate->DoDelayedWork(&delayed_work_time_);
  if (!delayed_work_time_.is_null()) {
    // A bit gratuitous to set delayed_work_time_ again, but oh well.
    ScheduleDelayedWork(delayed_work_time_);
  }
}

bool MessagePumpForUI::ProcessNextWindowsMessage() {
  // If there are sent messages in the queue then PeekMessage internally
  // dispatches the message and returns false. We return true in this
  // case to ensure that the message loop peeks again instead of calling
  // MsgWaitForMultipleObjectsEx again.
  bool sent_messages_in_queue = false;
  DWORD queue_status = GetQueueStatus(QS_SENDMESSAGE);
  if (HIWORD(queue_status) & QS_SENDMESSAGE)
    sent_messages_in_queue = true;

  MSG msg;
  if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE))
    return ProcessMessageHelper(msg);

  return sent_messages_in_queue;
}

bool MessagePumpForUI::ProcessMessageHelper(const MSG& msg) {
  TRACE_EVENT1("base", "MessagePumpForUI::ProcessMessageHelper",
               "message", msg.message);
  if (WM_QUIT == msg.message) {
    // Repost the QUIT message so that it will be retrieved by the primary
    // GetMessage() loop.
    state_->should_quit = true;
    PostQuitMessage(static_cast<int>(msg.wParam));
    return false;
  }

  // While running our main message pump, we discard kMsgHaveWork messages.
  if (msg.message == kMsgHaveWork && msg.hwnd == message_hwnd_)
    return ProcessPumpReplacementMessage();

  if (CallMsgFilter(const_cast<MSG*>(&msg), kMessageFilterCode))
    return true;

  WillProcessMessage(msg);

  if (state_->dispatcher) {
    if (!state_->dispatcher->Dispatch(msg))
      state_->should_quit = true;
  } else {
    TranslateMessage(&msg);
    DispatchMessage(&msg);
  }

  DidProcessMessage(msg);
  return true;
}

bool MessagePumpForUI::ProcessPumpReplacementMessage() {
  // When we encounter a kMsgHaveWork message, this method is called to peek
  // and process a replacement message, such as a WM_PAINT or WM_TIMER.  The
  // goal is to make the kMsgHaveWork as non-intrusive as possible, even though
  // a continuous stream of such messages are posted.  This method carefully
  // peeks a message while there is no chance for a kMsgHaveWork to be pending,
  // then resets the have_work_ flag (allowing a replacement kMsgHaveWork to
  // possibly be posted), and finally dispatches that peeked replacement.  Note
  // that the re-post of kMsgHaveWork may be asynchronous to this thread!!

  bool have_message = false;
  MSG msg;
  // We should not process all window messages if we are in the context of an
  // OS modal loop, i.e. in the context of a windows API call like MessageBox.
  // This is to ensure that these messages are peeked out by the OS modal loop.
  if (MessageLoop::current()->os_modal_loop()) {
    // We only peek out WM_PAINT and WM_TIMER here for reasons mentioned above.
    have_message = PeekMessage(&msg, NULL, WM_PAINT, WM_PAINT, PM_REMOVE) ||
                   PeekMessage(&msg, NULL, WM_TIMER, WM_TIMER, PM_REMOVE);
  } else {
    have_message = (0 != PeekMessage(&msg, NULL, 0, 0, PM_REMOVE));
  }

  DCHECK(!have_message || kMsgHaveWork != msg.message ||
         msg.hwnd != message_hwnd_);

  // Since we discarded a kMsgHaveWork message, we must update the flag.
  int old_have_work = InterlockedExchange(&have_work_, 0);
  DCHECK(old_have_work);

  // We don't need a special time slice if we didn't have_message to process.
  if (!have_message)
    return false;

  // Guarantee we'll get another time slice in the case where we go into native
  // windows code.   This ScheduleWork() may hurt performance a tiny bit when
  // tasks appear very infrequently, but when the event queue is busy, the
  // kMsgHaveWork events get (percentage wise) rarer and rarer.
  ScheduleWork();
  return ProcessMessageHelper(msg);
}

//-----------------------------------------------------------------------------
// MessagePumpForIO public:

MessagePumpForIO::MessagePumpForIO() {
  port_.Set(CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, NULL, 1));
  DCHECK(port_.IsValid());
}

void MessagePumpForIO::ScheduleWork() {
  if (InterlockedExchange(&have_work_, 1))
    return;  // Someone else continued the pumping.

  // Make sure the MessagePump does some work for us.
  BOOL ret = PostQueuedCompletionStatus(port_, 0,
                                        reinterpret_cast<ULONG_PTR>(this),
                                        reinterpret_cast<OVERLAPPED*>(this));
  if (ret)
    return;  // Post worked perfectly.

  // See comment in MessagePumpForUI::ScheduleWork() for this error recovery.
  InterlockedExchange(&have_work_, 0);  // Clarify that we didn't succeed.
  UMA_HISTOGRAM_ENUMERATION("Chrome.MessageLoopProblem", COMPLETION_POST_ERROR,
                            MESSAGE_LOOP_PROBLEM_MAX);
}

void MessagePumpForIO::ScheduleDelayedWork(const TimeTicks& delayed_work_time) {
  // We know that we can't be blocked right now since this method can only be
  // called on the same thread as Run, so we only need to update our record of
  // how long to sleep when we do sleep.
  delayed_work_time_ = delayed_work_time;
}

void MessagePumpForIO::RegisterIOHandler(HANDLE file_handle,
                                         IOHandler* handler) {
  ULONG_PTR key = reinterpret_cast<ULONG_PTR>(handler);
  HANDLE port = CreateIoCompletionPort(file_handle, port_, key, 1);
  DPCHECK(port);
}

//-----------------------------------------------------------------------------
// MessagePumpForIO private:

void MessagePumpForIO::DoRunLoop() {
  for (;;) {
    // If we do any work, we may create more messages etc., and more work may
    // possibly be waiting in another task group.  When we (for example)
    // WaitForIOCompletion(), there is a good chance there are still more
    // messages waiting.  On the other hand, when any of these methods return
    // having done no work, then it is pretty unlikely that calling them
    // again quickly will find any work to do.  Finally, if they all say they
    // had no work, then it is a good time to consider sleeping (waiting) for
    // more work.

    bool more_work_is_plausible = state_->delegate->DoWork();
    if (state_->should_quit)
      break;

    more_work_is_plausible |= WaitForIOCompletion(0, NULL);
    if (state_->should_quit)
      break;

    more_work_is_plausible |=
        state_->delegate->DoDelayedWork(&delayed_work_time_);
    if (state_->should_quit)
      break;

    if (more_work_is_plausible)
      continue;

    more_work_is_plausible = state_->delegate->DoIdleWork();
    if (state_->should_quit)
      break;

    if (more_work_is_plausible)
      continue;

    WaitForWork();  // Wait (sleep) until we have work to do again.
  }
}

// Wait until IO completes, up to the time needed by the timer manager to fire
// the next set of timers.
void MessagePumpForIO::WaitForWork() {
  // We do not support nested IO message loops. This is to avoid messy
  // recursion problems.
  DCHECK_EQ(1, state_->run_depth) << "Cannot nest an IO message loop!";

  int timeout = GetCurrentDelay();
  if (timeout < 0)  // Negative value means no timers waiting.
    timeout = INFINITE;

  WaitForIOCompletion(timeout, NULL);
}

bool MessagePumpForIO::WaitForIOCompletion(DWORD timeout, IOHandler* filter) {
  IOItem item;
  if (completed_io_.empty() || !MatchCompletedIOItem(filter, &item)) {
    // We have to ask the system for another IO completion.
    if (!GetIOItem(timeout, &item))
      return false;

    if (ProcessInternalIOItem(item))
      return true;
  }

  if (item.context->handler) {
    if (filter && item.handler != filter) {
      // Save this item for later
      completed_io_.push_back(item);
    } else {
      DCHECK_EQ(item.context->handler, item.handler);
      WillProcessIOEvent();
      item.handler->OnIOCompleted(item.context, item.bytes_transfered,
                                  item.error);
      DidProcessIOEvent();
    }
  } else {
    // The handler must be gone by now, just cleanup the mess.
    delete item.context;
  }
  return true;
}

// Asks the OS for another IO completion result.
bool MessagePumpForIO::GetIOItem(DWORD timeout, IOItem* item) {
  memset(item, 0, sizeof(*item));
  ULONG_PTR key = NULL;
  OVERLAPPED* overlapped = NULL;
  if (!GetQueuedCompletionStatus(port_.Get(), &item->bytes_transfered, &key,
                                 &overlapped, timeout)) {
    if (!overlapped)
      return false;  // Nothing in the queue.
    item->error = GetLastError();
    item->bytes_transfered = 0;
  }

  item->handler = reinterpret_cast<IOHandler*>(key);
  item->context = reinterpret_cast<IOContext*>(overlapped);
  return true;
}

bool MessagePumpForIO::ProcessInternalIOItem(const IOItem& item) {
  if (this == reinterpret_cast<MessagePumpForIO*>(item.context) &&
      this == reinterpret_cast<MessagePumpForIO*>(item.handler)) {
    // This is our internal completion.
    DCHECK(!item.bytes_transfered);
    InterlockedExchange(&have_work_, 0);
    return true;
  }
  return false;
}

// Returns a completion item that was previously received.
bool MessagePumpForIO::MatchCompletedIOItem(IOHandler* filter, IOItem* item) {
  DCHECK(!completed_io_.empty());
  for (std::list<IOItem>::iterator it = completed_io_.begin();
       it != completed_io_.end(); ++it) {
    if (!filter || it->handler == filter) {
      *item = *it;
      completed_io_.erase(it);
      return true;
    }
  }
  return false;
}

void MessagePumpForIO::AddIOObserver(IOObserver *obs) {
  io_observers_.AddObserver(obs);
}

void MessagePumpForIO::RemoveIOObserver(IOObserver *obs) {
  io_observers_.RemoveObserver(obs);
}

void MessagePumpForIO::WillProcessIOEvent() {
  FOR_EACH_OBSERVER(IOObserver, io_observers_, WillProcessIOEvent());
}

void MessagePumpForIO::DidProcessIOEvent() {
  FOR_EACH_OBSERVER(IOObserver, io_observers_, DidProcessIOEvent());
}

}  // namespace base


text:
/*	$NetBSD: com_supio.c,v 1.12 1998/09/16 21:30:58 is Exp $	*/

/*-
 * Copyright (c) 1998 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *        This product includes software developed by the NetBSD
 *        Foundation, Inc. and its contributors.
 * 4. Neither the name of The NetBSD Foundation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*-
 * Copyright (c) 1991 The Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)com.c	7.5 (Berkeley) 5/16/91
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/ioctl.h>
#include <sys/select.h>
#include <sys/tty.h>
#include <sys/proc.h>
#include <sys/user.h>
#include <sys/conf.h>
#include <sys/file.h>
#include <sys/uio.h>
#include <sys/kernel.h>
#include <sys/syslog.h>
#include <sys/types.h>
#include <sys/device.h>

#include <machine/intr.h>
#include <machine/bus.h>

/*#include <dev/isa/isavar.h>*/
#include <dev/ic/comreg.h>
#include <dev/ic/comvar.h>

#include <amiga/dev/supio.h>

struct comsupio_softc {
	struct com_softc sc_com;
	struct isr sc_isr;
};

int com_supio_match __P((struct device *, struct cfdata *, void *));
void com_supio_attach __P((struct device *, struct device *, void *));
void com_supio_cleanup __P((void *));

#if 0
static int      comconsaddr;
static bus_space_handle_t comconsioh;
static int      comconsattached;
static bus_space_tag_t comconstag;
static int comconsrate;
static tcflag_t comconscflag;
#endif

struct cfattach com_supio_ca = {
	sizeof(struct comsupio_softc), com_supio_match, com_supio_attach
};

int
com_supio_match(parent, match, aux)
	struct device *parent;
	struct cfdata *match;
	void *aux;
{
	bus_space_tag_t iot;
	int iobase;
	int rv = 1;
	struct supio_attach_args *supa = aux;

	iot = supa->supio_iot;
	iobase = supa->supio_iobase;

	if (strcmp(supa->supio_name,"com"))
		return 0;

	return (rv);
}

void
com_supio_attach(parent, self, aux)
	struct device *parent, *self;
	void *aux;
{
	struct comsupio_softc *sc = (void *)self;
	struct com_softc *csc = &sc->sc_com;
	int iobase;
	bus_space_tag_t iot;
	struct supio_attach_args *supa = aux;
	u_int16_t needpsl;

	/*
	 * We're living on a superio chip.
	 */
	iobase = csc->sc_iobase = supa->supio_iobase;
	iot = csc->sc_iot = supa->supio_iot;
	printf(" port 0x%x ipl %d", iobase, supa->supio_ipl);

	if (bus_space_map(iot, iobase, COM_NPORTS, 0, &csc->sc_ioh))
		panic("comattach: io mapping failed");

	csc->sc_frequency = supa->supio_arg;

	com_attach_subr(csc);

	/* XXX this should be really in the interupt stuff */
	needpsl = PSL_S | (supa->supio_ipl << 8);

	if (amiga_serialspl < needpsl) {
		printf("%s: raising amiga_serialspl from 0x%x to 0x%x\n",
		    csc->sc_dev.dv_xname, amiga_serialspl, needpsl);
		amiga_serialspl = needpsl;
	}
	sc->sc_isr.isr_intr = comintr;
	sc->sc_isr.isr_arg = csc;
	sc->sc_isr.isr_ipl = supa->supio_ipl;
	add_isr(&sc->sc_isr);

	/*
	 * Shutdown hook for buggy BIOSs that don't recognize the UART
	 * without a disabled FIFO.
	 */
	if (shutdownhook_establish(com_supio_cleanup, csc) == NULL)
		panic("comsupio: could not establish shutdown hook");
}

void
com_supio_cleanup(arg)
	void *arg;
{
	struct com_softc *sc = arg;

	if (ISSET(sc->sc_hwflags, COM_HW_FIFO))
		bus_space_write_1(sc->sc_iot, sc->sc_ioh, com_fifo, 0);
}

redpajama_set_name: RedPajamaGithub


text:
var EventEmitter = require('events').EventEmitter;

/**
 * Roll simply keep tracks of steps' positions inside a viewport.
 * Apart from the static helper functions and the `scroll` function, a roll instance doesn't depend on DOM manipulation.
 * That means you can use a Roll instance in contexts other than DOM.
 */
export default class Roll extends EventEmitter {

  /**
   * Create a new Roll.
   * @param viewSize viewport size (single dimension)
   */
  constructor( viewSize ) {
    super();

    this.viewportSize = viewSize;
    this.paneSize = 0;

    // store the steps object {y1, y2, size, pad}, See Roll.chunk
    this.steps = [];

    this.pos = 0; // current position
    this.current = 0; // current step
    this.last = -1; // last step

    this.movingInterval = -1;
  }

  /**
   * Add a step object. You can also use Roll.chunk() static helper function to create a step object easily.
   * @param s an object with {p1, p2, size, pad} properties, or an array of steps object
   * @returns {Roll}
   */
  addStep(s) {

    if (!Array.isArray(s)) {
      s = [s];
    }

    // get last recorded step
    var d = s[0].p1;
    if (this.steps.length > 0 ) {
      var last = this.steps[this.steps.length-1];
      d = last.p2 + last.pad;
    }

    // append new steps
    for (var i=0; i<s.length; i++) {
      s[i].p1 = d;
      s[i].p2 = s[i].p1 + s[i].size;
      d = s[i].p2 + s[i].pad;
      this.steps.push( s[i] );
    }

    // recalculate pane size
    this.getHeight( true );

    return this;
  }

  /**
   * Get step by index
   * @param index
   */
  getStepAt( index ) {
    return this.steps[ Math.max( 0, Math.min( this.steps.length-1, index) )];
  }

  /**
   * Calculate and return current step. When padding > 0, step will be -1 when current progress is on the padding area. This allows you to check progress against padding.
   * @returns {number}
   */
  getStep() {
    for (var i=0; i<this.steps.length; i++) {
      var st = this.steps[i];
      if (st.p1 >= -this.viewportSize && st.p2 <= st.size ) {
        this.current = i;
        return i;
      }
    }
    return -1;
  }

  /**
   * Get current progress within the current step
   * @returns 0-1 if step.pad is 0. Otherwise it will range from negative to positive.
   */
  getStepProgress() {
    var curr = this.steps[ this.current ];
    return 1 - (curr.p2 / curr.size);
  }

  /**
   * Get current position
   * @returns {number|*}
   */
  getPosition() {
    return this.pos;
  }

  /**
   * Get total height of the pane (including padding)
   * @returns {*}
   */
  getHeight( recalc = false ) {
    if (recalc ) this.paneSize = this.steps.reduce( (a,b) => a+b.size+b.pad, 0 );
    return this.paneSize;
  }

  /**
   * Get viewport's height (same as this.viewportSize)
   * @returns {*}
   */
  getViewportHeight() {
    return this.viewportSize;
  }

  /**
   * Move the roll. This will emit two events `roll(step, currProgress, currPosition, totalProgress)` and `step(curr, last)`
   * @param pos new position
   * @returns {Roll}
   */
  move( pos ) {
    var last = this.pos;
    this.pos = -pos;
    var diff = this.pos - last;

    for (var i=0; i<this.steps.length; i++) {
      let s = this.steps[i];
      s.p1 += diff;
      s.p2 = s.p1 + s.size;
    }

    var curr = this.getStep();
    var progress = this.getStepProgress();
    this.emit("roll", curr, progress, pos, pos/(this.paneSize-this.viewportSize) );

    if (curr != this.last && curr >= 0) {
      this.emit("step", curr, this.last, this.viewportSize );
      this.last = curr;
    }

    return this;
  }

  /**
   * Animated scrolling a DOM element
   * @param index step index
   * @param scrollPane a DOM element with scrolling (overflow-y).
   * @param speed optional speed of animated scroll. Defaults to 0.1. Larger is faster
   * @param isVertical optional boolean to indicate horizontal or vertical scroll
   */
  scroll( index, scrollPane, speed=0.1, isVertical=true) {
    if (!scrollPane || scrollPane.scrollTop == null) throw "scrollPane parameter requires a DOM element with scrollTop property";
    clearInterval( this.movingInterval );
    var _temp = Number.NEGATIVE_INFINITY;
    var dir = (isVertical) ? "scrollTop" : "scrollLeft";

    this.movingInterval = setInterval( () => {
      var target = this.getStepAt(index);
      var d = (target.p1 + target.size/4) * speed;
      scrollPane[dir] += d;
      if (Math.abs(d)<1 || _temp === scrollPane[dir]) clearInterval(this.movingInterval);
      _temp = scrollPane[dir];
    }, 17);
  }

  /**
   * A convenient static function to create a step object
   * @param size chunk size
   * @param pad optional padding (default to 0)
   * @returns {{p1: number, p2: *, size: *, pad: number}}
   */
  static chunk( size, pad=0) {
    return {
      p1: 0,
      p2: size,
      size: size,
      pad: pad
    }
  }

  /**
   * A convenient static function to compare a step with current step, and transform it to a name
   * @param step the step to check
   * @param currStep current step
   * @param prev optional class name for step is < currStep. Defaults to "prev"
   * @param next optional class name for step is > currStep. Defaults to "next"
   * @param match optional class name for step = currStep. Defaults to "curr"
   * @returns {string}
   */
  static stepName( step, currStep, prev="prev", next="next", match="curr") {
    return (step === currStep) ? match : ( (step < currStep) ? prev : next );
  }

  /**
   * Static helper to get a handle function for Roll's "step" event. The handler function will add class names to each step element based on current step value.
   * @param roll a Roll instance
   * @param views a list of DOM elements which are the steps
   * @param prev optional class name for step is < currStep. Defaults to "prev"
   * @param next optional class name for step is > currStep. Defaults to "next"
   * @param match optional class name for step = currStep. Defaults to "curr"
   * @returns {Function}
   */
  static stepHandler( roll, views, prev="prev", next="next", match="curr", trackTopPos=false) {
    return function ( curr, last, viewportHeight ) {
      for (var i = 0; i < roll.steps.length; i++) {
        var cls = Roll.stepName( i, curr, prev, next, match );
        views[i].className = "step " + cls;

        // if steps have different sizes, recalc top position and set style
        if (trackTopPos) {
          var p = (cls===prev) ?  roll.steps[i].size * -1 : ((cls===next) ? viewportHeight : 0);
          views[i].style.top = p+"px";
        }
      }
    }
  }

  /**
   * Static method to create a Roll instance with DOM elements
   * @param viewPortID id of viewport element, which is the parent of the viewPane. eg, "#viewport"
   * @param viewPaneID id of view pane element, eg, "#pane"
   * @param viewBox id of view box element, which is the parent the viewClass elements. eg, "#steps"
   * @param viewClass id of each step or slide element, eg, ".step"
   * @param pad optional padding between steps. Defaults to 0.
   * @returns the roll instance which you can listen for "step" and "roll" event via `roll.on(...)`
   */
  static DOM( viewPortID, viewPaneID, viewBoxID, viewClass, pad=0 ) {

    var viewport = document.querySelector( viewPortID );
    var viewpane = viewport.querySelector( viewPaneID );
    var viewbox = document.querySelector( viewBoxID );
    var views = viewbox.querySelectorAll( viewClass );

    if (!viewport || !viewpane) throw `Cannot find ${viewPortID} or ${viewPaneID} element id.`
    if (!viewClass) throw `Cannot find ${viewClass} element class name`;

    // create roll instance based on viewport element height
    var roll = new Roll( viewport.getBoundingClientRect().height );

    // add each viewClass element as a step
    for (var i = 0; i < views.length; i++) {
      var rect = views[i].getBoundingClientRect();
      roll.addStep( Roll.chunk( rect.height, pad ) );
    }

    // update viewpane height based on steps
    viewpane.style.height = roll.getHeight()+"px";

    // update viewbox width to account for scrollbar
    viewbox.style.width = viewpane.getBoundingClientRect().width+"px";

    // track scroll
    viewport.addEventListener("scroll", function(evt) {
      roll.move( viewport.scrollTop );
    });

    return roll;
  }

}


text:
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
require File.expand_path(File.dirname(__FILE__) + '/mock_app_helper')

describe "Response value verbs" do
  before do
  end

  it 'can halt with status code 404' do
    mock_app {
      use Rack::Block do
        bot_access { path('*'){ halt 404 } }
      end
      run DEFAULT_APP
    }

    header "User-Agent", "Googlebot"
    get '/'
    last_response.should be_not_found
  end

  it 'can halt with status code 500' do
    mock_app {
      use Rack::Block do
        bot_access { path('*'){ halt 500 } }
      end
      run DEFAULT_APP
    }

    header "User-Agent", "Googlebot"
    get '/'
    last_response.status.should eq(500)
  end

  it 'can redirect' do
    mock_app {
      use Rack::Block do
        bot_access { path('*'){ redirect 'http://www.google.com/' } }
      end
      run DEFAULT_APP
    }

    header "User-Agent", "Googlebot"
    get '/'
    last_response.should be_redirect
  end

  it 'can redirect internal' do
    mock_app {
      use Rack::Block do
        bot_access { path('/foo/*'){ redirect '/' } }
      end
      run DEFAULT_APP
    }

    header "User-Agent", "Googlebot"
    get '/foo/bar'
    last_response.should be_redirect

    follow_redirect!
    last_response.body.should match /It is summer/
  end

  context 'doubling application' do
    require 'sinatra/base'
    if RUBY_VERSION >= '1.9'
      let :dummy do
        Class.new(Sinatra::Base) do
          get '/' do
            "This is a dummy access to: #{request.path_info}"
          end
        end
      end
    else
      let(:dummy) { DummySinatra }
    end

    it 'can point access to a dummy application' do
      dummy = dummy()
      mock_app {
        use Rack::Block do
          bot_access do
            path('*') do
              dummy_app { dummy.new }
            end
          end
        end
        run DEFAULT_APP
      }

      header "User-Agent", "Googlebot"
      get '/'
      last_response.body.should match /This is a dummy access to: \//

      get '/not-found'
      last_response.should be_not_found
    end

    it 'should be called with env' do
      dummy = dummy()
      mock_app {
        use Rack::Block do
          bot_access do
            path('*') do
              double do |env|
                # Sinatra::Base and its subclasses have a #call method in his class methods
                dummy.call(env)
              end
            end
          end
        end
        run DEFAULT_APP
      }

      header "User-Agent", "Googlebot"
      get '/'
      last_response.body.should match /This is a dummy access to: \//
    end

    it 'should be aliased by #double' do
      dummy = dummy()
      mock_app {
        use Rack::Block do
          bot_access do
            path('*') do
              double { dummy.new }
            end
          end
        end
        run DEFAULT_APP
      }

      header "User-Agent", "Googlebot"
      get '/'
      last_response.body.should match /This is a dummy access to: \//
    end
  end
end


text:
import re
#from ast import Value, Op, Var, Block, Prog, Func, Assign, Tuple, Call, Index, Branch, Loop, If, Switch, Case, Import, Proto, ObjectLiteral, ForLoop, Format, ListLiteral
import ast
from typing import *
from runtime import EspString, EspNone, EspDict, EspList

import common

def unreachable(msg=None):
	if msg:
		raise AssertionError(msg)
	raise AssertionError()

class ParseError(RuntimeError):
	def __init__(self, msg, pre, p, l, c):
		super().__init__(f"{msg} ({l}|{c}:{p})\n{pre}")

		self.pos = p
		self.line = l
		self.col = c

KW = [
	"if", "then", "else", "loop", "while", "do", "for",
	"with", "switch", "case", "try", "after",
	"return", "yield", "fail", "await", "break", "continue", "redo",
	"proto", "struct", "function",
	"async", "strict", "public", "private", "static", "var", "const",
	"this", "super",
	"import", "export",
	"new", "delete", "in", "is", "as", "has",
	"and", "or", "not", "xor"

	# Implemented as global constants
	#"none", "inf", "nan", "true", "false"
]

KWALIASES = {
	"or": "||",
	"and": "&&",
	"not": "!",
	"xor": "^"
}

# Token regex, organized to allow them to be indexed by name
PATS = {
	# Syntax "operators" which require special parsing rules
	"punc": r"[()\[\]{}]|\.(?!\.)|\.{3}|[,;:]|[-=]>",
	"cmp": r"[!=]={1,2}|[<>!]=?|<>",
	"assign": r"(?P<d0>[~*/%&|^:])(?P=d0)?=|=",
	"op": "|".join([ # Ops can be contextual identifiers, puncs can't
		r"[@~?]", # Misc syntax
		r"[-=]>", # Arrows
		r"\.\.",
		r"(?P<d1>[-+*/%&|^:!])(?P=d1)?", # Arithmetic and logical
		r"<<[<>]|[<>]>>" # Shifts
	]),

	"bin": r"?:0b([_01]*[01])",
	"oct": r"?:0o([_0-7]*[0-7])",
	"hex": r"?:0x([_\da-fA-F]*[\da-fA-F])",
	"flt": r"(?:\.\d[_\d]*|\d[_\d]*\.(?!\.)(?:\d[_\d]*)?)(?:e[-+]\d[_\d]*)?",
	"dec": r"\d(?:[\d_]*\d)?",
	"sq": r"?:'((?:\\.|.+?)*?)'",
	"dq": r'?:"((?:\\.|.+?)*?)"',
	"bq": r"?:`((?:\\.|.+?)*?)`",
	"sq3": r"?:'''((?:\\.|.+?)*?)'''",
	"dq3": r'?:"""((?:\\.|.+?)*?)"""',
	"bq3": r"?:```((?:\\.|.+?)*?)```",
	"id": r"[\w_][\w\d]*"
}
# Operators which can't be overloaded because they encode syntax
SYNTACTIC = {
	",", "=", "...", ".", ":", "&&", "||", "===", "!==", "++", "--", "=>", "->"
}

# Each operator is its own precedence class
def separate(*v):
	return [[x] for x in v]

def build_precs(precs):
	p = 0
	for ops in precs:
		for op in ops:
			yield op, p
		p += 1

PRECS = dict(build_precs([
	# Loosest binding strength
	[';'],
	['=>'],
	['='],
	[',', ":"],
	['..', '...'],
	['()', '[]', "{}"],
	["==", "!=", "===", "!=="],
	["<", "<=", ">", ">=", "<>"],
	*separate("||", "^^", "&&"),
	*separate("|", "^", "&"),
	["<<", "<<<", "<<>", "<>>", ">>>", ">>"],
	*separate("!", "~"),
	["+", "-"], # Additive
	["*", "/", "%"], # Multiplicative
	["**", "//", "%%"], # Exponential
	["++", "--"],
	['.', '->', "::"],
	[":"]
	# Tightest binding strength
]))

# Unary operators can be prefix or postfix, but cannot be infix. All binary
#  operators can be prefix unary operators
UNARY = {
	"!", "?", "++", "--", "..."
}

# Right-associative operators
RIGHT = {
	"**"
}

RADIX = {
	"bin": 2,
	"oct": 8,
	"dec": 10,
	"hex": 16
}

NAMES = list(PATS.keys())
TOKEN = re.compile(f"({')|('.join(PATS.values())})", re.M)
COMMENT = re.compile(r"\s*(#\*(?:.|\n)+\*#\s*|#.*$)", re.M)
NEWLINE = re.compile("\n")
SPACE = re.compile(r"\s+", re.M)
WORD = re.compile(r'\S+')

class Token:
	def __init__(self, src, v, t, p, l, c):
		self.src = src
		self.value = v
		self.type = t
		self.pos = p
		self.line = l
		self.col = c

		# Flag indicating if this token was an operator followed by the =
		#  meta-operator. Without this, = has the lowest precedence so is
		#  likely to be less than min_prec, but the = will have already been
		#  consumed.
		self.assign = False

	def __str__(self):
		return f"{self.type}:{self.value}"

	def __repr__(self):
		return f"Token({self.type!r}, {self.value!r})"

	def __eq__(self, other):
		return self.p == other.p

def semiAsExpr(block):
	'''
	Convert a list of expressions to a single expression, possibly a Block
	'''
	block = list(filter(lambda x: x is not None, block))

	if len(block) == 0:
		return ast.Value(None)
	elif len(block) == 1:
		return block[0]
	else:
		return ast.Block(block)

# Context-less stream of tokens to be fed to the parser
class Lexer:
	def __init__(self, src):
		self.src = src
		self.pos = 0
		self.line = 1
		self.col = 0
		self.indent = 0

		self.cur = None

	def log(self, *args, **kwargs):
		print(' '*self.indent, *args, **kwargs)

	def error(self, tok, msg=None):
		'''
		Build an error based on the current parser state. Tok is the Token
		object which caused the error, or None if it's a parsing error

		tok is optional, if the function is called with only one parameter
		it's assumed to be None
		'''

		if msg is None:
			tok, msg = None, tok

		src = tok or self

		pos, line, col = src.pos, src.line, src.col

		# Cutoff at 80 chars or the end of the line
		pre = self.src.split('\n')[line - 1].replace('\t', ' ')
		# Point to the error
		pre += f"\n{'-'*(col)}^"

		# DEBUG
		#pre += f"\n{self.src[pos:]}"

		return ParseError(msg, pre, pos, line, col)

	def match(self, reg):
		'''Utility to match arbitrary regexes at the current index'''
		return reg.match(self.src, self.pos)

	# Adjust the parser position based on the token match
	def repos(self, m):
		'''Reposition the parser based on the token match'''

		self.pos += len(m[0])
		lines = NEWLINE.split(m[0])
		if len(lines) > 1:
			self.line += len(lines) - 1
			self.col = len(lines[-1])
		else:
			self.col += len(m[0])

	def next(self):
		'''Get the next token after discarding whitespace and comments'''

		if self.pos >= len(self.src):
			return None

		# Skip all comments
		while True:
			comment = self.match(COMMENT)
			if comment is None: break

			self.repos(comment)

		# Clean up ordinary space without comments
		m = self.match(SPACE)
		if m is not None:
			self.repos(m)

		# If the token regex doesn't match, we've run out
		m = self.match(TOKEN)
		if m is None:
			return None

		# Find the first match to determine which token type it is
		g = m.groups()

		# Named matches are included in .groups() so we need to explicitly
		#  skip them - they're 1-indexed
		skip = [x - 1 for x in TOKEN.groupindex.values()]

		x = 0
		for i in range(len(g)):
			if i not in skip:
				val = g[i]
				if val is not None:
					break
				x += 1

		try:
			tt = NAMES[x]
		except IndexError:
			# Common parsing issue, give us more context
			print(x, m.groups())
			raise

		if tt in RADIX:
			r = RADIX[tt]
			tt = "val"
			val = int(val.replace("_", ""), r)
		elif tt == "flt":
			tt = "val"
			val = float(val.replace("_", ""))
		elif tt in {"sq", "dq", "bq", "sq3", "dq3", "bq3"}:
			tt = "str"
			val = EspString(val)
		elif tt == "id":
			if val in KW:
				tt = "kw"

				if val in KWALIASES:
					tt = "op"
					val = KWALIASES[val]

		# Tokens with no special postprocessing
		elif tt in {"op", "cmp", "assign", "punc"}:
			pass
		else:
			raise self.error(f"Unimplemented token {tt}:{val}")

		ncur = Token(self.src, val, tt, self.pos, self.line, self.col)

		self.repos(m)

		return ncur

	def consume(self):
		'''Unconditionally consume the token and return the next one'''

		self.cur = self.next()
		return self.cur

	def peek(self, value=None, type=None) -> Union[Token, None]:
		'''
		Peek at the next token without consuming it, returning the token or None

		You can check for either value or token or both, if they're both None
		then it returns self.cur
		'''

		cur = self.cur
		if cur is None or value is None and type is None:
			return None

		# if set, check if it's in the set. Otherwise check equality
		def in_or_eq(value, desc):
			if desc is not None:
				if isinstance(desc, set):
					return value in desc
				else:
					return value == desc
			else:
				return True

		if in_or_eq(cur.value, value) and in_or_eq(cur.type, type):
			return self.cur
		else:
			return None

	def maybe(self, value=None, type=None):
		'''
		Consume the next token if it matches the string, returning the token,
		else return None
		'''

		v = self.peek(value, type)
		if v:
			self.consume()
		return v

	def expect(self, value=None, type=None):
		'''Next token must be the string or it errors'''

		m = self.maybe(value, type)
		if m is None:
			raise self.expected(value or type)
		return m

	def expected(self, what):
		return self.error(self.cur, f"Expected {what}, got {self.cur}")

class Parser(Lexer):
	def __init__(self, src):
		super().__init__(src)
		self.scope = []
		self.consume()

	def addVar(self, var):
		assert(ast.is_expr(var))
		self.scope[-1].append(var)

	def lvalue(self):
		'''Ensure the next element is an l-value'''

		x = self.atom()
		if x is None:
			return None

		if x.lvalue:
			return x
		else:
			raise self.error(x.origin, "Not an l-value")

	def relaxid(self):
		'''
		Parse an identifier with relaxed restrictions. Most token types are
		 converted to a corresponding identifier, eg "+"
		'''

		tok = self.maybe(type={"id", "kw", "op", "cmp", "assign", "str"})
		if not tok:
			return None

		if tok.type == "str":
			return self.process_string(tok)
		else:
			return ast.Value(tok.value).set_origin(tok)

	def listing(self, sep, pat):
		v = []
		x = pat()
		while x:
			v.append(x)
			if not self.maybe(sep):
				break

			x = pat()

		return v

	def qualifier(self):
		return self.maybe({"var", "const"})

	def funcargs(self):
		args = self.lvalue()
		if type(args) is ast.Tuple:
			return args.elems
		else:
			return [args]

	def condition(self):
		if self.maybe("("):
			ex = self.expr()
			self.expect(")")
			return ex
		else:
			return self.expr()

	def parse_objectliteral(self, cur):
		obj = []

		if self.maybe("}"):
			return ast.Value(cur, EspDict())

		while not self.maybe("}"):
			tok = self.cur
			if tok.type in {"val", "id", "op"}:
				key = ast.Value(tok.value)
			elif tok.type == "kw":
				if tok.value in {"get", "set"}:
					raise NotImplementedError("get/set")
				key = ast.Value(tok.value)
			elif tok.type == "punc":
				if tok.value == "[":
					raise NotImplementedError("Computed properties")
				raise self.error(tok, "Unexpected punctuation")
			else:
				raise self.error(tok, f"Unknown token {tok!r}")

			key = key.set_origin(tok)

			self.consume()

			if self.peek("("):
				# Object method
				functok = self.cur
				args = self.funcargs()
				value = ast.Func(key, args, self.block()).set_origin(functok)
			elif self.maybe(":"):
				# Normal property
				value = self.expr()
			else:
				# NOTE: Doesn't work for computed properties
				value = ast.Var(key.value)

			obj.append((key, value))

			# Should commas be optional?
			self.maybe(",")

		return ast.ObjectLiteral(obj).set_origin(cur)

	def parse_proto(self, cur):
		# TODO: anonymous proto

		name = self.maybe(type="id")

		if self.maybe("is"):
			parent = self.relaxid()
		else:
			parent = None

		self.expect("{")

		pub = []
		priv = []
		stat = []

		while not self.maybe("}"):
			qual = self.maybe({"public", "var", "private", "static"})
			member = self.maybe(type="id")

			if self.maybe("("):
				params = self.listing(",", self.lvalue)
				self.expect(")")

				body = self.block()

				members = [ast.Func(member.value, params, body).set_origin(member)]
			else:
				members = [ast.Var(member.value).set_origin(member)]

				while self.maybe(","):
					mem = self.maybe(type="id")
					if mem is None:
						raise RuntimeError("None name")

					members.append(ast.Var(mem, mem.value))

			if not qual or qual.value in {"public", "var"}:
				pub += members
			elif qual.value == "private":
				priv += members
			elif qual.value == "static":
				stat += members
			else:
				raise RuntimeError("Shouldn't happen")

			while self.maybe(";"): pass

		name = name.value if name else None

		p = ast.Proto(name, parent, pub, priv, stat).set_origin(cur)

		if name is None:
			return p
		else:
			v = ast.Var(name, mutable=False)
			self.addVar(v)
			return ast.Assign(cur, v, p)

	def parse_while(self):
		# "while" token already consumed
		cond = self.condition()
		bl = self.block()

		th, el = self.then_else()

		return cond, bl, th, el

	def parse_switch(self, cur):
		swtok = cur

		ex = self.condition()

		# List of cases clauses which will be linked after the
		#  parsing stage
		cs = []

		# Then, else, and default
		th = el = de = None

		# First, collect all the cases as separate blocks

		self.expect("{")
		while not self.maybe("}"):
			# Token of the case/else
			cur = self.cur

			# case or else clause?

			if self.maybe("case"):
				op = "in" if self.maybe("in") else "="
				val = self.expr()
			elif self.maybe("else"):
				op = "else"
				val = None
			else:
				raise self.expected("case or else")

			# Immediate or fallthrough?

			if self.maybe("=>"): ln = None
			elif self.maybe(":"): ln = True
			else:
				raise self.expected(": or =>")

			# Block of the clause

			bl = self.block()
			c = ast.Case(op, val, bl, ln).set_origin(cur)
			if op == "else":
				if de:
					raise self.error(cur, "Duplicate else case")
				de = c

			cs.append(c)

		# Process follow up blocks

		th, el = self.then_else()

		# Link adjacent fallthrough blocks together

		update = None

		for case in cs:
			if update:
				update.next = case

			update = None
			if case.next:
				update = case

		# Remove else-case from the case list
		if de in cs:
			cs.remove(de)

		return ast.Switch(ex, cs, de, th, el).set_origin(swtok)

	def then_else(self):
		th = el = None

		if self.maybe("then"):
			th = self.block()
		if self.maybe("else"):
			el = self.block()

		return th, el

	def process_string(self, cur):
		val = cur.value

		# Lazy hack, there are edge cases where this would be incorrect
		val = val.replace("\\n", "\n").replace("\\t", "\t")

		# Scan for format strings
		parts = []
		scan = val
		upto = 0
		while True:
			x = scan.find("\\{", upto)
			if x != -1:
				y = scan.find("}", x)
				if upto != x:
					parts.append(ast.Value(scan[upto:x]))
				sp = Parser(scan[x + 2:y]).parse()
				parts.append(semiAsExpr(sp.elems))
				upto = y + 1
			else:
				break

		if upto < len(val):
			parts.append(ast.Value(val[upto:]))

		if len(parts) == 1:
			return parts[0].set_origin(cur)
		else:
			return ast.Format(parts).set_origin(cur)

	def parse_if(self, cur):
		cond = self.condition()
		self.maybe("then")

		th = self.block()
		el = self.maybe("else") and self.block()

		return ast.If(cond, th, el).set_origin(cur)

	def parse_import(self, cur):
		return ast.Import(self.expr()).set_origin(cur)

	def parse_listliteral(self, cur):
		if self.maybe("]"):
			return ast.Value(EspList()).set_origin(cur)

		vals = []
		while True:
			vals.append(self.expr())
			if not self.maybe(","):
				break
		self.expect("]")

		return ast.ListLiteral(vals).set_origin(cur)

	def parse_unary(self, cur):
		'''Prefix unary'''
		op = cur.value
		prec = PRECS[op]

		rhs = self.expr(prec)

		# Need to convert these to assignment
		if op in {"--", "++"}:
			val = ast.Assign(rhs, ast.Value(1), op[0])
		else:
			val = ast.Op(op, ast.Value(None), rhs)

		return val.set_origin(cur)

	def parse_loop(self, cur):
		always = self.block()

		if self.maybe("while"):
			cond, bl, th, el = self.parse_while()

			return ast.Loop(ast.Block([
				always, ast.If(cond, bl, semiAsExpr([th, ast.Branch("break")]))
			]), el=el).set_origin(cur)
		else:
			return ast.Loop(always).set_origin(cur)

	def parse_for(self, cur):
		self.expect("(")
		qual = self.qualifier()
		itvar = self.parse_declvar(qual)
		self.expect("in")
		toiter = self.expr()
		self.expect(")")
		body = self.block()

		th, el = self.then_else()

		# TODO: n is a generalized lvalue, not a Var
		return ast.ForLoop(itvar, toiter, body, th, el).set_origin(cur)

	def parse_declvar(self, mut):
		name = self.expect(type="id")
		v = ast.Var(name.value, mut).set_origin(name)
		self.addVar(v)

		return v

	def parse_declgroup(self, cur):
		'''
		Var declarations are split into two separate concerns,
		 variable name hoisting to the innermost enclosing
		 scope and assignment of the variable at the correct
		 point (by returning assignments as a group of
		 assignment s-exprs)
		'''

		mut = (cur.value != "const")

		group = []
		while True:
			v = self.parse_declvar(mut)

			if self.maybe("="):
				x = self.expr()
				if x is None:
					raise self.expected("expression")

				group.append(ast.Assign(v, x).set_origin(cur))

			if not self.maybe(","):
				break

		return group

	def parse_funcliteral(self, cur):
		name = None
		if self.cur.type == "id":
			name = ast.Value(self.cur.value)
			self.consume()

		args = self.funcargs()

		block = self.block()

		func = ast.Func(name, args, block).set_origin(cur)

		if name:
			v = ast.Var(str(name.value))
			self.addVar(v)
			return ast.Assign(v, func).set_origin(cur)
		else:
			return func

	def atom(self):
		if self.cur is None:
			return None

		cur = self.cur
		val = cur.value
		ct = cur.type

		# Exceptions which shouldn't be consumed
		if ct == "punc" and val in {")", ";", "}", "]"}:
			return None
		elif ct == "kw" and val in {"case", "else"}:
			return None

		self.consume()

		if ct == "val":
			return ast.Value(cur.value).set_origin(cur)
		elif ct == "str":
			return self.process_string(cur)

		elif ct == "id":
			v = ast.Var(cur.value).set_origin(cur)

			# Check string call
			s = self.maybe(type="str")
			if s:
				return ast.Call(v, [self.process_string(s)]).set_origin(s)

			return v
		elif ct == "punc":
			if val == "(":
				if self.maybe(")"):
					return ast.Tuple([]).set_origin(cur)

				x = self.semichain()
				self.expect(")")
				return semiAsExpr(x)
			elif val == "{":
				return self.parse_objectliteral(cur)
			elif val == "[":
				return self.parse_listliteral(cur)
			#elif val == ")": pass
			#elif val == "}": pass
			# TODO: dot functions
			else:
				raise NotImplementedError(val)
		elif ct == "op":
			return self.parse_unary(cur)
		elif ct == "kw":
			if val == "if":
				return self.parse_if(cur)
			elif val == "proto":
				return self.parse_proto(cur)

			elif val == "import":
				return self.parse_import(cur)
			elif val == "loop":
				return self.parse_loop(cur)

			elif val == "while":
				cond, bl, th, el = self.parse_while()

				# Not account for el
				return ast.Loop(ast.Block([
					ast.If(cond, bl, ast.Branch("break"))
				]), el=el).set_origin(cur)

			elif val == "for":
				return self.parse_for(cur)

			elif val == "switch":
				return self.parse_switch(cur)

			elif val in {"break", "continue", "redo"}:
				return ast.Branch(val).set_origin(cur)
				# Todo: targeted branches

			elif val in {"var", "const"}:
				return semiAsExpr(self.parse_declgroup(cur))

			elif val == "function":
				return self.parse_funcliteral(cur)

			elif val == "try":
				raise NotImplementedError("try")

			else:
				raise self.error(f"Unimplemented keyword {val}")

		raise self.error(f"Unknown token {val}")

	def postfix_unary(self):
		lhs = self.atom()

		while True:
			op = self.peek(type="op")

			if op and op.value in UNARY:
				self.consume()
				lhs = ast.Op([lhs, EspNone]).set_origin(op)
			else:
				break

		return lhs

	def block(self):
		'''
		Parses a "block", which is an expression which prioritizes curly-brace
		blocks over object literals and considers the semicolon to end the
		expression rather than continue it.
		'''

		cur = self.cur
		if self.maybe("{"):
			vars = []
			with common.stack(self.scope, vars):
				b = self.semichain()
			self.expect("}")
			if len(vars) > 0:
				return ast.Block(b, vars).set_origin(cur)
			else:
				return semiAsExpr(b)
		else:
			x = self.expr()
			self.maybe(";")
			return x

	def accessexpr(self, lhs, min_prec):
		'''
		Accessor operators have special parsing requirements because they
		 allow relaxed ids
		'''
		while self.cur:
			cur = self.cur
			op = cur.value

			if not self.peek({".", "->", "::"}):
				break

			prec = PRECS[op]
			if prec < min_prec:
				break

			self.consume()
			rhs = self.relaxid()
			if not rhs:
				raise self.expected("Relaxed identifier")
			# Associativity logic is inverted because we parse rhs first
			rhs = self.accessexpr(rhs, prec + 1)
			if not rhs:
				raise self.expected("rhs")

			if op == ".":
				lhs = ast.Index(lhs, [rhs]).set_origin(cur)
			elif op == "->":
				lhs = ast.Bind(lhs, rhs).set_origin(cur)
			elif op == "::":
				lhs = ast.Descope(lhs, rhs).set_origin(cur)
			else:
				unreachable()

		return lhs

	def expr(self, min_prec=0):
		'''
		Uses precedence climbing
		'''

		lhs = self.atom()

		while self.cur:
			cur = self.cur
			# Calls and indexing are functionally equivalent to binary
			# operators with a parenthesized rhs
			if self.maybe("("):
				args = self.listing(",", self.expr)
				self.expect(")")
				lhs = ast.Call(lhs, args).set_origin(cur)

			elif self.maybe("["):
				args = self.listing(":", self.expr)
				self.expect("]")
				lhs = ast.Index(lhs, args).set_origin(cur)

			# Accessor operators require special parsing because they allow
			#  relaxed identifiers
			elif self.peek({".", "->", "::"}):
				prec = PRECS[cur.value]
				if prec < min_prec:
					break

				# Don't consume the token here, accessexpr will do it
				lhs = self.accessexpr(lhs, prec)

			# Postfix operators
			elif self.peek({"++", "--"}):
				# Disable for continuation statements
				if type(lhs) in {ast.If, ast.Loop}:
					break

				self.consume()
				lhs = ast.After(
					lhs, ast.Assign(lhs, ast.Value(1), "+")
				).set_origin(cur)

			else:
				op = cur.value
				assign = False

				if self.peek(type="assign"):
					op = op[:-1]
					assign = True
				elif self.peek(type={"op", "cmp"}):
					pass
				else:
					break

				prec = PRECS['=' if assign else op]
				if prec < min_prec:
					break

				self.consume()

				rhs = self.expr(prec + (op not in RIGHT))
				if not rhs:
					break

				if assign:
					lhs = ast.Assign(lhs, rhs, op).set_origin(cur)
				else:
					if op == ",":
						if type(lhs) is ast.Tuple:
							lhs.append(rhs)
						else:
							lhs = ast.Tuple([lhs, rhs])
					elif op == ".":
						lhs = ast.Index(lhs, [rhs])
					else:
						lhs = ast.Op(op, lhs, rhs)

		return lhs

	def semichain(self):
		'''
		Parse a block as a chain of expressions possibly separated by
	 	semicolons.
		'''

		st = []
		while self.cur is not None:
			cur = self.cur
			if self.maybe({"var", "const"}, type="kw"):
				st += self.parse_declgroup(cur)
			else:
				x = self.expr()
				if x is None:
					break
				st.append(x)

			while self.maybe(";"):
				pass

		return st

	def parse(self):
		vars = []
		with common.stack(self.scope, vars):
			return ast.Prog(self.semichain(), vars)

# Debug stuff

'''
def indentify(method):
	def wrap(self, *args, **kwargs):
		p = ', '.join(repr(x) for x in args) if args else ''
		k = ', '.join(f"{x}={y!r}" for x,y in kwargs.items()) if kwargs else ''
		pk = f"{p!r}, {k!r}" if p and k else p + k
		self.log(f"{method.__name__}({pk})")
		self.indent += 1
		tmp = method(self, *args, **kwargs)
		self.indent -= 1
		return tmp
	return wrap

for name in dir(Parser):
	if not name.startswith("__") and name not in {"log", "match", "next", "repos"}:
		member = getattr(Parser, name)
		if isinstance(member, type(Parser.parse)):
			setattr(Parser, name, indentify(member))
'''


text:
import os
from models import Advertisement, Municipality, ObjectType
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sqlalchemy import create_engine
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.linear_model import LassoLarsCV, Ridge, RidgeCV, LassoCV, Lasso, LinearRegression, LogisticRegression

from sklearn.naive_bayes import MultinomialNB
from sqlalchemy.orm import sessionmaker, defer, load_only, aliased
from sqlalchemy.sql import select
from sqlalchemy.sql.expression import join
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import SGDClassifier, SGDRegressor
import re
import ast
import json
import nltk
import sys
from nltk.corpus import stopwords # Import the stop word list
from nltk.stem import SnowballStemmer
from jupyter import statistics, plot

OBJECT_TYPES = {'description': str, 'desc': str, 'groupint': str, 'characteristics': object}
import pdb

def select_from_database(url=os.environ.get('DATABASE_URL', None)):
    engine = create_engine(os.environ.get('DATABASE_URL', None))
    Session = sessionmaker(bind=engine)
    session = Session()
    ot = aliased(ObjectType)
    ad = aliased(Advertisement)
    s = select([ad.price_brutto.label('price'),
                ad.description.label('description'),
                ad.living_area.label('living_area'),
                ad.characteristics.label('characteristics'),
                # ad.additional_data.label('additional_data'),  # Not much important stuff some distances
                ot.grouping.label('grouping'),
                #Municipality.canton_id.label('canton')
                # ended.quarter.label('equarter'),
               ]
              ).select_from(
                  join(ad, Municipality, isouter=False)
                  .join(ot, ad.object_types_id==ot.id, isouter=False)
                  #.join(ended, Ad.ended_id == ended.id, isouter=False)
              ).where(ad.build_year >= 1000)
    return pd.read_sql_query(s, session.bind)

def select_from_file(file):
    return pd.read_csv(file, index_col=0, engine='c', dtype=OBJECT_TYPES)

def detect_language(text):
    """ detect the language by the text where
    the most stopwords for a language are found
    """
    languages_ratios = {}
    tokens = nltk.wordpunct_tokenize(text)
    words_set = set([word.lower() for word in tokens])
    for language in ['german', 'french', 'italian']:
        stopwords_set = set(stopwords.words(language))
        languages_ratios[language] = len(words_set.intersection(stopwords_set)) # language "score"

    return max(languages_ratios, key=languages_ratios.get)

def stemm_words(row):
    language = detect_language(row.description)
    #letters_only = re.sub("[^a-z0-9üäöèéàêâ]", " ", str(row.desc.lower()))
    letters_only = row.description.split()
    stops = set(stopwords.words(language))
    meaningful_words = [w for w in letters_only if not w in stops]
    stemmer = SnowballStemmer(language)
    stemmed_words = [stemmer.stem(w) for w in meaningful_words]
    return " ".join(stemmed_words)

def category_ads(low, high):
    def return_func(row):
        if row.price > high:
            return 'high'

        if row.price <= high and row.price > low:
            return 'normal'

        return 'budget'
    return return_func

    # def to_words(self, desc):
    #     letters_only = re.sub("[^a-zA-Z]", " ", str(desc))
    #     words = letters_only.lower().split()
    #     stops = set(stopwords.words("german"))
    #     meaningful_words = [w for w in words if not w in stops]
    #     return( " ".join( meaningful_words ))

def ape(y_true, y_pred):
    return np.abs(y_true - y_pred) / y_true

def mape(y_true, y_pred):
    return ape(y_true, y_pred).mean()

def mdape(y_true, y_pred):
    return np.median(ape(y_true, y_pred))

def statistics(y, pred):
    diff = np.fabs(y - pred)
    print("             R²-Score: {:10n}".format(metrics.r2_score(y, pred)))
    print("                 MAPE: {:.3%}".format(mape(y, pred)))
    print("                MdAPE: {:.3%}".format(mdape(y, pred)))
    print("            Min error: {:10n}".format(np.amin(diff)))
    print("            Max error: {:10n}".format(np.amax(diff)))
    print("  Mean absolute error: {:10n}".format(metrics.mean_absolute_error(y, pred)))
    print("Median absolute error: {:10n}".format(metrics.median_absolute_error(y, pred)))
    print("   Mean squared error: {:10n}".format(metrics.mean_squared_error(y, pred)))

    num_elements = len(pred)
    apes = ape(y, pred)
    for i in np.arange(5, 100, 5):
        print("I {}: {}".format(i, len(np.where(apes < i/100)[0])/num_elements))

if __name__ == "__main__":
    # From database
    #advertisements = select_from_database(url=os.environ.get('DATABASE_URL', None))
    #advertisements.to_csv('description_analyse.csv', header=True, encoding='utf-8')
    # From file
    #advertisements = select_from_file('all.csv')
    #advertisements.drop(['noise_level', 'm_noise_level', 'floor', 'last_renovation_year'], axis=1, inplace=True)

    #advertisements = advertisements.dropna()  # Remove empty entries
    #advertisements.reindex()

    # Convert decription to unicode, otherwise NaN is inserted
    #advertisements['description'] = advertisements['description'].astype('U')

    #advertisements['desc'] = advertisements.apply(stemm_words, axis=1)
    #advertisements.to_csv('category_ads.csv', header=True, encoding='utf-8')
    # From file
    advertisements = select_from_file('desc_analyse.csv')
    advertisements['characteristics'] = advertisements['characteristics'].apply(ast.literal_eval)
    series = []
    bath_keys = ['Badezimmer', 'Badewannen', 'WCs', 'Duschen', 'Toiletten']
    for index, ad in advertisements.iterrows():
        if type(ad['characteristics']) == list:
            continue
        bath = {}
        for key, value in ad['characteristics'].items():
            if key in bath_keys:
                bath[key] = int(value)
                bath['living_area'] = ad['living_area']
                bath['desc'] = ad['desc']
                #bath['num_room'] = ad['num_room']
                series.append(bath)

    df = pd.DataFrame(series)

    df['bath_max'] = df[['Badezimmer', 'Badewannen', 'WCs', 'Duschen', 'Toiletten']].apply(lambda x: np.mean(x), axis=1)
    sns.lmplot('bath_max', 'living_area', data=df)
    plt.show()

    X_desc = df['desc'].astype('U').values
    y = df['bath_max'].values
    count_vect = CountVectorizer(analyzer="word", tokenizer=None, preprocessor=None, stop_words=None, max_features=5000)
    X_train_counts = count_vect.fit_transform(X_desc)

    tfidf_transformer = TfidfTransformer()
    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

    temp_df = pd.DataFrame(X_train_counts.todense())
    temp_df['living_area'] = df['living_area'].values

    X = temp_df.values

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)

    alpha = 0.006
    print("Best alpha: {}".format(alpha))

    lassoreg = Lasso(alpha=alpha, normalize=True, max_iter=1e5)
    lassoreg.fit(X_train, y_train)
    y_pred = lassoreg.predict(X_test)
    statistics(y_test, y_pred)
    plot(y_test, y_pred, show=True, plot_name="lasso")

    #  plt.figure(figsize=(10, 8))
    # g = sns.barplot(y=list(chars.keys()), x=list(chars.values()))
    # g.tick_params(axis='y', labelsize=5)

    sys.exit(0)
    for group in [('haus',(700000, 1500000)), ('wohnung', (500000, 1000000))]:

        ads = pd.DataFrame(advertisements[advertisements['grouping'] == group[0]])

        print("Read {} {} data".format(len(ads), group[0]))
        # sns.distplot(ads['price'], kde=True, bins=300, hist_kws={'alpha': 0.6})
        # plt.show()
        # print(ads.price.describe())
        # Make categories low, high
        ads['category'] = ads.apply(category_ads(group[1][0], group[1][1]), axis=1)

        X = ads['desc'].astype('U').values
        y = ads['category'].values
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5)
        count_vect = CountVectorizer(analyzer="word", tokenizer=None, preprocessor=None, stop_words=None, max_features=5000)
        X_train_counts = count_vect.fit_transform(X_train)

        tfidf_transformer = TfidfTransformer()
        X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

        #clf = RandomForestClassifier(n_estimators = 100)
        #clf = clf.fit(X_train_tfidf, y_train)

        for alpha in [1e-12, 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 3e-4, 1e-3]:  # 3e-3, 0.01, 0.03 ,0.1, 0.3, 1, 3, 10, 30
            clf = SGDClassifier(loss="log", penalty='l2', alpha=alpha, n_iter=100, random_state=42).fit(X_train_tfidf, y_train)
            #  clf = MultinomialNB().fit(X_train_tfidf, y_train)

            X_new_counts = count_vect.transform(X_test)
            X_new_tfidf = tfidf_transformer.transform(X_new_counts)
            predicted = clf.predict(X_new_tfidf)

            # for doc, category in zip(y_test, predicted):
            #     print('%r => %s' % (doc, category))
            print("RESULTS for {} with alpha: {}".format(group, alpha))
            print(np.mean(predicted == y_test))
            print(metrics.classification_report(y_test, predicted))

    # words = []
    # for i in range(0, len(ds.ads) ):
    #     # If the index is evenly divisible by 1000, print a message
    #     if (i+1)%1000 == 0:
    #         print("Review {} of {}".format(i+1, len(ds.ads)))
    #     words.append(ds.to_words(ds.ads['description'][i]))

    # ds.ads['desc'] = words
    # ds.ads.to_csv('description_analyse.csv', header=True, encoding='utf-8')


text:
package org.iban4j;

import org.iban4j.bban.BbanEntryType;
import org.iban4j.bban.BbanStructure;
import org.iban4j.bban.BbanStructureEntry;

import static org.iban4j.IbanFormatException.IbanFormatViolation.*;
/**
 * Iban Utility Class
 */
public final class IbanUtil {

    private static final int MOD = 97;
    private static final long MAX = 999999999;

    private static final int COUNTRY_CODE_INDEX = 0;
    private static final int COUNTRY_CODE_LENGTH = 2;
    private static final int CHECK_DIGIT_INDEX = COUNTRY_CODE_LENGTH;
    private static final int CHECK_DIGIT_LENGTH = 2;
    private static final int BBAN_INDEX = CHECK_DIGIT_INDEX + CHECK_DIGIT_LENGTH;

    private static final String ASSERT_UPPER_LETTERS = "[%s] must contain only upper case letters.";
    private static final String ASSERT_DIGITS_AND_LETTERS = "[%s] must contain only digits or letters.";
    private static final String ASSERT_DIGITS = "[%s] must contain only digits.";

    private IbanUtil() {
    }

    /**
     * Calculates Iban
     * Check Digit.
     *
     * @param iban string value
     * @throws IbanFormatException if iban contains invalid character.
     *
     * @return check digit as String
     */
    public static String calculateCheckDigit(final String iban) throws IbanFormatException {
        final String reformattedIban = replaceCheckDigit(iban,
                Iban.DEFAULT_CHECK_DIGIT);
        final int modResult = calculateMod(reformattedIban);
        final int checkDigitIntValue = (98 - modResult);
        final String checkDigit = Integer.toString(checkDigitIntValue);
        return checkDigitIntValue > 9 ? checkDigit : "0" + checkDigit;
    }

    /**
     * Validates iban.
     *
     * @param iban to be validated.
     * @throws IbanFormatException if iban is invalid.
     *         UnsupportedCountryException if iban's country is not supported.
     *         InvalidCheckDigitException if iban has invalid check digit.
     */
    public static void validate(final String iban) throws IbanFormatException,
            InvalidCheckDigitException, UnsupportedCountryException {
        try {
            validateEmpty(iban);
            validateCountryCode(iban);
            validateCheckDigitPresence(iban);

            final BbanStructure structure = getBbanStructure(iban);

            validateBbanLength(iban, structure);
            validateBbanEntries(iban, structure);

            validateCheckDigit(iban);
        } catch (Iban4jException e) {
            throw e;
        } catch (RuntimeException e) {
            throw new IbanFormatException(UNKNOWN, e.getMessage());
        }
    }

    /**
     * Validates iban.
     *
     * @param iban to be validated.
     * @param format to be used in validation.
     * @throws IbanFormatException if iban is invalid.
     *         UnsupportedCountryException if iban's country is not supported.
     *         InvalidCheckDigitException if iban has invalid check digit.
     */
    public static void validate(final String iban, final IbanFormat format) throws IbanFormatException,
            InvalidCheckDigitException, UnsupportedCountryException {
        switch (format) {
            case Default:
                final String ibanWithoutSpaces = iban.replace(" ", "");
                validate(ibanWithoutSpaces);
                if(!toFormattedString(ibanWithoutSpaces).equals(iban)) {
                    throw new IbanFormatException(IBAN_FORMATTING,
                            String.format("Iban must be formatted using 4 characters and space combination. " +
                                    "Instead of [%s]", iban));
                }
                break;
            default:
                validate(iban);
                break;
        }
    }

    /**
     * Checks whether country is supporting iban.
     * @param countryCode {@link org.iban4j.CountryCode}
     *
     * @return boolean true if country supports iban, false otherwise.
     */
    public static boolean isSupportedCountry(final CountryCode countryCode) {
        return BbanStructure.forCountry(countryCode) != null;
    }

    /**
     * Returns iban length for the specified country.
     *
     * @param countryCode {@link org.iban4j.CountryCode}
     * @return the length of the iban for the specified country.
     */
    public static int getIbanLength(final CountryCode countryCode) {
        final BbanStructure structure = getBbanStructure(countryCode);
        return COUNTRY_CODE_LENGTH + CHECK_DIGIT_LENGTH + structure.getBbanLength();
    }

    /**
     * Returns iban's check digit.
     *
     * @param iban String
     * @return checkDigit String
     */
    public static String getCheckDigit(final String iban) {
        return iban.substring(CHECK_DIGIT_INDEX,
                CHECK_DIGIT_INDEX + CHECK_DIGIT_LENGTH);
    }

    /**
     * Returns iban's country code.
     *
     * @param iban String
     * @return countryCode String
     */
    public static String getCountryCode(final String iban) {
        return iban.substring(COUNTRY_CODE_INDEX,
                COUNTRY_CODE_INDEX + COUNTRY_CODE_LENGTH);
    }

    /**
     * Returns iban's country code and check digit.
     *
     * @param iban String
     * @return countryCodeAndCheckDigit String
     */
    public static String getCountryCodeAndCheckDigit(final String iban) {
        return iban.substring(COUNTRY_CODE_INDEX,
                COUNTRY_CODE_INDEX + COUNTRY_CODE_LENGTH + CHECK_DIGIT_LENGTH);
    }

    /**
     * Returns iban's bban (Basic Bank Account Number).
     *
     * @param iban String
     * @return bban String
     */
    public static String getBban(final String iban) {
        return iban.substring(BBAN_INDEX);
    }

    /**
     * Returns iban's account number.
     *
     * @param iban String
     * @return accountNumber String
     */
    public static String getAccountNumber(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.account_number);
    }

    /**
     * Returns iban's bank code.
     *
     * @param iban String
     * @return bankCode String
     */
    public static String getBankCode(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.bank_code);
    }

    /**
     * Returns iban's branch code.
     *
     * @param iban String
     * @return branchCode String
     */
    static String getBranchCode(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.branch_code);
    }

    /**
     * Returns iban's national check digit.
     *
     * @param iban String
     * @return nationalCheckDigit String
     */
    static String getNationalCheckDigit(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.national_check_digit);
    }

    /**
     * Returns iban's account type.
     *
     * @param iban String
     * @return accountType String
     */
    static String getAccountType(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.account_type);
    }

    /**
     * Returns iban's owner account type.
     *
     * @param iban String
     * @return ownerAccountType String
     */
    static String getOwnerAccountType(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.owner_account_number);
    }

    /**
     * Returns iban's identification number.
     *
     * @param iban String
     * @return identificationNumber String
     */
    static String getIdentificationNumber(final String iban) {
        return extractBbanEntry(iban, BbanEntryType.identification_number);
    }

    static String calculateCheckDigit(final Iban iban) {
        return calculateCheckDigit(iban.toString());
    }

    /**
     * Returns an iban with replaced check digit.
     *
     * @param iban The iban
     * @return The iban without the check digit
     */
    static String replaceCheckDigit(final String iban, final String checkDigit) {
        return getCountryCode(iban) + checkDigit + getBban(iban);
    }

    /**
     * Returns formatted version of Iban.
     *
     * @return A string representing formatted Iban for printing.
     */
    static String toFormattedString(final String iban) {
        final StringBuilder ibanBuffer = new StringBuilder(iban);
        final int length = ibanBuffer.length();

        for (int i = 0; i < length / 4; i++) {
            ibanBuffer.insert((i + 1) * 4 + i, ' ');
        }

        return ibanBuffer.toString().trim();
    }

    private static void validateCheckDigit(final String iban) {
        if (calculateMod(iban) != 1) {
            final String checkDigit = getCheckDigit(iban);
            final String expectedCheckDigit = calculateCheckDigit(iban);
            throw new InvalidCheckDigitException(
                    checkDigit, expectedCheckDigit,
                    String.format("[%s] has invalid check digit: %s, " +
                                    "expected check digit is: %s",
                            iban, checkDigit, expectedCheckDigit));
        }
    }

    private static void validateEmpty(final String iban) {
        if(iban == null) {
            throw new IbanFormatException(IBAN_NOT_NULL,
                    "Null can't be a valid Iban.");
        }

        if(iban.length() == 0) {
            throw new IbanFormatException(IBAN_NOT_EMPTY,
                    "Empty string can't be a valid Iban.");
        }
    }

    private static void validateCountryCode(final String iban) {
        // check if iban contains 2 char country code
        if(iban.length() < COUNTRY_CODE_LENGTH) {
            throw new IbanFormatException(COUNTRY_CODE_TWO_LETTERS, iban,
                    "Iban must contain 2 char country code.");
        }

        final String countryCode = getCountryCode(iban);

        // check case sensitivity
        if(!countryCode.equals(countryCode.toUpperCase()) ||
            !Character.isLetter(countryCode.charAt(0)) ||
            !Character.isLetter(countryCode.charAt(1))) {
            throw new IbanFormatException(COUNTRY_CODE_UPPER_CASE_LETTERS, countryCode,
                    "Iban country code must contain upper case letters.");
        }

        if(CountryCode.getByCode(countryCode) == null) {
            throw new IbanFormatException(COUNTRY_CODE_EXISTS, countryCode,
                    "Iban contains non existing country code.");
        }

        // check if country is supported
        final BbanStructure structure = BbanStructure.forCountry(
                CountryCode.getByCode(countryCode));
        if (structure == null) {
            throw new UnsupportedCountryException(countryCode,
                    "Country code is not supported.");
        }
    }

    private static void validateCheckDigitPresence(final String iban) {
        // check if iban contains 2 digit check digit
        if(iban.length() < COUNTRY_CODE_LENGTH + CHECK_DIGIT_LENGTH) {
            throw new IbanFormatException(CHECK_DIGIT_TWO_DIGITS,
                    iban.substring(COUNTRY_CODE_LENGTH),
                    "Iban must contain 2 digit check digit.");
        }

        final String checkDigit = getCheckDigit(iban);

        // check digits
        if(!Character.isDigit(checkDigit.charAt(0)) ||
           !Character.isDigit(checkDigit.charAt(1))) {
            throw new IbanFormatException(CHECK_DIGIT_ONLY_DIGITS, checkDigit,
                    "Iban's check digit should contain only digits.");
        }
    }

    private static void validateBbanLength(final String iban,
                                           final BbanStructure structure) {
        final int expectedBbanLength = structure.getBbanLength();
        final String bban = getBban(iban);
        final int bbanLength = bban.length();
        if (expectedBbanLength != bbanLength) {
            throw new IbanFormatException(BBAN_LENGTH,
                    bbanLength, expectedBbanLength,
                    String.format("[%s] length is %d, expected BBAN length is: %d",
                            bban, bbanLength, expectedBbanLength));
        }
    }

    private static void validateBbanEntries(final String iban,
                                            final BbanStructure structure) {
        final String bban = getBban(iban);
        int bbanEntryOffset = 0;
        for(final BbanStructureEntry entry : structure.getEntries()) {
            final int entryLength = entry.getLength();
            final String entryValue = bban.substring(bbanEntryOffset,
                    bbanEntryOffset + entryLength);

            bbanEntryOffset = bbanEntryOffset + entryLength;

            // validate character type
            validateBbanEntryCharacterType(entry, entryValue);
        }
    }

    private static void validateBbanEntryCharacterType(final BbanStructureEntry entry,
                                                       final String entryValue) {
        switch (entry.getCharacterType()) {
            case a:
                for(char ch: entryValue.toCharArray()) {
                    if(!Character.isUpperCase(ch)) {
                        throw new IbanFormatException(BBAN_ONLY_UPPER_CASE_LETTERS,
                                entry.getEntryType(), entryValue, ch,
                                String.format(ASSERT_UPPER_LETTERS, entryValue));
                    }
                }
                break;
            case c:
                for(char ch: entryValue.toCharArray()) {
                    if(!Character.isLetterOrDigit(ch)) {
                        throw new IbanFormatException(BBAN_ONLY_DIGITS_OR_LETTERS,
                                entry.getEntryType(), entryValue, ch,
                                String.format(ASSERT_DIGITS_AND_LETTERS, entryValue));
                    }
                }
                break;
            case n:
                for(char ch: entryValue.toCharArray()) {
                    if(!Character.isDigit(ch)) {
                        throw new IbanFormatException(BBAN_ONLY_DIGITS,
                                entry.getEntryType(), entryValue, ch,
                                String.format(ASSERT_DIGITS, entryValue));
                    }
                }
                break;
        }
    }

    /**
     * Calculates
     * Iban Modulo.
     *
     * @param iban String value
     * @return modulo 97
     */
    private static int calculateMod(final String iban) {
        final String reformattedIban = getBban(iban) + getCountryCodeAndCheckDigit(iban);
        long total = 0;
        for (int i = 0; i < reformattedIban.length(); i++) {
            final int numericValue = Character.getNumericValue(reformattedIban.charAt(i));
            if (numericValue < 0 || numericValue > 35) {
                throw new IbanFormatException(IBAN_VALID_CHARACTERS, null, null,
                        reformattedIban.charAt(i),
                        String.format("Invalid Character[%d] = '%d'", i, numericValue));
            }
            total = (numericValue > 9 ? total * 100 : total * 10) + numericValue;

            if (total > MAX) {
                total = (total % MOD);
            }

        }
        return (int) (total % MOD);
    }

    private static BbanStructure getBbanStructure(final String iban) {
        final String countryCode = getCountryCode(iban);
        return getBbanStructure(CountryCode.getByCode(countryCode));
    }

    private static BbanStructure getBbanStructure(final CountryCode countryCode) {
        return BbanStructure.forCountry(countryCode);
    }

    private static String extractBbanEntry(final String iban, final BbanEntryType entryType) {
        final String bban = getBban(iban);
        final BbanStructure structure = getBbanStructure(iban);
        int bbanEntryOffset = 0;
        for(final BbanStructureEntry entry : structure.getEntries()) {
            final int entryLength = entry.getLength();
            final String entryValue = bban.substring(bbanEntryOffset,
                    bbanEntryOffset + entryLength);

            bbanEntryOffset = bbanEntryOffset + entryLength;
            if(entry.getEntryType() == entryType) {
                return entryValue;
            }
        }
        return null;
    }

}


text:
package org.motechproject.hub.web;

import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

/**
 * This class is to return the randomly generated UUID at runtime for positive
 * scenario during subscriber's intent verification
 *
 * @author Anuranjan
 *
 */
public class RestTemplateMock extends RestTemplate {

    @SuppressWarnings({ "unchecked", "rawtypes" })
    @Override
    public <T> ResponseEntity<T> exchange(String url, HttpMethod method,
            HttpEntity<?> requestEntity, Class<T> responseType,
            Object... uriVariables) throws RestClientException {
        String uuid = (String) uriVariables[2];
        return new ResponseEntity(uuid, HttpStatus.OK);
    }
}


text:
import React from 'react';
import PropTypes from 'prop-types';
import classNames from 'classnames';

const propTypes = {
    active: PropTypes.bool,
    className: PropTypes.string,
    component: PropTypes.oneOfType([
        PropTypes.string,
        PropTypes.element,
        PropTypes.func
    ]),
    cssPrefix: PropTypes.string,
    onTabClick: PropTypes.func,
    style: PropTypes.object,
    tabId: PropTypes.number
};

const defaultProps = {
    style: {}
};

const Tab = (props) => {
    const { active, className, component, children, cssPrefix,
        onTabClick, style, tabId, ...otherProps } = props;

    const classes = classNames({
        [`${cssPrefix}__tab`]: true,
        'is-active': active
    }, className);

    const finalStyle = { ...style, cursor: 'pointer' };

    return React.createElement(component || 'a', {
        className: classes,
        onClick: () => onTabClick(tabId),
        style: finalStyle,
        ...otherProps
    }, children);
};

Tab.propTypes = propTypes;
Tab.defaultProps = defaultProps;

export default Tab;


text:
<?php
$this->load->view("block/vendor_topNavigation");
$this->load->view("block/vendor_leftMenu");

$profile_info   = $profile['data'];
$bank_info      = $bank['data'];

?>

            User Profile

 Home

Examples

User profile

Nina Mcintire

Software Engineer

                                Followers 1,322

                                Following 543

                                Friends 13,287

                        Follow

About Me

                         Education

                            B.S. in Computer Science from the University of Tennessee at Knoxville

                         Location

Malibu, California

                         Skills

                            UI Design
                            Coding
                            Javascript
                            PHP
                            Node.js

                         Notes

Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam fermentum enim neque.

Personal Info

Bank Details

Reset Password

                            <form class="form-horizontal">

                                    First Name

                                        <input type="text" disabled class="form-control" id="inputFirstName" placeholder="First Name" value="<?php echo $profile_info[0]->person_first_name; ?>">

                                    Last Name

                                        <input type="text" disabled class="form-control" id="inputLastName" placeholder="Last Name" value="<?php echo $profile_info[0]->person_last_name; ?>">

                                    Email

                                        <input type="email" disabled class="form-control" id="inputEmail" placeholder="Email" value="<?php echo $profile_info[0]->person_email; ?>">

                                    Address

                                        <input type="text" disabled class="form-control" id="inputAddress" placeholder="Address" value="<?php echo $profile_info[0]->person_address; ?>">

                                    Address 2

                                        <input type="text" disabled class="form-control" id="inputAddress2" placeholder="Address" value="<?php echo $profile_info[0]->person_address1; ?>">

                                    City

                                        <input type="text" disabled class="form-control" id="inputCity" placeholder="City" value="<?php echo $profile_info[0]->person_city; ?>">

                                    State

                                        <input type="text" disabled class="form-control" id="inputSate" placeholder="State" value="<?php echo $profile_info[0]->person_state; ?>">

                                    Mobile

                                        <input type="text" disabled class="form-control" id="inputMobile" placeholder="Mobile" value="<?php echo $profile_info[0]->person_mobile; ?>">

                                    Telephone

                                        <input type="text" disabled class="form-control" id="inputTelephone" placeholder="Telephone" value="<?php echo $profile_info[0]->person_telephone; ?>">

                                    Identity Card

                                        <input type="text" disabled class="form-control" id="inputIdCard" placeholder="Identity Card" value="<?php echo $profile_info[0]->person_identity_card." - ".$profile_info[0]->person_identity_card_number; ?>">

                            </form>

                            <form class="form-horizontal" method="POST" id="bankDetailsForm">

                                            Bank Name*:

                                                <input type="text" name="bnkname" class="form-control" required id="bnkname" placeholder="Bank Name" value="<?php echo ($bank_info) ? $bank_info[0]->bank_name : '';?>">

                                            Beneficiary/Account Holder Name*:

                                                <input type="text" name="holdername" class="form-control" required id="holdername" placeholder="Beneficiary/Account Holder Name" value="<?php echo ($bank_info) ? $bank_info[0]->bank_holder_name : '';?>">

                                            Bank Account Number*:

                                                <input type="text" name="accnumber" class="form-control" required id="accnumber" placeholder="Bank Account Number" value="<?php echo ($bank_info) ? $bank_info[0]->bank_account_number : '';?>">

                                            IFSC Code/Swift Code*:

                                                <input type="text" class="form-control" name="ifsc" required id="ifsc" placeholder="IFSC Code/Swift Code" value="<?php echo ($bank_info) ? $bank_info[0]->bank_ifsc_code : '';?>">

                                            Bank Address*:

                                                <textarea type="text" class="form-control" name="bnkaddress" id="bnkaddress" required placeholder="Bank Address"><?php echo ($bank_info) ? $bank_info[0]->bank_address : '';?></textarea>

                                                <button type="submit" class="btn btn-danger" id="submitBankDetail">Submit</button>

                            </form>

                            <form class="form-horizontal">

                            </form>


text:
package org.dataportal.model;

import java.io.Serializable;
import javax.persistence.*;
import java.util.Set;

/**
 * The persistent class for the users database table.
 *
 */
@Entity
@Table(name="users")
public class User implements Serializable {
	private static final long serialVersionUID = 1L;

	@Id
	private String id;

	private String hash;

	private String password;

	private String state;

	//bi-directional many-to-one association to Download
	@OneToMany(mappedBy="userBean")
	private Set<Download> downloads;

	//bi-directional many-to-one association to Search
	@OneToMany(mappedBy="userBean")
	private Set<Search> searches;

    public User() {
    }

    public User(String id) {
        this.id = id;
    }

    public User(String id, String password) {
        this.id = id;
        this.password = password;
    }

	public String getId() {
		return this.id;
	}

	public void setId(String id) {
		this.id = id;
	}

	public String getHash() {
		return this.hash;
	}

	public void setHash(String hash) {
		this.hash = hash;
	}

	public String getPassword() {
		return this.password;
	}

	public void setPassword(String password) {
		this.password = password;
	}

	public String getState() {
		return this.state;
	}

	public void setState(String state) {
		this.state = state;
	}

	public Set<Download> getDownloads() {
		return this.downloads;
	}

	public void setDownloads(Set<Download> downloads) {
		this.downloads = downloads;
	}

	public Set<Search> getSearches() {
		return this.searches;
	}

	public void setSearches(Set<Search> searches) {
		this.searches = searches;
	}

}


text:
<?php

/**
 * Output a text at current position
 *
 * @phpstub
 *
 * @param resource $psdoc
 * @param string $text
 * @param int $len
 *
 * @return bool
 */
function ps_show2($psdoc, $text, $len)
{
}