text:
import scipy
import numpy as np
import pandas as pd

def hellopipilworld():
    return u'Niltze Cemanahuac! (Hello "World"! in Nahuatl.)'

def list2intlist(in_list, custom_order=None):
    unique_labels = pd.unique(in_list)
    translate = dict(zip(unique_labels, range(len(unique_labels))))
    if custom_order is not None:
        # print(in_list)
        new_list = [in_list[j] for j in custom_order]
        # print(new_list)
        # print([translate[i] for i in new_list])
        return [translate[i] for i in new_list]
    else:
        return [translate[i] for i in in_list]

def list2cls(in_list, name_of_out='output.cls', sep='\t'):
    """This function creates a CLS file from a list-like object"""
    # print("~~~~~"+str(metadata_subset.shape)+'~~~~~')
    cls = open(name_of_out, 'w')
    cls.write("{}{}{}{}1\n".format(len(in_list), sep, len(np.unique(in_list)), sep))
    cls.write("#{}{}\n".format(sep, sep.join(np.unique(in_list).astype(str))))
    cls.write(sep.join(in_list.astype(str))+'\n')
    cls.close()

def df2gct(df, extra_columns=0, use_index=True, name='output.gct', add_dummy_descriptions=False):
    if add_dummy_descriptions:
        if add_dummy_descriptions not in list(df):
            df.insert(0, 'Description', df.index)
            extra_columns += 1
    f = open(name, 'w')
    f.write("#1.2\n")
    f.write("\t".join([str(df.shape[0]), str(df.shape[1] - extra_columns)]) + "\n")
    f.write(df.to_csv(sep='\t', index=use_index))
    f.close()
    return

def custom_pearson_corr(x, y):
    # [-1,1]
    return scipy.stats.pearsonr(x, y)[0]

def custom_pearson_dist(x, y):
    # [0,2]
    return 1 - scipy.stats.pearsonr(x, y)[0]

def custom_spearman_corr(x, y):
    # [-1,1]
    return scipy.stats.spearmanr(x, y)[0]

def custom_spearman_dist(x, y):
    # [0,2]
    return 1 - scipy.stats.spearmanr(x, y)[0]

def absolute_spearman(x, y):
    # [0,1]
    return abs(scipy.stats.spearmanr(x, y)[0])

def custom_kendall_tau_corr(x, y):
    # [-1,1]
    return scipy.stats.kendalltau(x, y)[0]

def custom_kendall_tau_dist(x, y):
    # [0,2]
    return 1 - scipy.stats.kendalltau(x, y)[0]

def absolute_pearson_corr(x, y):
    # [0,1]
    return np.abs(scipy.stats.pearsonr(x, y)[0])

def absolute_pearson_dist(x, y):
    # [-1,0]
    return 1 - np.abs(scipy.stats.pearsonr(x, y)[0])

def uncentered_pearson_corr(x, y):
    # [-1,1]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        # x_squared =
        # y_squared =
        return np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y))))

def uncentered_pearson_dist(x, y):
    # [0,2]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        # x_squared =
        # y_squared =
        return 1 - (np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y)))))

def absolute_uncentered_pearson_corr(x, y):
    # [0,1]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        return np.abs(np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y)))))

def absolute_uncentered_pearson_dist(x, y):
    # [-1,0]
    if len(x) != len(y):
        # Uncentered Pearson Correlation cannot be computed for vectors of different length.
        print('Uncentered Pearson Correlation cannot be computed for vectors of different length.')
        return np.nan

    else:
        # Using the definition from eq (4) in https://www.biomedcentral.com/content/supplementary/1477-5956-9-30-S4.PDF
        return 1- (np.abs(np.sum(np.multiply(x, y)) / (np.sqrt(np.sum(np.square(x))) * np.sqrt(np.sum(np.square(y))))))

def mydist(p1, p2):
    # [0,inf)
    # a custom function that just computes Euclidean distance
    diff = p1 - p2
    return np.vdot(diff, diff) ** 0.5

def custom_euclidean_dist(x, y):
    # [0,inf)
    return scipy.spatial.distance.euclidean(x, y)

def custom_euclidean_sim(x, y):
    # (0,1]
    return 1/(1+scipy.spatial.distance.euclidean(x, y))

def dendodist(V, dist=mydist):
    dists = np.array([dist(a[0], a[1]) for a in V])
    return np.cumsum(dists)

def custom_manhattan_dist(x, y):
    # [0,inf)
    return scipy.spatial.distance.cityblock(x, y)

def custom_manhattan_sim(x, y):
    # (0,1]
    return 1/(1+scipy.spatial.distance.cityblock(x, y))

def custom_cosine_dist(x, y):
    # [0,2]
    return scipy.spatial.distance.cosine(x, y)

def custom_cosine_sim(x, y):
    # [-1,1]
    return 1-scipy.spatial.distance.cosine(x, y)

def makeODF(output_data, vals, file_name='noname.odf'):
    f = open(file_name, 'w')
    f.write("ODF 1.0\n")  # Hard-coding spance, not tab here.
    f.write("HeaderLines=18\n")  # hard-coding 19+2-3 lines here. Needs to change.
    # f.write("COLUMN_NAMES:Rank\tFeature\tDescription\tScore\tFeature P\tFeature P Low\tFeature P High\tFDR(BH)\tQ Value"
    #         "\tBonferroni\tmaxT\tFWER\tFold Change\tALL Mean\tALL Std\tAML Mean\tAML Std\tk\n")
    # f.write("COLUMN_TYPES:int\tString\tString\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble\tdouble"
    #         "\tdouble\tdouble\tdouble\tdouble\tdouble\tint\n")

    f.write("COLUMN_NAMES:"+"\t".join(list(output_data))+"\n")
    # f.write("COLUMN_DESCRIPTIONS:"+"\t".join(list(output_data))+"\n")
    f.write("COLUMN_TYPES:"+"\t".join(['String', 'String', 'double', 'String'])+"\n")  # TODO: automate this.
    #
    # print(list(output_data))
    # f.write("COLUMN_DESCRIPTIONS:\n")
    # f.write("Model=DiffEx\n")
    f.write("Model=Comparative Marker Selection\n")
    f.write("Dataset File="+vals['gct']+"\n")
    f.write("Class File="+vals['cls']+"\n")
    f.write("Permutations="+str(vals['n_perm'])+"\n")
    # f.write("Balanced=false\n")  # TODO: remove this one?
    # f.write("Complete=false\n")  # TODO: remove this one?
    # f.write("Test Direction=2 Sided\n")  # TODO: remove this one?
    f.write("Class 0="+vals['class_0']+"\n")
    f.write("Class 1="+vals['class_1']+"\n")
    f.write("Test Statistic="+vals['func']+"\n")
    f.write("pi0=\n")
    f.write("lambda=\n")
    f.write("pi0(lambda)=\n")
    f.write("cubic spline(lambda)=\n")
    f.write("Random Seed="+str(vals['rand_seed'])+"\n")
    f.write("Smooth p-values=\n")
    f.write("DataLines="+str(vals['dat_lines'])+"\n")
    f.write("RowNamesColumn=0\n")
    f.write("RowDescriptionsColumn=1\n")
    f.write(output_data.to_csv(sep='\t', index=False, header=False))
    return


text:
package org.snaker.designer.palette;

import java.util.ArrayList;
import java.util.List;

import org.eclipse.gef.palette.CombinedTemplateCreationEntry;
import org.eclipse.gef.palette.ConnectionCreationToolEntry;
import org.eclipse.gef.palette.MarqueeToolEntry;
import org.eclipse.gef.palette.PaletteContainer;
import org.eclipse.gef.palette.PaletteDrawer;
import org.eclipse.gef.palette.PaletteEntry;
import org.eclipse.gef.palette.PaletteGroup;
import org.eclipse.gef.palette.PaletteRoot;
import org.eclipse.gef.palette.PaletteSeparator;
import org.eclipse.gef.palette.PanningSelectionToolEntry;
import org.eclipse.gef.palette.ToolEntry;
import org.eclipse.jface.resource.ImageDescriptor;
import org.snaker.designer.config.Component;
import org.snaker.designer.config.ConfigManager;
import org.snaker.designer.figures.TaskFigure;
import org.snaker.designer.model.ModelCreationFactory;
import org.snaker.designer.model.Simple;
import org.snaker.designer.model.Transition;

/**
 * 工具板工厂类
 *
 * @author yuqs
 * @version 1.0
 */
public class PaletteFactory {
	public PaletteRoot getPaletteRoot(String type) {
		PaletteRoot palette = new PaletteRoot();
		palette.addAll(createCategories(palette, type));
		return palette;
	}

	private List<PaletteEntry> createCategories(PaletteRoot root, String type) {
		List<PaletteEntry> categories = new ArrayList<PaletteEntry>();
		categories.add(createControlGroup(root));
		if (type.equalsIgnoreCase(ConfigManager.COMPONENT_TYPE_FIELD)) {
			categories.add(createFieldDrawer());
		} else if (type.equalsIgnoreCase(ConfigManager.COMPONENT_TYPE_PROCESS)) {
			categories.add(createTaskDrawer(root));
		}
		return categories;
	}

	private PaletteContainer createTaskDrawer(PaletteRoot root) {
		String group = "流程组件";

		List<PaletteEntry> entries = new ArrayList<PaletteEntry>();
		PaletteDrawer drawer = new PaletteDrawer(group,
				ImageDescriptor.createFromFile(TaskFigure.class,
						"icons/home.png"));

		ConnectionCreationToolEntry connectionToolEntry = new ConnectionCreationToolEntry(
				"transition", "创建连接", new ModelCreationFactory(null,
						Transition.class), ImageDescriptor.createFromFile(
						TaskFigure.class, "icons/flow_sequence.png"),
				ImageDescriptor.createFromFile(TaskFigure.class,
						"icons/flow_sequence.png"));

		drawer.add(connectionToolEntry);

		List<Component> listComp = ConfigManager.getComponents();

		for (int i = 0; i < listComp.size(); i++) {
			Component comp = (Component) listComp.get(i);
			if (!validate(ConfigManager.COMPONENT_TYPE_PROCESS, comp)) {
				continue;
			}
			PaletteEntry paletteEntry = createCombinedEntry(comp);
			if (paletteEntry == null) {
				continue;
			}
			entries.add(paletteEntry);
		}

		drawer.addAll(entries);
		return drawer;
	}

	private PaletteContainer createControlGroup(PaletteRoot root) {
		PaletteGroup controlGroup = new PaletteGroup("Control Group");

		List<PaletteEntry> entries = new ArrayList<PaletteEntry>();

		ToolEntry tool = new PanningSelectionToolEntry();
		entries.add(tool);
		root.setDefaultEntry(tool);

		tool = new MarqueeToolEntry();
		entries.add(tool);

		PaletteSeparator sep = new PaletteSeparator(
				"org.snaker.designer.editor.separator");
		sep.setUserModificationPermission(1);
		entries.add(sep);

		controlGroup.addAll(entries);
		return controlGroup;
	}

	private PaletteContainer createFieldDrawer() {
		List<PaletteEntry> entries = new ArrayList<PaletteEntry>();
		String group = "表单控件";
		PaletteDrawer drawer = new PaletteDrawer(group,
				ImageDescriptor.createFromFile(TaskFigure.class,
						"icons/home.png"));

		List<Component> listComp = ConfigManager.getComponents();

		for (int i = 0; i < listComp.size(); i++) {
			Component comp = (Component) listComp.get(i);
			if (!validate(ConfigManager.COMPONENT_TYPE_FIELD, comp)) {
				continue;
			}
			PaletteEntry paletteEntry = createCombinedEntry(comp);
			if (paletteEntry == null) {
				continue;
			}
			entries.add(paletteEntry);
		}

		drawer.addAll(entries);
		drawer.setInitialState(2);
		return drawer;
	}

	private boolean validate(String groupName, Component comp) {
		if (comp.getGroup() == null || !comp.getGroup().equalsIgnoreCase(groupName)) {
			return false;
		}

		return (comp.getVisible() == null)
				|| (!"false".equalsIgnoreCase(comp.getVisible()));
	}

	private CombinedTemplateCreationEntry createCombinedEntry(
			Component compponent) {
		Class<?> clazz = null;
		try {
			clazz = Class.forName(compponent.getClazz());
		} catch (Exception e) {
			clazz = Simple.class;
		}
		if (clazz == null) return null;

		CombinedTemplateCreationEntry combined = new CombinedTemplateCreationEntry(
				compponent.getDisplayName(), compponent.getDescript(), clazz,
				new ModelCreationFactory(compponent, clazz),
				ImageDescriptor.createFromFile(TaskFigure.class,
						compponent.getIconSmall()),
				ImageDescriptor.createFromFile(TaskFigure.class,
						compponent.getIconLarge()));
		return combined;
	}
}

redpajama_set_name: RedPajamaGithub


text:
////////////////////////////////////////////////////////////////////////////////
/// @brief input-output scheduler using libev
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
/// @author Achim Brandt
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2008-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "Scheduler/SchedulerLibev.h"

#ifdef _WIN32
#include "Basics/win-utils.h"
#include <evwrap.h>
#else
#include <ev.h>
#endif

#include "Basics/Exceptions.h"
#include "Basics/logging.h"
#include "Scheduler/SchedulerThread.h"
#include "Scheduler/Task.h"

using namespace triagens::basics;
using namespace triagens::rest;

// -----------------------------------------------------------------------------
// --SECTION--                                                             libev
// -----------------------------------------------------------------------------

/* EV_TIMER is an alias for EV_TIMEOUT */
#ifndef EV_TIMER
#define EV_TIMER EV_TIMEOUT
#endif

// -----------------------------------------------------------------------------
// --SECTION--                                                    private helper
// -----------------------------------------------------------------------------

namespace {

////////////////////////////////////////////////////////////////////////////////
/// @brief async event watcher
////////////////////////////////////////////////////////////////////////////////

  struct AsyncWatcher : public ev_async, Watcher {
    struct ev_loop* loop;
    Task* task;

    AsyncWatcher (struct ev_loop* loop, Task* task)
      : Watcher(EVENT_ASYNC),
        loop(loop),
        task(task) {
    }
  };

////////////////////////////////////////////////////////////////////////////////
/// @brief async event callback
////////////////////////////////////////////////////////////////////////////////

  void asyncCallback (struct ev_loop*, ev_async* w, int revents) {
    AsyncWatcher* watcher = (AsyncWatcher*) w; // cast from C type to C++ class
    Task* task = watcher->task;

    if (task != nullptr && (revents & EV_ASYNC) && task->isActive()) {
      task->handleEvent(watcher, EVENT_ASYNC);
    }
  }

////////////////////////////////////////////////////////////////////////////////
/// @brief waker callback
////////////////////////////////////////////////////////////////////////////////

  void wakerCallback (struct ev_loop* loop, ev_async*, int) {
    ev_unloop(loop, EVUNLOOP_ALL);
  }

////////////////////////////////////////////////////////////////////////////////
/// @brief socket event watcher
////////////////////////////////////////////////////////////////////////////////

  struct SocketWatcher : public ev_io, Watcher {
    struct ev_loop* loop;
    Task* task;

    SocketWatcher (struct ev_loop* loop, Task* task)
      : Watcher(EVENT_SOCKET_READ),
        loop(loop),
        task(task) {
    }
  };

////////////////////////////////////////////////////////////////////////////////
/// @brief socket event callback
////////////////////////////////////////////////////////////////////////////////

  void socketCallback (struct ev_loop*, ev_io* w, int revents) {
    SocketWatcher* watcher = (SocketWatcher*) w; // cast from C type to C++ class
    Task* task = watcher->task;

    if (task != nullptr && task->isActive()) {
      if (revents & EV_READ) {
        if (revents & EV_WRITE) {
          task->handleEvent(watcher, EVENT_SOCKET_READ | EVENT_SOCKET_WRITE);
        }
        else {
          task->handleEvent(watcher, EVENT_SOCKET_READ);
        }
      }
      else if (revents & EV_WRITE) {
        task->handleEvent(watcher, EVENT_SOCKET_WRITE);
      }
    }
  }

////////////////////////////////////////////////////////////////////////////////
/// @brief periodic event watcher
////////////////////////////////////////////////////////////////////////////////

  struct PeriodicWatcher : public ev_periodic, Watcher {
    struct ev_loop* loop;
    Task* task;

    PeriodicWatcher (struct ev_loop* loop, Task* task)
      : Watcher(EVENT_PERIODIC),
        loop(loop),
        task(task) {
    }
  };

////////////////////////////////////////////////////////////////////////////////
/// @brief periodic event callback
////////////////////////////////////////////////////////////////////////////////

  void periodicCallback (struct ev_loop*, ev_periodic* w, int revents) {
    PeriodicWatcher* watcher = (PeriodicWatcher*) w; // cast from C type to C++ class
    Task* task = watcher->task;

    if (task != nullptr && (revents & EV_PERIODIC) && task->isActive()) {
      task->handleEvent(watcher, EVENT_PERIODIC);
    }
  }

////////////////////////////////////////////////////////////////////////////////
/// @brief signal event watcher
////////////////////////////////////////////////////////////////////////////////

  struct SignalWatcher : public ev_signal, Watcher {
    struct ev_loop* loop;
    Task* task;

    SignalWatcher (struct ev_loop* loop, Task* task)
      : Watcher(EVENT_SIGNAL),
        loop(loop),
        task(task) {
    }
  };

////////////////////////////////////////////////////////////////////////////////
/// @brief signal event callback
////////////////////////////////////////////////////////////////////////////////

  void signalCallback (struct ev_loop*, ev_signal* w, int revents) {
    SignalWatcher* watcher = (SignalWatcher*) w; // cast from C type to C++ class
    Task* task = watcher->task;

    if (task != nullptr && (revents & EV_SIGNAL) && task->isActive()) {
      task->handleEvent(watcher, EVENT_SIGNAL);
    }
  }

////////////////////////////////////////////////////////////////////////////////
/// @brief timer event watcher
////////////////////////////////////////////////////////////////////////////////

  struct TimerWatcher : public ev_timer, Watcher {
    struct ev_loop* loop;
    Task* task;

    TimerWatcher (struct ev_loop* loop, Task* task)
      : Watcher(EVENT_TIMER),
        loop(loop),
        task(task) {
    }
  };

////////////////////////////////////////////////////////////////////////////////
/// @brief timer event callback
////////////////////////////////////////////////////////////////////////////////

  void timerCallback (struct ev_loop*, ev_timer* w, int revents) {
    TimerWatcher* watcher = (TimerWatcher*) w; // cast from C type to C++ class
    Task* task = watcher->task;

    if (task != nullptr && (revents & EV_TIMER) && task->isActive()) {
      task->handleEvent(watcher, EVENT_TIMER);
    }
  }
}

// -----------------------------------------------------------------------------
// --SECTION--                                              class SchedulerLibev
// -----------------------------------------------------------------------------

// -----------------------------------------------------------------------------
// --SECTION--                                          private static variables
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not the allocator was switched
////////////////////////////////////////////////////////////////////////////////

bool SchedulerLibev::SwitchedAllocator = false;

// -----------------------------------------------------------------------------
// --SECTION--                                             static public methods
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief returns the available backends
////////////////////////////////////////////////////////////////////////////////

int SchedulerLibev::availableBackends () {
  return ev_supported_backends();
}

////////////////////////////////////////////////////////////////////////////////
/// @brief set the libev allocator to our own allocator
///
/// this is done to avoid the numerous memory problems as reported by Valgrind
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::switchAllocator () {
  if (! SwitchedAllocator) {
    SwitchedAllocator = true;

    // set the libev allocator to our own allocator
    ev_set_allocator(
#ifdef EV_THROW
      reinterpret_cast<void *(*)(void *ptr, long size) EV_THROW>
#endif
      (&TRI_WrappedReallocate));
  }
}

// -----------------------------------------------------------------------------
// --SECTION--                                      constructors and destructors
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief creates a scheduler
////////////////////////////////////////////////////////////////////////////////

SchedulerLibev::SchedulerLibev (size_t concurrency, int backend)
  : Scheduler(concurrency),
    _backend(backend) {

  switchAllocator();

  //_backend = 1;

  // report status
  LOG_TRACE("supported backends: %d", (int) ev_supported_backends());
  LOG_TRACE("recommended backends: %d", (int) ev_recommended_backends());
  LOG_TRACE("embeddable backends: %d", (int) ev_embeddable_backends());
  LOG_TRACE("backend flags: %d", (int) backend);

  // construct the loops
  _loops = new struct ev_loop*[nrThreads];

  ((struct ev_loop**) _loops)[0] = ev_default_loop(_backend);

  for (size_t i = 1;  i < nrThreads;  ++i) {
    ((struct ev_loop**) _loops)[i] = ev_loop_new(_backend);
  }

  // construct the scheduler threads
  threads = new SchedulerThread* [nrThreads];
  _wakers = new ev_async*[nrThreads];

  for (size_t i = 0;  i < nrThreads;  ++i) {
    threads[i] = new SchedulerThread(this, EventLoop(i), i == 0);

    ev_async* w = new ev_async;

    ev_async_init(w, wakerCallback);
    ev_async_start(((struct ev_loop**) _loops)[i], w);

    ((ev_async**) _wakers)[i] = w;
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief deletes a scheduler
////////////////////////////////////////////////////////////////////////////////

SchedulerLibev::~SchedulerLibev () {

  // begin shutdown sequence within threads
  for (size_t i = 0;  i < nrThreads;  ++i) {
    threads[i]->beginShutdown();
  }

  // force threads to shutdown
  for (size_t i = 0;  i < nrThreads;  ++i) {
    threads[i]->stop();
  }

  for (size_t i = 0;  i < 100 && isRunning();  ++i) {
    usleep(100);
  }

  // shutdown loops
  for (size_t i = 1;  i < nrThreads;  ++i) {
    ev_async_stop(((struct ev_loop**) _loops)[i], ((ev_async**) _wakers)[i]);
    ev_loop_destroy(((struct ev_loop**) _loops)[i]);
  }

  ev_async_stop(((struct ev_loop**) _loops)[0], ((ev_async**) _wakers)[0]);
  ev_default_destroy();

  // and delete threads
  for (size_t i = 0;  i < nrThreads;  ++i) {
    delete threads[i];
    delete ((ev_async**) _wakers)[i];
  }

  // delete loops buffer
  delete[] ((struct ev_loop**) _loops);

  // delete threads buffer and wakers
  delete[] threads;
  delete[] (ev_async**)_wakers;
}

// -----------------------------------------------------------------------------
// --SECTION--                                                 Scheduler methods
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::eventLoop (EventLoop loop) {
  struct ev_loop* l = (struct ev_loop*) lookupLoop(loop);

  ev_loop(l, 0);
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::wakeupLoop (EventLoop loop) {
  if (size_t(loop) >= nrThreads) {
    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unknown loop");
  }

  ev_async_send(((struct ev_loop**) _loops)[loop], ((ev_async**) _wakers)[loop]);
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::uninstallEvent (EventToken watcher) {
  if (watcher == nullptr) {
    return;
  }

  EventType type = watcher->type;

  switch (type) {
    case EVENT_ASYNC: {
      AsyncWatcher* w = (AsyncWatcher*) watcher;
      ev_async_stop(w->loop, (ev_async*) w);
      delete w;

      break;
    }

    case EVENT_PERIODIC: {
      PeriodicWatcher* w = (PeriodicWatcher*) watcher;
      ev_periodic_stop(w->loop, (ev_periodic*) w);
      delete w;

      break;
    }

    case EVENT_SIGNAL: {
      SignalWatcher* w = (SignalWatcher*) watcher;
      ev_signal_stop(w->loop, (ev_signal*) w);
      delete w;

      break;
    }

    case EVENT_SOCKET_READ: {
      SocketWatcher* w = (SocketWatcher*) watcher;
      ev_io_stop(w->loop, (ev_io*) w);
      delete w;

      break;
    }

    case EVENT_TIMER: {
      TimerWatcher* w = (TimerWatcher*) watcher;
      ev_timer_stop(w->loop, (ev_timer*) w);
      delete w;

      break;
    }
  }
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

EventToken SchedulerLibev::installAsyncEvent (EventLoop loop, Task* task) {
  AsyncWatcher* watcher = new AsyncWatcher((struct ev_loop*) lookupLoop(loop), task);

  ev_async* w = (ev_async*) watcher;
  ev_async_init(w, asyncCallback);
  ev_async_start(watcher->loop, w);

  return watcher;
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::sendAsync (EventToken token) {
  AsyncWatcher* watcher = (AsyncWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_async* w = (ev_async*) watcher;
  ev_async_send(watcher->loop, w);
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

EventToken SchedulerLibev::installPeriodicEvent (EventLoop loop, Task* task, double offset, double interval) {
  PeriodicWatcher* watcher = new PeriodicWatcher((struct ev_loop*) lookupLoop(loop), task);

  ev_periodic* w = (ev_periodic*) watcher;
  ev_periodic_init(w, periodicCallback, offset, interval, 0);
  ev_periodic_start(watcher->loop, w);

  return watcher;
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::rearmPeriodic (EventToken token, double offset, double interval) {
  PeriodicWatcher* watcher = (PeriodicWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_periodic* w = (ev_periodic*) watcher;
  ev_periodic_set(w, offset, interval, 0);
  ev_periodic_again(watcher->loop, w);
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

EventToken SchedulerLibev::installSignalEvent (EventLoop loop, Task* task, int signal) {
  SignalWatcher* watcher = new SignalWatcher((struct ev_loop*) lookupLoop(loop), task);

  ev_signal* w = (ev_signal*) watcher;
  ev_signal_init(w, signalCallback, signal);
  ev_signal_start(watcher->loop, w);

  return watcher;
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

  // ..........................................................................
  // Windows likes to operate on SOCKET types (sort of handles) while libev
  // likes to operate on file descriptors
  // Our abstraction for sockets allows to use exactly the same code
  // ..........................................................................

EventToken SchedulerLibev::installSocketEvent (EventLoop loop, EventType type, Task* task, TRI_socket_t socket) {
  SocketWatcher* watcher = new SocketWatcher((struct ev_loop*) lookupLoop(loop), task);

  int flags = 0;

  if (type & EVENT_SOCKET_READ) {
    flags |= EV_READ;
  }

  if (type & EVENT_SOCKET_WRITE) {
    flags |= EV_WRITE;
  }

  ev_io* w = (ev_io*) watcher;
  // Note that we do not use TRI_get_fd_or_handle_of_socket here because even
  // under Windows we want get the entry fileDescriptor here because
  // of the reason that is mentioned above!
  ev_io_init(w, socketCallback, socket.fileDescriptor, flags);
  ev_io_start(watcher->loop, w);

  return watcher;
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::startSocketEvents (EventToken token) {
  SocketWatcher* watcher = (SocketWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_io* w = (ev_io*) watcher;

  if (! ev_is_active(w)) {
    ev_io_start(watcher->loop, w);
  }
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::stopSocketEvents (EventToken token) {
  SocketWatcher* watcher = (SocketWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_io* w = (ev_io*) watcher;

  if (ev_is_active(w)) {
    ev_io_stop(watcher->loop, w);
  }
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

EventToken SchedulerLibev::installTimerEvent (EventLoop loop, Task* task, double timeout) {
  TimerWatcher* watcher = new TimerWatcher((struct ev_loop*) lookupLoop(loop), task);

  ev_timer* w = (ev_timer*) watcher;
  ev_timer_init(w, timerCallback, timeout, 0.0);
  ev_timer_start(watcher->loop, w);

  return watcher;
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::clearTimer (EventToken token) {
  TimerWatcher* watcher = (TimerWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_timer* w = (ev_timer*) watcher;
  ev_timer_stop(watcher->loop, w);
}

////////////////////////////////////////////////////////////////////////////////
/// {@inheritDoc}
////////////////////////////////////////////////////////////////////////////////

void SchedulerLibev::rearmTimer (EventToken token, double timeout) {
  TimerWatcher* watcher = (TimerWatcher*) token;

  if (watcher == nullptr) {
    return;
  }

  ev_timer* w = (ev_timer*) watcher;
  ev_timer_set(w, 0.0, timeout);
  ev_timer_again(watcher->loop, w);
}

// -----------------------------------------------------------------------------
// --SECTION--                                                   private methods
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief looks up an event lookup
////////////////////////////////////////////////////////////////////////////////

void* SchedulerLibev::lookupLoop (EventLoop loop) {
  if (size_t(loop) >= nrThreads) {
    THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "unknown loop");
  }

  return ((struct ev_loop**) _loops)[loop];
}

// -----------------------------------------------------------------------------
// --SECTION--                                                       END-OF-FILE
// -----------------------------------------------------------------------------

// Local Variables:
// mode: outline-minor
// outline-regexp: "/// @brief\\|/// {@inheritDoc}\\|/// @page\\|// --SECTION--\\|/// @\\}"
// End:


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">

		<title> FAQ: WVO 'Vette </title>
		<link href="vette.css" rel="stylesheet" type="text/css">

veggie 'Vette Frequently Asked Questions

 What kind of milage do you get?

						the 'vette gets about 40mpg on the highway on biodiesel.  Reports say that milage and power output are roughly equivalent on veggie, but I haven't taken any accurate measurements of milage yet, but I'd like to get a reading soon.

 Do you really get fuel for free?

My favorite question when hitting up a restaurant for free oil is "so, how much are you going to charge me to take this stuff?".   Negative cost fuel!     I do it for free, as a community service of course.    Pre-processing the oil is labour intensive though, so I don't like to think about my
hourly pay for doing all the filtering and collecting work.

 Why doesn't this rocketsled have a front seat?

How many times have you been given a ride in a limousine and sat in front with the driver?     Never.   I wanted to enforce the festive atmosphere and let the passengers wrap themselves in all the luxurious splendor the 'Vette has to offer, in case they got some fool idea that this is just any other car and mistakenly sat in the front seat.

 Who is Ellen Feiss?

http://www.everything2.org/index.pl?node_id=1338098

http://www.apple.com/switch/ads/ellenfeiss.html

http://ellenfeiss.gloriousnoise.com/

<nobr>
[ h o m e  ]
[  g a l l e r y  ]
[  m a d  s c i e n t i s t  ]
</nobr>


text:
package ca.tylerwest.greenhouse.web.auth;

import ca.tylerwest.greenhouse.web.auth.beans.Role;

public interface AccessControl {

    public boolean signIn(String username, String password);

    public boolean isUserSignedIn();

    public boolean isUserInRole(Role role);

    public String getPrincipalName();
}


text:
import os
import sys
import argparse
from cuttsum.event import read_events_xml
from cuttsum.nuggets import read_nuggets_tsv
from cuttsum.util import gen_dates
import cuttsum.wtmf
import streamcorpus as sc
from sklearn.metrics.pairwise import cosine_similarity
from collections import defaultdict
import numpy as np

def main():

    event_file, rc_dir, event_title, nuggets_tsv, ss_params, ofile = parse_args()
    ss_model, ss_vocab, ss_dims = ss_params
    event = load_event(event_title, event_file)
    nuggets = read_nuggets_tsv(nuggets_tsv, filter_query_id=event.query_id)
    hours = [dth for dth in gen_dates(event.start, event.end)]
    print u"Found", len(nuggets), u"nuggets."

    print u"Loading sentence-sim model..."
    wmat_model = cuttsum.wtmf.load_model(ss_model, ss_vocab, latent_dims=ss_dims)
    nugget_lvecs = wmat_model.factor_unicode([n.text for n in nuggets])

    meta_data = []
    unicodes = []
    print u"Loading sentence data..."
    nhours = len(hours)
    for h, hour in enumerate(hours, 1):
        chunk = os.path.join(rc_dir, u'{}.sc.gz'.format(hour))

        for si_idx, si in enumerate(sc.Chunk(path=chunk)):
            if u'article-clf' not in si.body.sentences:
                continue
            sent_idx_map = {}
            for idx, sent in enumerate(si.body.sentences[u'serif']):
                sent_idx_map[sentence_uni(sent)] = idx
            for sent in si.body.sentences[u'article-clf']:
                uni = sentence_uni(sent)
                meta_data.append((hour, si.stream_id, sent_idx_map[uni]))
                unicodes.append(uni)

    print u"Computing similarities..."
    sent_lvecs = wmat_model.factor_unicode(unicodes)
    S = cosine_similarity(sent_lvecs, nugget_lvecs)
    S = np.ma.masked_array(S, np.isnan(S))

    Szmuv = (S - S.mean(axis=0)) / S.std(axis=0)
    M = np.amax(Szmuv, axis=1)
    m = np.amin(Szmuv, axis=1)
    U = np.mean(Szmuv, axis=1)
    T = np.sum(Szmuv, axis=1)

    ### WRITE TSV HEADER AND DATA ###
    print u"Writing to", ofile
    header = 'date-hour\tstream-id\tsent-id\tmax-sim\tmin-sim' + \
             '\tmean-sim\ttotal-sim'
    for i in range(ss_dims):
        header += '\tlv{}'.format(i)

    with open(ofile, 'w') as f:
        f.write(header)
        f.write('\n')
        for idx, meta_datum in enumerate(meta_data):
            f.write('{}\t{}\t{}\t{}\t{}\t{}\t{}'.format(meta_datum[0], meta_datum[1],
                                                    meta_datum[2], M[idx], m[idx], U[idx]))
            for c in range(ss_dims):
                f.write('\t{}'.format(sent_lvecs[idx,c]))
            f.write('\n')
            f.flush()

def sentence_uni(sent):
    return u' '.join(token.token.decode(u'utf-8') for token in sent.tokens)

def get_active_nuggets(hour, nuggets, lvecs):
    act_nugs = []
    idx = 0
    for nugget in nuggets:
        if nugget.timestamp.strftime("%Y-%m-%d-%H") <= hour:
            idx += 1
        else:
            break
    if idx > 0:
        return lvecs[0:idx,:]
    else:
        return None

def load_event(event_title, event_xml):
    events = read_events_xml(event_xml)
    for event in events:
        if event_title == event.title:
            return event
    raise ValueError(("No event title matches \"{}\" " \
                      + "in file: {}").format(event_title, event_xml))

def parse_args():
    parser = argparse.ArgumentParser()

    parser.add_argument('-e', '--event-file',
                        help=u'Event xml file.',
                        type=unicode, required=True)

    parser.add_argument('-r', '--rel-chunks-dir',
                        help=u'Relevance Chunks dir',
                        type=str, required=True)

    parser.add_argument('-n', '--nuggets-tsv',
                        help=u'Nuggets tsv file',
                        type=str, required=True)

    parser.add_argument('-t', '--event-title',
                        help=u'Event title',
                        type=unicode, required=True)

    parser.add_argument('-s', '--sent-sim-model',
                        help=u'Location of sentence sim model',
                        type=unicode, required=True)

    parser.add_argument('-v', '--sent-sim-vocab',
                        help=u'Location of sentence sim vocab',
                        type=unicode, required=True)

    parser.add_argument('-d', '--sent-sim-dims',
                        help=u'Sentence-sim model dimensions',
                        type=int, required=True)

    parser.add_argument('-o', '--output-file',
                        help=u'Location to write sims',
                        type=unicode, required=True)

    args = parser.parse_args()
    event_file = args.event_file
    rc_dir = args.rel_chunks_dir
    event_title = args.event_title
    nuggets_tsv = args.nuggets_tsv
    ss_model = args.sent_sim_model
    ss_vocab = args.sent_sim_vocab
    dims = args.sent_sim_dims
    ofile = args.output_file

    odir = os.path.dirname(ofile)
    if odir != u'' and not os.path.exists(odir):
        os.makedirs(odir)

    if not os.path.exists(event_file) or os.path.isdir(event_file):
        sys.stderr.write((u'--event-file argument {} either does not exist' \
                          + u' or is a directory!\n').format(event_file))
        sys.stderr.flush()
        sys.exit()

    if not os.path.exists(rc_dir) or not os.path.isdir(rc_dir):
        sys.stderr.write((u'--rel-chunks-dir argument {} either does not' \
                          + u' exist or is not a directory!\n').format(rc_dir))
        sys.stderr.flush()
        sys.exit()

    if not os.path.exists(nuggets_tsv) or os.path.isdir(nuggets_tsv):
        sys.stderr.write((u'--nuggets-tsv argument {} either does not' \
                          + u' exist or is a directory!\n').format(
                            nuggets_tsv))
        sys.stderr.flush()
        sys.exit()

    if not os.path.exists(ss_model) or os.path.isdir(ss_model):
        sys.stderr.write((u'--sent-sim-model argument {} either does not' \
                          + u' exist or is a directory!\n').format(
                            ss_model))
        sys.stderr.flush()
        sys.exit()

    if not os.path.exists(ss_vocab) or os.path.isdir(ss_vocab):
        sys.stderr.write((u'--sent-sim-vocab argument {} either does not' \
                          + u' exist or is a directory!\n').format(
                            ss_vocab))
        sys.stderr.flush()
        sys.exit()

    return (event_file, rc_dir, event_title, nuggets_tsv,
            (ss_model, ss_vocab, dims), ofile)

if __name__ == '__main__':
    main()


text:
This is the Opal corelib implementation API documentation.
The whole corelib is loaded upon `require 'opal'`.

The `runtime.js` documentation is [availble here](runtime.js.html)


text:
<!doctype html>
<title>CSS test reference</title>
<style>
p {
  width: 40ch;
  font: 16px/1 monospace;
  text-overflow: ellipsis;
  white-space: nowrap;
  overflow: hidden;
}
</style>
<p>short<p>


text:
<!DOCTYPE html>

        <meta charset="utf-8">
        <meta http-equiv="x-ua-compatible" content="ie=edge">
        <title><%= projectName %></title>
        <meta name="description" content="">
        <meta name="viewport" content="width=device-width, initial-scale=1">

        <link rel="apple-touch-icon" href="apple-touch-icon.png">

        <% if (hasGulp === false) { %>
        <link rel="stylesheet" href="css/normalize.css">
        <% } %>
        <link rel="stylesheet" href="css/main.css">

Go, Mechanic Go!

<%= projectName %> is ready for development

        <% if (hasGulp === true) { %>

        <% } else { %>

        <% } %>


text:
import Pyro4
Pyro4.config.HMAC_KEY = 'deadbeef'

from django.analysis.persisted import resolver_position_to_id, req_socket
import django.conf
import django.htoken as htoken
import django.htoken.serializer
from django.http import get_changeset

USED_NUMBERS = {}
import random
gen_guess = lambda : random.randint(2**10,2**20)

def pre_save_nolabel(**kwargs):
    sender_class, instance, raw = (kwargs["sender"], kwargs["instance"], kwargs["raw"])
    global USED_NUMBERS, gen_guess
    if raw or instance.id is not None:
        return
    pk = gen_guess()
    if sender_class in USED_NUMBERS:
        while pk in USED_NUMBERS[sender_class]:
            pk = gen_guess()
        USED_NUMBERS[sender_class].add(pk)
    else:
        USED_NUMBERS[sender_class] = set((pk,))
    instance.id = pk

class ViewServer(object):
    def get_response(self, request, token, delta):
        request = self.cereal.deserialize(request)
        self.cereal.apply_req_delta(request, delta)
        request.reset_changeset()
        htoken.set_token(token)
        from django.core import exceptions, urlresolvers
        from django.conf import settings

        from django.db.models.signals import pre_save as pre_save_signaler
        if self.starting:
            self.starting = False
            pre_save_signaler.connect(pre_save_nolabel)
        try:
            urlconf = settings.ROOT_URLCONF
            urlresolvers.set_urlconf(urlconf)
            resolver = urlresolvers.RegexURLResolver(r'^/', urlconf)
            cb, cb_args, cb_kwargs, position, view_name = resolver.resolve(request.path_info)
            assert resolver_position_to_id(position) == django.conf.view_id
            response = cb(request, *cb_args, **cb_kwargs)
            if hasattr(response, "render"):
                response.render()
            return htoken.get_token(), request.get_changeset(), response
        except Exception:
            import traceback
            tb = traceback.format_exc()
            print tb
            return None

def run_view_server(view_id):
    django.conf.view_id = view_id

    req_socket_str = req_socket(view_id)
    vs = ViewServer()
    vs.starting = True

    daemon = False
    try:
        daemon = Pyro4.Daemon(unixsocket=req_socket_str)
        daemon.serializer = django.htoken.serializer.Serializer()
        vs.cereal = daemon.serializer
        daemon.register(vs, 'request')
        daemon.requestLoop()
    except Exception:
        import traceback
        tb = traceback.format_exc()
        print tb
        return None

    finally:
        if daemon:
            daemon.close()