text:
from gi.repository import Gtk, GdkPixbuf, GLib, Pango
import dhash
import os
import collections
import threading
import Queue

class mainWindow(Gtk.Window):

    def __init__(self):
        Gtk.Window.__init__(self, title="DupDeleter")
        self.set_border_width(10)

        self.grid = Gtk.Grid()
        self.grid.set_column_homogeneous(True)
        self.grid.set_row_homogeneous(True)
        self.add(self.grid)

        # set up the model
        # columns = [imgName, imgLocation, # of Dups]
        self.dupe_store = Gtk.TreeStore(bool, str, str, int)

        self.current_filter_ext = None
        self.ext_filter = self.dupe_store.filter_new()
        self.ext_filter.set_visible_func(self.ext_filter_func)

        # Create model's view
        self.treeview = Gtk.TreeView.new_with_model(self.ext_filter)
        check_renderer = Gtk.CellRendererToggle()
        check_renderer.set_activatable(True)
        check_renderer.set_active(False)
        check_renderer.connect("toggled", self.on_toggled)
        column = Gtk.TreeViewColumn("", check_renderer, active=0)
        self.treeview.append_column(column)
        for i, column_title in enumerate(["Name", "Location", "# of Dups"]):
            renderer = Gtk.CellRendererText()
            column = Gtk.TreeViewColumn(column_title, renderer, text=i+1)
            column.set_sort_column_id(i)
            column.set_fixed_width(200)
            column.set_resizable(True)
            self.treeview.append_column(column)
        self.treeview.connect("cursor-changed", self.on_row_changed)

        self.scrollable_treelist = Gtk.ScrolledWindow()
        self.scrollable_treelist.set_vexpand(True)

        # Create buttons for filtering results by image extension
        self.extensions = ("jpg", "gif", "png", "tiff", "All")
        self.buttons = list()
        for ext in self.extensions:
            button = Gtk.Button(ext)
            self.buttons.append(button)
            button.connect("clicked", self.on_selection_button_clicked)

        self.create_toolbar()

        # Create labels for showing scan progress
        self.scan_status_label = Gtk.Label("No Scan in Progress")
        self.scan_status_label.set_halign(3)  # 3 = CENTER

        self.file_scan_label = Gtk.Label(None)
        self.file_scan_label.set_ellipsize(Pango.EllipsizeMode.START)
        self.file_scan_label.set_width_chars(30)
        self.file_scan_label.set_halign(3)  # 3 = CENTER

        # Make a frame to hold image previews
        self.img_frame = Gtk.Frame(label="Selected Image")
        self.img_frame.set_label_align(0.5, 0.5)
        self.img_frame.set_shadow_type(Gtk.ShadowType.ETCHED_OUT)
        self.img_frame.set_size_request(200, 200)

        # Assemble the GUI
        self.grid.attach(self.scrollable_treelist, 0, 1, 8, 10)
        self.grid.attach_next_to(self.img_frame, self.scrollable_treelist,
                                 Gtk.PositionType.RIGHT, 5, 6)
        self.grid.attach_next_to(self.buttons[0], self.scrollable_treelist,
                                 Gtk.PositionType.BOTTOM, 1, 1)
        for i, button in enumerate(self.buttons[1:]):
            self.grid.attach_next_to(button, self.buttons[i],
                                     Gtk.PositionType.RIGHT, 1, 1)
        self.scrollable_treelist.add(self.treeview)
        self.grid.attach_next_to(self.scan_status_label, self.buttons[3],
                                 Gtk.PositionType.BOTTOM, 6, 1)
        self.grid.attach_next_to(self.file_scan_label, self.scan_status_label,
                                 Gtk.PositionType.BOTTOM, 6, 1)
        self.grid.set_column_spacing(10)
        self.grid.set_row_spacing(5)

        self.queue = Queue.Queue()  # Queue for holding fetched images
        self.delete_list = list()  # List for holding to-be-deleted items

        self.show_all()

    def getDups(self, path, queue):
        '''Collects all image duplicates starting from PATH.
        Fills a queue with lists of lists
        containing image names and locations.'''
        images = collections.defaultdict(list)
        image_exts = ('.jpg', '.png', '.gif', '.tiff')

        GLib.idle_add(self.scan_status_label.set_text, "Scanning...")

        for root, dirs, files in os.walk(path):
            for name in files:
                GLib.idle_add(self.file_scan_label.set_text,
                              root)
                if name[-4:] in image_exts:
                    img_loc = os.path.join(root, name)
                    img_hash = dhash.hash(img_loc)
                    if img_hash != -1:
                        # Have to add False at beginning because of
                        # togglebutton status.
                        images[img_hash].append([False, name, root])

        GLib.idle_add(self.scan_status_label.set_text, "Done")

        for group in images:
            if len(images[group]) > 1:
                queue.put(images[group])

        GLib.idle_add(self.generateModelData)

    def generateModelData(self):
        '''Fills treeModel rows with found duplicates'''
        while not self.queue.empty():
            image_set = self.queue.get()
            parent = ''
            no_of_dups = len(image_set)
            for img in image_set:
                img.append(no_of_dups)
                if not parent:
                    parent = self.dupe_store.append(None, img)
                else:
                    self.dupe_store.append(parent, img)

    def on_button_clicked_open(self, widget):
        '''Brings up the file browser window.
        Returns the path of the root folder.'''
        dialog = Gtk.FileChooserDialog("Select the root folder", self,
                                       Gtk.FileChooserAction.SELECT_FOLDER,
                                       (Gtk.STOCK_CANCEL,
                                        Gtk.ResponseType.CANCEL,
                                        "Select", Gtk.ResponseType.OK))
        dialog.set_default_size(800, 400)

        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            root = dialog.get_uri()[8:]  # have to remove file:///
            thread = threading.Thread(target=self.getDups,
                                      args=(root, self.queue))
            thread.daemon = True
            thread.start()
            GLib.timeout_add(200, self.generateModelData)
            dialog.destroy()
        elif response == Gtk.ResponseType.CANCEL:
            dialog.destroy()

    def ext_filter_func(self, model, iter, data):
        '''Tests if the image extension in the row is the one in the filter'''
        if self.current_filter_ext is None or self.current_filter_ext == "All":
            return True
        else:
            return model[iter][0][-3:] == self.current_filter_ext

    def create_toolbar(self):
        toolbar = Gtk.Toolbar()
        self.grid.attach(toolbar, 0, 0, 8, 1)

        button_open = Gtk.ToolButton.new_from_stock(Gtk.STOCK_OPEN)
        button_open.set_tooltip_text("Scan Directory")
        toolbar.insert(button_open, 0)

        button_delete = Gtk.ToolButton.new_from_stock(Gtk.STOCK_DELETE)
        button_delete.set_tooltip_text("Delete Selected Images")
        toolbar.insert(button_delete, 1)

        button_auto_prune = Gtk.ToolButton.new_from_stock(Gtk.STOCK_NO)
        button_auto_prune.set_tooltip_text("Auto-Prune")
        toolbar.insert(button_auto_prune, 2)

        button_exit = Gtk.ToolButton.new_from_stock(Gtk.STOCK_QUIT)
        button_exit.set_tooltip_text("Exit")
        toolbar.insert(button_exit, 3)

        button_open.connect("clicked", self.on_button_clicked_open)
        button_auto_prune.connect("clicked", self.on_button_clicked_auto_prune)
        button_delete.connect("clicked", self.on_button_clicked_delete)
        button_exit.connect("clicked", self.on_button_clicked_exit)

    def on_selection_button_clicked(self, widget):
        '''Called on any selection button click'''
        self.current_filter_ext = widget.get_label()
        self.ext_filter.refilter()

    def on_button_clicked_auto_prune(self, widget):
        '''
        Automatically deletes all files except for parents
        '''
        dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.WARNING,
                                   Gtk.ButtonsType.OK_CANCEL,
                                   "Delete all selected images?")
        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            rootiter = self.dupe_store.get_iter_first()
            self.prune_helper(rootiter, False)
            dialog.destroy()
        elif response == Gtk.ResponseType.CANCEL:
            dialog.destroy()

    def prune_helper(self, treeiter, toDelete):
        '''
        Deletes all files except for parents

        toDelete indicates whether or not treeiter should be deleted.
        It should be set to False on call unless you want to delete
        everything.
        '''
        deleted = 0
        isValid = True
        while (treeiter is not None) and isValid:
            if self.dupe_store.iter_has_child(treeiter):
                childiter = self.dupe_store.iter_children(treeiter)
                deleted += self.prune_helper(childiter, True)
                self.dupe_store[treeiter][3] = self.dupe_store.iter_n_children(treeiter)
            if toDelete:
                path = os.path.join(self.dupe_store[treeiter][2],
                                    self.dupe_store[treeiter][1])
                os.remove(path)
                isValid = self.dupe_store.remove(treeiter)
                deleted += 1
            # If treestore.remove() is successful iter is automatically
            # updated to the next iter, so we just need to check to
            # make sure that isn't the case before using iter_next()
            else:
                treeiter = self.dupe_store.iter_next(treeiter)
        return deleted

    def on_button_clicked_delete(self, widget):
        '''Deletes all selected files'''
        dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.WARNING,
                                   Gtk.ButtonsType.OK_CANCEL,
                                   "Delete all selected images?")
        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            rootiter = self.dupe_store.get_iter_first()
            self.delete_helper(rootiter)
            dialog.destroy()
        elif response == Gtk. ResponseType.CANCEL:
            dialog.destroy()

    def delete_helper(self, treeiter):
        '''
        Recursively searches through all rows searching for files to
        delete.
        '''
        deleted = 0
        isValid = True
        while (treeiter is not None) and isValid:
            if self.dupe_store.iter_has_child(treeiter):
                childiter = self.dupe_store.iter_children(treeiter)
                deleted += self.delete_helper(childiter)
            if self.dupe_store[treeiter][0]:
                path = os.path.join(self.dupe_store[treeiter][2],
                                    self.dupe_store[treeiter][1])
                if self.dupe_store.iter_has_child(treeiter):
                    child = self.dupe_store.iter_children(treeiter)
                    isValid = self.childToParent(treeiter, child)
                else:
                    isValid = self.dupe_store.remove(treeiter)
                os.remove(path)
                deleted += 1
                self.dupe_store[treeiter][3] = self.dupe_store.iter_n_children(treeiter)
            else:
                treeiter = self.dupe_store.iter_next(treeiter)
        return deleted

    def on_button_clicked_exit(self, widget):
        '''Exits the program'''
        dialog = Gtk.MessageDialog(self, 0, Gtk.MessageType.WARNING,
                                   Gtk.ButtonsType.OK_CANCEL,
                                   "Are you sure you want to exit?")
        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            Gtk.main_quit()
        elif response == Gtk.ResponseType.CANCEL:
            dialog.destroy()

    def childToParent(self, parent_iter, child_iter):
        '''
        Replaces parent_iter with child_iter, effectively moving child to
        parent's position.

        Returns the next iter after parent, or invalidates it if there
        isn't one.
        '''
        for col in xrange(self.dupe_store.get_n_columns()):
            childval = self.dupe_store.get_value(child_iter, col)
            self.dupe_store.set_value(parent_iter, col, childval)
        self.dupe_store.remove(child_iter)
        return self.dupe_store.iter_next(parent_iter)

    def on_row_changed(self, widget):
        (model, pathlist) = widget.get_selection().get_selected_rows()
        tree_iter = model.get_iter(pathlist[0])
        img_name = model.get_value(tree_iter, 1)
        img_root = model.get_value(tree_iter, 2)
        img_loc = os.path.join(img_root, img_name)
        child = self.img_frame.get_child()
        if child:
            self.img_frame.remove(child)
        alloc = self.img_frame.get_allocation()
        pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(img_loc,
                                                          alloc.width - 20,
                                                          alloc.height - 20,
                                                          True)
        image = Gtk.Image.new_from_pixbuf(pixbuf)
        image.show()
        self.img_frame.add(image)

    def on_toggled(self, widget, path):
        '''
        Adds or removes the row's treeiter to a list that designates it
        for removal.
        '''
        self.dupe_store[path][0] = not self.dupe_store[path][0]

win = mainWindow()
win.connect("delete-event", Gtk.main_quit)
win.show_all()
Gtk.main()


text:
package org.gearvrf.widgetlib.widget.animation;

import java.util.LinkedHashSet;
import java.util.Set;

import org.gearvrf.GVRHybridObject;
import org.gearvrf.animation.GVRAnimation;
import org.gearvrf.animation.GVRAnimationEngine;
import org.gearvrf.animation.GVRInterpolator;
import org.gearvrf.animation.GVROnFinish;
import org.json.JSONException;
import org.json.JSONObject;

import org.gearvrf.widgetlib.widget.Widget;
import org.gearvrf.widgetlib.log.Log;

import static org.gearvrf.widgetlib.widget.properties.JSONHelpers.getFloat;

/**
 * A wrapper for {@link GVRAnimation} and derived classes. Client code can
 * derive directly from this class (or its descendants) and override
 * {@link #animate(Widget, float)} to implement {@link Widget} animations.
 *

 * The class provides a number of handy features:
 *

 *
Support for multiple {@link OnFinish} listeners

 *
A {@link #finish()} method that stops the animation and runs the
 * animation's last frame

 *
Support for instantiation from JSON metadata

 *
Checks whether the target {@code Widget's} transform has changed during
 * an animation frame and {@linkplain Widget#requestLayout() requests a layout}
 * if needed.

 *

 */
public abstract class Animation {
    public interface OnFinish {
        void finished(Animation animation);
    }

    public void track(SimpleAnimationTracker tracker, final Widget target) {
        track(tracker, target, null);
    }

    public void track(SimpleAnimationTracker tracker, final Widget target,
            final OnFinish onFinish) {
        track(tracker, target, null, onFinish);
    }

    public void track(SimpleAnimationTracker tracker, final Widget target,
            final Runnable onStart, OnFinish onFinish) {
        tracker.track(target.getSceneObject(), (GVRAnimation) getAnimation(),
                      onStart, new GVROnFinishProxy(onFinish));
    }

    public void track(SimpleAnimationTracker tracker) {
        track(tracker, (OnFinish) null);
    }

    public void track(SimpleAnimationTracker tracker, final OnFinish onFinish) {
        track(tracker, (Runnable) null, onFinish);
    }

    public void track(SimpleAnimationTracker tracker, final Runnable onStart,
            OnFinish onFinish) {
        tracker.track(mTarget.getSceneObject(), (GVRAnimation) getAnimation(),
                      onStart, new GVROnFinishProxy(onFinish));
    }

    public Animation(Widget target, float duration) {
        mTarget = target;
        final Adapter animationAdapter = new Adapter(target, duration);
        mAnimation = animationAdapter;
    }

    public Animation setInterpolator(GVRInterpolator interpolator) {
        mInterpolator = interpolator;
        getAnimation().setInterpolator(interpolator);
        return this;
    }

    public Animation setRepeatMode(int mode) {
        mMode = mode;
        getAnimation().setRepeatMode(mode);
        return this;
    }

    public Animation setRepeatCount(int count) {
        getAnimation().setRepeatMode(count);
        return this;
    }

    /**
     * @return Whether {@link Widget#requestLayout()} will be called on the
     *         {@linkplain Widget target} when its transform
     *         {@linkplain Widget#isChanged() is changed} during each animation
     *         frame.
     */
    public boolean getRequestLayoutOnTargetChange() {
        return mRequestLayoutOnTargetChange;
    }

    /**
     * Sets whether {@link Widget#requestLayout()} will be called on the
     * {@linkplain Widget target} when its transform
     * {@linkplain Widget#isChanged() is changed} during each animation frame.
     *

     * By default, this feature is enabled.
     *
     * @param enable
     *            {@code true} to enable this feature, {@code false} to disable
     *            it.
     * @return The {@link Animation} instance for call chaining.
     */
    public Animation setRequestLayoutOnTargetChange(boolean enable) {
        mRequestLayoutOnTargetChange = enable;
        return this;
    }

    @Deprecated
    public Animation setOnFinish(final GVROnFinish onFinish) {
        return addOnFinish(new OnFinish() {
            @Override
            public void finished(Animation animation) {
                onFinish.finished((GVRAnimation) animation.getAnimation());
            }
        });
    }

    @Deprecated
    public Animation setOnFinish(OnFinish onFinish) {
        return addOnFinish(onFinish);
    }

    public Animation addOnFinish(OnFinish onFinish) {
        if (mOnFinish == null) {
            mOnFinish = new OnFinishManager();
            getAnimation().setOnFinish(mOnFinish);
        }
        mOnFinish.addOnFinish(onFinish);
        return this;
    }

    public void removeOnFinish(OnFinish onFinish) {
        if (mOnFinish != null) {
            mOnFinish.removeOnFinish(onFinish);
        }
    }

    /**
     * Start the animation.
     *
     * Changing properties once the animation is running can have unpredictable
     * results.
     *
     * @return {@code this}, so you can save the instance at the end of a chain
     *         of calls.
     */
    public Animation start() {
        return start(GVRAnimationEngine.getInstance(mTarget.getGVRContext()));
    }

    /**
     * Start the animation.
     *
     * Changing properties once the animation is running can have unpredictable
     * results.
     *
     * @param engine
     *            The global animation engine.
     * @return {@code this}, so you can save the instance at the end of a chain
     *         of calls.
     */
    public Animation start(GVRAnimationEngine engine) {
        ((GVRAnimation) getAnimation()).start(engine);
        mIsRunning = true;
        return this;
    }

    /**
     * Stop the animation, even if it is still running: the animated object will
     * be left in its current state, not reset to the start or end values.
     *
     * This is probably not what you want to do! Usually you will either
     *

     *
Use {@link #setRepeatCount(int) setRepeatCount(0)} to 'schedule'
     * termination at the end of the current repetition, or
     *
Call {@link #finish(GVRAnimationEngine) finish()} to set the
     * animation to its end state and notify any {@linkplain OnFinish listeners}
     *

     * You may want to {@code stop()} an animation if you are also
     * removing the animated object the same time. For example, you may be
     * spinning some sort of In Progress object. In a case like this, stopping
     * in mid-animation is harmless.
     */
    public void stop() {
        stop(GVRAnimationEngine.getInstance(mTarget.getGVRContext()));
    }

    /**
     * Stop the animation, even if it is still running: the animated object will
     * be left in its current state, not reset to the start or end values.
     *
     * This is probably not what you want to do! Usually you will either
     *

     *
Use {@link #setRepeatCount(int) setRepeatCount(0)} to 'schedule'
     * termination at the end of the current repetition, or
     *
Call {@link #finish(GVRAnimationEngine) finish()} to set the
     * animation to its end state and notify any {@linkplain OnFinish listeners}
     *

     * You may want to {@code stop()} an animation if you are also
     * removing the animated object the same time. For example, you may be
     * spinning some sort of In Progress object. In a case like this, stopping
     * in mid-animation is harmless.
     *
     * @param engine
     *            The global animation engine.
     */
    public void stop(GVRAnimationEngine engine) {
        engine.stop((GVRAnimation) getAnimation());
        mIsRunning = false;
    }

    /**
     * If the animation is still running, stops the animation, runs the last
     * frame, and calls any {@linkplain #addOnFinish(OnFinish) registered}
     * {@linkplain OnFinish listeners}.
     *
     * @return {@code True} if the animation was running and finishing was
     *         successful; {@code false} if the animation was not running.
     */
    public boolean finish() {
        return finish(GVRAnimationEngine.getInstance(mTarget.getGVRContext()));
    }

    /**
     * If the animation is still running, stops the animation, runs the last
     * frame, and calls any {@linkplain #addOnFinish(OnFinish) registered}
     * {@linkplain OnFinish listeners}.
     *
     * @param engine
     *            The global animation engine.
     * @return {@code True} if the animation was running and finishing was
     *         successful; {@code false} if the animation was not running.
     */
    public boolean finish(GVRAnimationEngine engine) {
        if (mIsRunning) {
            stop(engine);
            getAnimation().animate(mTarget.getSceneObject(), 1);
            if (mOnFinish != null) {
                mOnFinish.finished((GVRAnimation) getAnimation());
            }
            return true;
        }
        return false;
    }

    public void reset() {
        getAnimation().reset();
    }

    public boolean isFinished() {
        return getAnimation().isFinished();
    }

    public int getRepeatCount() {
        return getAnimation().getRepeatCount();
    }

    public float getDuration() {
        return getAnimation().getDuration();
    }

    public float getElapsedTime() {
        return getAnimation().getElapsedTime();
    }

    public enum Properties { duration }

    protected Animation(Widget target, JSONObject metadata)
            throws JSONException {
        this(target, getFloat(metadata, Properties.duration));
    }

    protected abstract void animate(Widget target, float ratio);

    /* package */
    Animation(Widget target) {
        mTarget = target;
    }

    /* package */
    void doAnimate(float ratio) {
        Log.v(Log.SUBSYSTEM.WIDGET, TAG, "doAnimate(): animating %s", mTarget.getName());

        animate(mTarget, ratio);
        if (mRequestLayoutOnTargetChange && mTarget.isChanged()) {
            mTarget.requestLayout();
        }
    }

    /**
     * @return An implementation of {@code AnimationAdapter}.
     *

     *         NOTE: Implementations of
     *         this method must be idempotent!
     */
    /* package */
    AnimationAdapter getAnimation() {
        return mAnimation;
    }

    /* package */
    public Widget getTarget() {
        return mTarget;
    }

    /* package */
    interface AnimationAdapter {
        void animate(GVRHybridObject target, float ratio);

        float getElapsedTime();

        float getDuration();

        int getRepeatCount();

        void reset();

        boolean isFinished();

        GVRAnimation setOnFinish(GVROnFinish onFinish);

        GVRAnimation setRepeatMode(int mode);

        GVRAnimation setInterpolator(GVRInterpolator interpolator);
    }

    private class Adapter extends GVRAnimation implements AnimationAdapter {

        public Adapter(Widget target, float duration) {
            super(target.getSceneObject(), duration);
        }

        @Override
        public void animate(GVRHybridObject target, float ratio) {
            doAnimate(ratio);
        }
    }

    private class GVROnFinishProxy implements GVROnFinish {

        public GVROnFinishProxy(OnFinish onFinish) {
            mOnFinish = onFinish;
        }

        @Override
        public void finished(GVRAnimation animation) {
            mOnFinish.finished(Animation.this);
        }

        private final OnFinish mOnFinish;
    }

    private class OnFinishManager implements GVROnFinish {
        @Override
        public void finished(GVRAnimation unused) {
            mIsRunning = false;
            for (OnFinish listener : mListeners) {
                try {
                    listener.finished(Animation.this);
                } catch (Exception e) {
                    e.printStackTrace();
                    Log.e(TAG, e, "OnFinishManager.finished()");
                }
            }
        }

        public void addOnFinish(final OnFinish onFinish) {
            mListeners.add(onFinish);
        }

        public void removeOnFinish(final OnFinish onFinish) {
            mListeners.remove(onFinish);
        }

        private Set<OnFinish> mListeners = new LinkedHashSet<OnFinish>();
    }

    protected GVRInterpolator mInterpolator;
    protected int mMode = -1;
    private final Widget mTarget;
    private AnimationAdapter mAnimation;
    protected OnFinishManager mOnFinish;
    private boolean mIsRunning;
    private boolean mRequestLayoutOnTargetChange = true;

    private static final String TAG = Animation.class.getSimpleName();
}

redpajama_set_name: RedPajamaGithub


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.skife.jdbi.v2;

/**
 * Used as a callback which guarantees that the inTransaction method is invoked in
 * a transaction, and will be committed or rolled back as specified.
 */
public interface TransactionCallback<ReturnType>
{
    ReturnType inTransaction(Handle conn, TransactionStatus status) throws Exception;
}


text:
layout: post
title: "Dimensionality Reduction via JL Lemma and Random Projection"
permalink: /random-projection
date: 2017-10-10
categories: ['Theory', 'Machine Learning']
author: Junxiong Wang
authorlink: http://ovss.github.io
tags: [Theory]
---

Nowadays, dimensionality is a serious problem of data analysis as the huge data we experience today results in very sparse sets and very high dimensions. Although, data scientists have long used tools such as principal component analysis (PCA) and independent component analysis (ICA) to project the high-dimensional data onto a subspace, but all those techniques reply on the computation of the eigenvectors of a $n \times n$ matrix, a very expensive operation (e.g., spectral decomposition) for high dimension $n$. Moreover, even though eigenspace has many important properties, it does not lead good approximations for many useful measures such as vector norms. We discuss another method random projection to reduce dimensionality.

In 1984, two mathematicians introduced and proved the following lemma.

## Johnson-Lindenstrauss lemma
For any $\epsilon \in (0,\frac{1}{2})$, $\forall x_1, x_2, \dots, x_d \in \mathbb{R}^{n}$, there exists a matrix $M \in \mathbb{R}^{m \times n}$ with $m = O(\frac{1}{\epsilon^2} \log{d})$ such that $\forall 1 \leq i,j \leq d$, we have

$$
(1-\epsilon)||x_i - x_j||_2 \leq ||Mx_i - Mx_j||_2 \leq (1+\epsilon)||x_i - x_j||_2
$$

Remark: This lemma states that for any pair vector $x_i, x_j$ in $\mathbb{R}^n$ dimension, there exist a sketch matrix $M$ which maps $\mathbb{R}^n \rightarrow \mathbb{R}^m$ and the Euclidean distance is preserved within $\epsilon$ factor. The result dimension does not relate to origin dimension $n$ (only relates to the number of vector pairs $d$).

During a long time, no one can figure out how to get this sketch matrix.

## Random Projection
Until 2003, some researches point out that this sketch matrix can be created using Gaussian distribution.

Consider the following matrix $A \in \mathbb{R}^{m \times n}$, where $A_{ij} \sim \mathcal{N}(0,1)$ and all $A_{ij}$ are independent. We claim that this matrix satisfies the statement of JL lemma.

Proof. It is obvious that sketch has an additional property,
$\forall i, (Ax)\_i = \sum\_{j=1}^{n} A\_{ij} x\_j \sim \mathcal{N}(0, ||x||_2^2)$. In other word, Gaussian distribution is 2-stable distribution. Then we can obtain $\|\|Ax\|\|\_2^2 = \sum\_{i=1}^{m} y\_i^2$, where $y\_i \sim \mathcal{N}(0, \|\|x\|\|\_2^2)$. That is to say, $\|\|Ax\|\|\_2^2$ follows a $\chi^2$ (chi-squared) distribution with degrees of freedom $m$. For tail bound of $\chi^2$ distribution, we can get

$$
P(||Ax||_2^2 - m||X||_2^2| > \epsilon m||X||_2^2) < \exp(-C \epsilon^2 m)
$$

for a constant $C > 0$.

Fix two index $i, j$, and let $y^{ij} = x_i - x_j$ and $M = \frac{1}{\sqrt{m}} A$, and set $m = \frac{4}{C \epsilon^2} \log{n}$ to get

$$
P(\left| ||M y^{ij}||_2^2 - ||y^{ij}||_2^2 \right| > \epsilon ||y^{ij}||_2^2) < \exp(-C \epsilon^2 m) = \frac{1}{n^4}
$$

Take the union bound to obtain,

$$
P(\forall i \neq j, \left| ||M y^{ij}||_2^2 - ||y^{ij}||_2^2 \right| > \epsilon ||y^{ij}||_2^2 ) < \sum_{i \neq j} P(|||M y^{ij}||_2^2 - ||y^{ij}||_2^2| < \epsilon ||y^{ij}||_2^2 ) < {n \choose 2} n^4 < \frac{1}{n^2}
$$

Thus,

$$
P(\forall i \neq j, \left| ||M (x_i - x_j)||_2^2 - ||x_i - x_j||_2^2 \right| \leq \epsilon ||x_i - x_j||_2^2 ) > 1 - \frac{1}{n^2}
$$

which is same as the guarantee in Johnson-Lindenstrauss lemma.

## Application
In this or other forms, the JL lemma has been used for a large variety of computational tasks, especially in streaming algorithm, such as

* [Computing a low-rank approximation to the original matrix A.](https://www.stat.berkeley.edu/~mmahoney/f13-stat260-cs294/Lectures/lecture19.pdf)

* [Finding nearest neighbors in high-dimensional space.](http://web.stanford.edu/class/cs369g/files/lectures/lec16.pdf)

* [Simplify the calculation of the effective resistance in Graph Spectral Sparsification.](https://simons.berkeley.edu/sites/default/files/docs/1768/slidessrivastava1.pdf)

* [Relates to Graph Sketches.](https://people.cs.umass.edu/~mcgregor/papers/12-pods1.pdf)

## Reference
- EPFL Topics in Theoretical Computer Science (Sublinear Algorithm for Big Data Analysis), 2017
- EPFL Advanced Algorithm, 2016
- Johnson, William B.; Lindenstrauss, Joram (1984). "Extensions of Lipschitz mappings into a Hilbert space". In Beals, Richard; Beck, Anatole; Bellow, Alexandra; et al. Conference in modern analysis and probability (New Haven, Conn., 1982). Contemporary Mathematics. 26. Providence, RI: American Mathematical Society. pp. 189–206.
- Kane, Daniel M.; Nelson, Jelani (2012). "Sparser Johnson-Lindenstrauss Transforms". Proceedings of the Twenty-Third Annual ACM-SIAM Symposium on Discrete Algorithms,. New York: Association for Computing Machinery (ACM).

The post is used for study purpose only.


text:
<?php

namespace convert;

/**
 * Iteratively reduce an array or a traversable to a single value
 * using a callback function.
 *
 * Function applies iteratively the callback function to elements and keys
 * of the array or the traversable, so as to reduce it to a single value.
 *
 * @param callable $callback the callback to apply to each element and key;
 * current intermediate result will be passed as the first argument
 * (in the case of the first iteration it instead holds the value of
 * <code>$initial</code>), current value will be passed as the second one,
 * current key will be passed as the third one; callback must return
 * new intermediate result
 * @param mixed $initial initial value
 * @param array|\Traversable $traversable the input traversable
 * @return mixed the resulting value
 */
function traversable_reduce( callable $callback, $initial, $traversable ) {
    $result = $initial;
    foreach( to_traversable( $traversable ) as $key => $value ) {
        $result = call_user_func( $callback, $result, $value, $key );
    }
    return $result;
}

const traversable_reduce = '\\convert\\traversable_reduce';


text:
import click

from osp_pipeline.core.jobs import run_job

@click.command()
@click.argument('name')
def main(name):
    run_job(name)

if __name__ == '__main__':
    main()


text:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
    <!-- NOTE: This is the properties used for tests! -->

    <!-- 'First' index location;
         Used only for testing;
         create symlink from the actual index location to /tmp/index:
         ln -s indexlocation /tmp/index -->
    <entry key="directory_name">/tmp/index</entry>
    <!-- Parallel cluster index location -->
    <entry key="directory_name1">/tmp/par_clu_ind</entry>

    <!-- Default field; where is this used? -->
    <entry key="default_field">freetext</entry>

    <!-- ******************************************************************* -->
    <!-- Cluster presentation -->
    <!-- ******************************************************************* -->

    <entry key="default_maxClusterFacets">30</entry>
    <entry key="default_maxClusterTags">5</entry>
    <entry key="default_maxClusterObjects">5</entry>

    <!-- Expand hits to shortformat -->
    <entry key="default_expandObjects">true</entry>

    <!-- -1: No limit -->
    <entry key="default_maxResults">10000</entry>

    <!-- -1: No limit -->
    <entry key="default_maxMS">-1</entry>

    <!--
    0: Use program default
    1: Type only
    2: Fields
    3: Fields + TermVector + Specials
    This is largely deprecated in favor of the more specific settings below.
    -->
    <entry key="default_completeness">3</entry>

    <!-- Except for sort_title, these are defined by Dorete -->
<!--    <entry key="ClusterTermVectors">author_normalised, lsu_oai, llang, ldbk</entry>-->
<!--    <entry key="ClusterTermVectors">location_normalised, series_normalised, llcl, lnlm, ldbk, ludk, ldds, lkl, lfn, linst, ldk5, lma_long, su_pe, su_corp, su_lc, mesh, su_dk, lsu_oai, llang</entry>-->

    <!-- A list of the wanted facets. The facet names in the list must be exactly
         the same for building and displaying, including order, excluding
         maxTags.
         If the number of maxTags for a given Facet should be different than
         the default one (default_maxClusterTags), this can be stated in a
         paranthesis.
         Example: lma_long (200)
         There MUST be a space before the paranthesis.
         A facet name MUST NOT contain spaces. -->
    <entry key="ClusterTermVectors">commercials_subject (25), su_lc, ldk5, lma_long (100), lkl, lnlm, linst, location_normalised, llang, su_dk, lddc, llcc, mesh, cluster, ludk, sort_title, lsu_oai, ldbk, su_corp, series_normalised, lfn, author_normalised, su_pe, lcsa</entry>
    <!--    <entry key="ClusterTermVectors">author_normalised, sort_title, location_normalised, series_normalised, llcc, lnlm, ldbk, ludk, lddc, lkl, lfn, linst, ldk5, lma_long (200), su_pe, su_corp, su_lc, mesh, su_dk, lsu_oai, llang, cluster, commercials_subject</entry>-->
    <entry key="Cluster_vector_exceptions_freetext">og, er, den</entry>
    <entry key="ClusterFields"></entry>
    <!--
    frbr: Add the facet "FRBR" and populate with
          tags (aka clusters) "author - title".
          WARNING: Don't use FRBR in its current
          incarnation, as it does not provide
          queries. Use manyInOne instead.
    type: Extract the types from shortformat
    creator: Extrach the creators from shortformat
    autoClusters: Experimental clusters, generated by Bolette and Toke
    -->
    <entry key="ClusterSpecialFacets">manyInOne, autoCluster</entry>
    <entry key="ClusterManyInOneKeys">sort_title, author_normalised</entry>
    <entry key="ClusterManyInOneFailOnMissingKey">false</entry>

    <!--
    INSERT: Insertion order
    ALPHA:  Alpha numerical order
    REVERSEALPHA: Reverse alpha
    SCORE:  Score, from the highest to the lowest
    REVERSESCORE: Reverse score
    CUSTOM: Order specified by Cluster*SortCustom
    REVERSECUSTOM: Reverse custom
    SIZE: The size of Facet (the number of contained tags) or the size of the Tag (the number of contained objects)
    -->
    <entry key="ClusterFacetSort">CUSTOM, SIZE</entry>
    <entry key="ClusterFacetSortCustom">author_normalised, location_normalised, series_normalised</entry>
    <entry key="ClusterTagSort">SCORE, ALPHA</entry>
    <entry key="ClusterTagSortCustom"></entry>

    <!-- ******************************************************************* -->
    <!-- Complete cluster -->
    <!-- ******************************************************************* -->

    <!-- Where to store the map of all the wanted facets and tags in the
         index, meant for complete cluster processing. -->
    <entry key="CompleteClustermapLocation">/tmp/completeClustermap</entry>

    <!-- The number of threads used for building the real-time cluster
         presentation structure -->
    <entry key="ClusterMapThreads">2</entry>

    <!-- The tag handler is responsible to translate the internal references to
         tag names. Valid values are
         MemoryTagHandler: Loads all tagnames in memory. Fast, but requires
                           a lot of memory (ca. ½ GB / million documents).
         DiskTagHandler:   Performs lookup on disk. Relies on disk-cache for
                           quick response-time. Worst-case (if no tags names are
                           cached) is about 1 extra second.
                           It requires 8 * the number of tags bytes in RAM.
                           The number of tags is (_very_ roughly) 4-5 million /
                           1 million documents.
                           Note that indexes for the tag names needs to be build
                           each time the taglists have changed. This is done
                           automatically and tages about 2 minutes / 1 million
                           documents.-->
    <entry key="TagHandler">DiskTagHandler</entry>

    <!-- ******************************************************************* -->
    <!-- Cluster creation -->
    <!-- ******************************************************************* -->

    <!-- cluster map location used for testing -->
    <entry key="ClusterMapLocation">/tmp/auto_clusters_dist_based.map</entry>

    <!-- The fields to use as facets for cluster creation (initial clusters) -->
    <entry key="CCFields">su_pe|su_corp|su_lc|mesh|su_dk|lsu_oai</entry>
    <!-- <entry key="CCFields">subject_controlled|subject_other|subject_dk</entry> -->
    <!-- The fields not to use as facets for cluster creation -->
    <entry key="CCNegativeFields"></entry>
    <!-- The terms to use as tags for cluster creation -->
    <entry key="CCTerms"></entry>
    <!-- The terms not to use as tags for cluster creation -->
    <!-- qzwq is a delimiter -->
    <entry key="CCNegativeTerms">[0-9]+[.]?-? ?[0-9]+|[0-9]|og|view|and|of|the|.|new|with|on|to|etc|den|for|in|von|qzwq|til|med|om|af|på|det|er|as|ii|ud|no|st|se|na|al|is|man|it</entry>

    <!-- The minimum number of times that the terms has to occur in the
         index for a given field, in order to be considered tag candidates -->
    <entry key="CCMinTermSum">15</entry><!-- TODO test 100 / small clusters 25 -->
    <!-- The minimum number of times that the terms has to occur in the
         index for a single field_X, in order to be considered tag candidates.
         If collapse is false, this should be equal to CCMinTermSum. -->
    <entry key="CCMinTermLocalSum">15</entry><!-- TODO 100 / 25 -->
    <!-- The maximum number of times that the terms must occur in the
         index, in order to be considered tag candidates -->
    <entry key="CCMaxTermSum">99999</entry><!-- TODO used to be 999999 -->

    <!-- Note: the following settings are not used in simple cluster creation -->

    <!-- The minimum number of unique Terms that a Field must
         have in ordet to be tagged as valid -->
    <entry key="CCMinTermCount">1</entry>
    <!-- The maximum number of unique Terms that a Field can
         have in ordet to be tagged as valid -->
    <entry key="CCMaxTermCount">99999</entry>
    <!-- Stop the searching for clusters in a specific Field, if no candidates
         has been found after searching this number of terms -->
    <entry key="CCSkipAfterXWithNoHits">1000000</entry>
    <!-- Reduce the list of candidates to this number after searching. The most
         promising candidates are kept; applies to each considered field (CCFields) -->
    <!-- Remember to set this high in production -->
    <entry key="CCReduceTo">300</entry>
    <!-- Collapse fields like creator_1 and creator_2 to creator. This is not
         needed in the current index (20060626) -->
    <entry key="CCCollapseFields">false</entry>

    <!-- The maximum number of documents, that are used for calculating a
         centroid for a given cluster candidate -->
    <!-- This should probably be a fairly large number in production -->
    <!-- Note: currently not used in distance based creation -->
    <!-- Still used in 'first generation' cluster creation-->
    <entry key="CCMaxPointsInCentroid">67</entry><!-- TODO used to be 6700... -->

    <!-- The maximum size (number of terms) that a centroid can have after
         a reduce has been performed. This is a performance tweak. Lower
         values = better performance and worse approximation to the ideal
         result. -->
    <entry key="CCMaximumCentroidReduceSize">300</entry>
    <!-- The maximum size (number of terms) that a centroid can have after it
         has been build -->
    <entry key="CCMaximumCentroidFinalSize">50</entry>

    <!-- The normalize average number of usages for a given term needs to be
         above this number (tricky: We probably need to experiment a lot to
         get the right value here; OR since CCMaximumCentroidFinalSize is set
         low we can simply ignore this value (set it to zero) -->
    <entry key="CCMinimumCentroidEntrySize">0.01</entry>

    <entry key="CCOptimizeCentroidEveryXPoints">500</entry>

    <!-- This should probably be a very large number in production -->
    <!-- Note: currently not used in distance based creation -->
    <!-- Is used in simple clusters (getSimpleClusters) -->
    <entry key="CCMaxClusterSize">80000</entry><!-- TODO used to be 200000 -->
    <!-- NEW Thu Sep 07 -->
    <entry key="CCMaxClusterSizeFractionOfSearchResult">.6</entry>
    <!-- NEW Thu Sep 07 -->
    <entry key="CCScoreCutoffForClusterMembersFractionOfTopScore">.2</entry>
    <!-- Documents need to have at least this score, to be considered part of
         a cluster -->
    <!-- Note: not used in distance based creation -->
    <entry key="CCScoreCutoffForClusterMembers">1</entry>
    <!-- 1: out of memory (512MB) ved 2977/4233 -->

    <!-- Store the ClusterMap on the file system for every X clusters created
         Note: currently not used in distance based creation
         Note: set to zero for no check point saves (used to be 50) -->
    <entry key="CCCheckpointEveryXClusters">0</entry>

    <!-- NEW PARAMETERS USED IN DISTANCE BASED CLUSTERING -->

    <!-- Fields used for document vectors (inclusive list) -->
    <!-- no longer used in vectors: author_normalised|author_coorporation|author_person -->
    <entry key="FieldsUsedInVectors">su_pe|su_corp|su_lc|mesh|su_dk|lsu_oai|title|main_titel|subject_controlled|subject_dk5|no</entry>
    <!-- Fields NOT used for document vectors (exclusive list) -->
    <entry key="VectorNegativeFields"></entry>

    <!-- Should the points (corresponding to the documents) be normalised when
         added to the incremental centroid? -->
    <entry key="NormalisePointsWhenAddingToCentroid">false</entry><!--TODO: true ?-->

    <!-- How many (fraction) of the original cluster members (search based cluster)
         should be inside the defined cluster diameter? -->
    <entry key="ClusterDiameterCut">.6</entry>

    <!-- How distant should the (normalised) centroids of two different clusters be?
         If two centroids are closer than this distance,
         the two corresponding clusters are joined. -->
    <entry key="MinimumDistanceBetweenCentroids">.01</entry>

    <!-- How distant should the centroids of two different clusters
         with the same name be? -->
    <entry key="MinimumDistanceBetweenCentroidsWithTheSameName">1</entry>

    <!-- Should the original cluster members be kept in the cluster though,
         though they are not within the defined cluster diameter? -->
    <entry key="KeepInitialClusterMembers">true</entry>

    <!-- TODO: tidy up cluster.properties! -->

    <!-- Location of temporary cluster map used by ParallelIndexCreator;
         the directory is assumed to exists!!! -->
    <entry key="tmpCCmap">/tmp/tmpCCmaps/map</entry>

    <!-- Should parallel index building be resumed?
         If false, ClusterAndParallelIndexCreator is run from the beginning. -->
    <entry key="resumeCC">false</entry>

    <!-- Which cluster creation algorithm should be used?
         Currently the choices are 'FG' (First Generation), 'DB'
         (Distance Based) and 'simple'. The default is 'simple'.-->
    <entry key="CCalgorithm">simple</entry>

</properties>


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

<title>1002 - The One Where Ross Is Fine</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

1002 - The One Where Ross Is Fine

Written by: Sherry Bilsing-Graham & Ellen Plummer

  Directed by: Ben Weiss

  Transcribed by: Kreidy

[Scene: Joey and Rachel's apartment. The scene starts where we took off in
the last episode with Ross seeing Joey and Rachel kissing. Rosss tares at them.]

Joey: Okay, Ross, I realise that you didn't expect to walk in and see  that, but.. Let me explain, okay?

Rachel: We weren't doing anything!

Joey: Rach, he just saw us.

Rachel: Shhh.

Joey: But what you saw, that is the extent of it, okay? One kiss.

Rachel: No, come on, that is a lie. We also kissed in Barbados.

Joey (to Rachel): Dude, chill! (to Ross) Okay, we also kissed in  Barbados,
but we didn't plan it, okay? And the only reason that that happened was because
I saw you kissing Charlie.

Rachel: Yeah, you started it! I've got to chill.

Joey: Look, we probably should have talked to you about this before it  ever happened, but..

Rachel: We feel so terrible about this, Ross.

Joey: Yeah, but it did happen, so...

(Ross looks shocked and says nothing.)

Joey: Ross?

Rachel: Ross? (to Joey) Can we just close the door?

  OPENING CREDITS

  [Scene: Joey and Rachel's apartment. Continued from earlier.]

Rachel: Ross, say something. Anything.

Ross: So you two are..?

Joey and Rachel: Yeah.

Ross: And have you .. ed?

Joey: No, no, no!

Rachel: No, no, no!

Ross: But if I hadn't walked in here, would you..?

Joey: Probably.

(Rachel looks at him.)

Joey: No, no!

Rachel: Ross, this is not how we wanted you to find out about this.  You have every right to go nuts.

Ross: I'm not going nuts. Do you see me go nuts?

Rachel: No, but you know what I mean.

Ross: Hey, hey, hey... If you two are happy, then I'm happy for you.  (Squeaky.) I'm fine!

Joey: Really?

Ross: Absolutely. (Very Squeaky.) I'm fine! Totally fine. I don't know  why
  it's coming out all loud and squeaky, 'cause really, (deep voice)  I'm fine.
  I'm not saying I wasn't a little surprised to see you guys  kissing. I mean,
at first I was like.. (Screams.) But now that I've had time to absorb it; Lovin'
this.

Joey: Ross..

Ross: It's all working out! Me & Charlie, and you two. You know what
  we should do?

Rachel: Calm ourselves?

Ross: No. We should all have dinner. Yes, we'll do it tomorrow night.
  I'll cook!!

Joey: Look, don't you think that will be a little weird?

Ross: Weird? What? What's weird? The only thing weird would be if
  someone didn't like Mexican food, because I'm making fajitas!! (Storms
  off.)

Joey: I do like fajitas.

  [Scene: Central Perk. Monica and Chandler are there. They have lots of
  brochures about adoption in front of them.]

Monica: God, this adoption stuff is so overwhelming. There's
  inter-country adoption, dependency adoption.. There are so many ways
  to go, and this is like the biggest decision of our lives.

Chandler: There's a hair in my coffee.

(Phoebe enters.)

Phoebe: Hey guys!

Monica: Hey.

Phoebe: Hey, have you seen Frank Jr., 'cause he's meeting me here with
  the triplets.

Chandler: You know, it's funny. Every time you say "triplets," I
  immediately think of three hot blonde 19-year olds.

(Monica glares at him.)

Monica: That's sweet. Drink your hair.

Phoebe: Hey, what's all this stuff?

Monica: Oh, they're brochures from different adoption agencies.

Phoebe: Ooh, babies! Oh, this one is so cute, get this one!

Monica: That's not really how it works.

Phoebe: Oh, how does it work?

Monica: I don't know!

Phoebe: Well, if you're having a hard time, you should talk to my
  friends, Bill and Colleen. They adopted a kid. I'm sure they'd help
  you.

Monica: Thanks, that would be great. Hey, honey, wouldn't that be
  great?

(Chandler looks like he did the time he swallowed the toy in 605 TOW
  Joey's Porsche. It's the hair in his coffee.)

(Frank Jr. and the triplets enter.)

Frank Jr.: Alright, alright, alright. Remember what we talked about.
  When we're in a public place, there are certain rules.

(The triplets scream and run amok in the coffeehouse.)

Frank Jr.: That's not what we talked about!!

Phoebe: Hey!

Frank Jr.: Hey.

Phoebe: Good to see you.

Frank Jr.: Good to see you, too.

Monica: Hi Frank.

Frank Jr.: Hi, how you doin'?

Monica: Oh, my goodness, they've all gotten so big!

(Little Chandler is pulling Chandler's sweater, while Leslie is
  throwing bagels at him.)

Monica: Which one is which again?

Frank Jr.: Oh, that's Frank Jr. Jr. pulling the tampons out of the
  lady's purse. And that's Chandler climbing on Chandler, and that's
  Leslie throwing bagels at him.

Monica (reads a form in her lap): "Willing to adopt triplets?" No!

  [Scene: The hallway in Ross's building. Joey and Rachel are on their
  way to Ross's dinner.]

Joey: Ah, can I just say I know we're doing this for Ross, and that's
  cool, but if it was up to me, this is not what we'd be doing on our
  first date.

Rachel: Well, what would we be doing?

Joey: I'd take you out for a romantic night. Some champagne, fancy
  dinner, feel you up on the carriage ride home...

Rachel: Feel me up?

Joey: In a carriage!

(Charlie walks up to them.)

Joey: Hey, Charlie!

Rachel: Hey.

Charlie: Hi, hi. So.. Dreading this?

Rachel: Oh, you bet.

(Joey sees that she's carrying a small red bag.)

Joey: So, did you bring a little something for Ross?

Charlie: Actually.. It's stuff you left at my apartment.

Joey: Oh. Oh, thanks.

Charlie: And you know, you can just give me my stuff whenever you
  want.

Joey: Yeah, I didn't throw any of that out..

(They enter Ross's apartment. Ross is already quite hyper.)

Ross: I thought I heard voices! Hi Charlie! (Kisses her.) Hi Joey.
  (Hugs him.) And.. Oh! You're gonna have to introduce me to your new
  girlfriend. (Laughs.) I'm just kidding, I know Rachel, I know. (He
  squeezes her hand.) Come, please come in. Come in.

Rachel: Okay, well, we brought you some wine.

Ross: Oh! That is so thoughtful. (To Joey.) She's a keeper. And what
  did you bring me? (Grabs the bag that Charlie brought for Joey.)

Joey: Uh, actually, that's..

Ross: Underwear, a toothbrush, and Van Halen CD. I can use all these
  things!!

Charlie: Gosh, Ross, you know, you seem a little...

Ross: What? Fine? Because I am! Aren't you? Aren't you? Aren't you?
  You see? Who else is fine?

Joey: Okay, listen, hey, Ross. Why don't you try to relax, okay? Maybe
  have a drink.

Ross: You know what? That is a very good idea. I'm gonna go make a
  pitcher of Margaritas.

(Does a Mexican dancing-thing before going to the kitchen.)

  [Scene: Central Perk. Frank Jr., Phoebe and the triplets are there.
  The triplets are now sleeping on top of each other on the couch.]

Phoebe: Oh, god. So adorable. Look at them sleeping there like angels.

Frank Jr.: Yeah, I really cherish these moments, 'cause before you
  know it, they're gonna be awake again.

Phoebe: Well, they may be a handful, but they're so cute.

Frank Jr.: Yeah.

Phoebe: Oh, god, the last time I babysat them, they did the funniest
  thing..

Frank Jr.: I haven't slept in four years!

Phoebe: That's a, that's a long time.

Frank Jr.: You just don't know how hard it is, Phoebe. There's just so
  many of them. You know, two I can handle. Two's great. You just hold
  one in each hand, but what do I do when the third one runs at me with
  his bike helmet on. I've got no more hands to protect my area! There's
  three of them, Phoebe, three!

Phoebe: Yeah, I know, Frank. I counted them when they were coming out
  of my area.

Frank Jr.: Sometimes I think that.. Oh, no, no, no, I can't say it,
  it's too horrible. No.

Phoebe: What?

Frank Jr.: No, I can't.

Phoebe: Oh my god, Frank, are you thinking of leaving? Because I
  didn't have those triplets so you could just run out on them!

Frank Jr.: Oh, no! I would never do that. No. I just was thinking
  that, you know, maybe you could take one.

Phoebe: What?! You can't separate them! That's terrible. Which one?

  [Scene: Bill and Colleen's apartment. Chandler and Monica enter.]

Monica: Thank you so much for seeing us. Phoebe has told us such great
  things about you guys.

Colleen: Oh, please, we're happy to help.

Bill: We went through the same thing when we were adopting.

Chandler: So, a lot of malfunctioning wee-wees and hoo-hoos in this
  room, huh?

(Bill and Colleen look shocked.)

Chandler: I mean, you have a lovely home.

Monica: Well, we appreciate anything you can tell us.

Colleen: Well, actually, I think this might help.

(She gives Monica a big binder that's perfectly in order.)

Colleen: It's pretty much all the information you need.

Monica: Oh my god!

Colleen: Everything is broken down into categories, and then
  cross-referenced, and then colour-coded to correspond with the forms
  in the back.

Monica: Thank you. (To Chandler.) I think I just had a tiny orgasm.

Bill: I know the process is frustrating, but it's so worth it.
  Adopting Owen was the best thing that ever happened to us.

Chandler: That's great. (To Monica.) Can I see the book?

(Monica looks at him.)

Chandler: You want me to wash my hands first, don't you?

Monica: It's.. It's just so pretty and white.

Colleen: The bathroom is down the hall, to your left.

(Chandler leaves.)

Colleen: I would have told him to do it too.

Monica: Can I adopt you?

(Cut to the hall. Owen is wearing his scout-uniform and is looking
  through a box when Chandler walks up to him.)

Chandler: Hey, you must be Owen.

Owen: Yeah.

Chandler: I'm Chandler. Hey, I was in the scouts too.

Owen: You were?

Chandler: Yeah, in fact my father was a den-mother.

Owen: Huh?

Chandler: You know how to use a compass?

Owen: I have a badge in it.

Chandler: You do? That's fantastic!

Owen: You wanna see it?

Chandler: I'd love to, but I gotta get back to talking to your
  parents. They're telling us all about how they adopted you.

Owen: What?!?

Chandler: What?

Owen: I'm adopted?

(Chandler tries to come up with a good answer.)

Chandler: I got nothing.

  [Scene: Ross's apartment. Ross is totally wasted, but he's still
  drinking all the margaritas.]

Ross: The first batch of margaritas was not so great, but the second
  batch is gooooood.

Rachel: Well, maybe the next batch, we could all get some.

Ross: Oh, guys, this is fun, isn't it? You know? Just the four of us.
  Just hangin'.

Joey: Dude, are you okay? And when are the fajitas gonna be ready?

Ross: I'm fine! Hey, I'm great! I'm just.. I'm just proud of us.
  There's no weirdness, no tension.

Rachel: No awareness.

Ross: We make a great foursome. We should do more stuff together. Ooh!
  Let's take a trip. Okay, where do you think we - we can go?

(The oven timer pings in the kitchen.)

Ross: My fajitas!!

(He runs off to the kitchen.)

Rachel: Look, Charlie, I just want you to know. Ross is just having a
  little trouble adjusting to the thought of Joey and me. You know, he
  normally doesn't drink like this.

Charlie: Oh, you know what? This is nothing. My father is a raging
  alcoholic.

(Joey and Rachel don't know how to respond to that.)

Charlie: Oh, I'm sorry, have I made this evening uncomfortable?

(Ross enters carrying a frying pan with fajitas - without any oven
  mitts.)

Ross: Fajitas! Be careful, very hot plate, very hot plate!!

Rachel: Ross, you don't even have oven mitts on!

(Ross laughs.)

Ross: That is gonna hurt tomorrow!

  [Scene: Central Perk. Frank Jr., Phoebe and the triplets are there.
  Continued from earlier.]

Phoebe: Okay, well, this is crazy. Can't seriously be talking about me
  taking one of your kids, can we?

Frank Jr.: No, of course we're not.

Phoebe: Insane.

Frank Jr.: I know.

Phoebe: Alice would never go for it, right?

Frank Jr.: Oh, I don't know, she's pretty tired, too, I think we've
  got her onboard.

Phoebe: Well, just you know, for argument's sake, you know,
  hypothetically. Which one would you be willing to give up?

Frank Jr.: Huh.

Phoebe: Frank Jr.Jr.?

Frank Jr.: Oh, you'd be getting a really good one. I mean, you know,
  he's really funny. Like, the other day he made up this joke.

Phoebe: Uh-huh.

Frank Jr.: What's green and says "hey, I'm a frog"? A talking frog!
  (Laughs.) Oh, no, you can't have him, he's too funny.

Phoebe: Well, alright, that's fine. What about Leslie?

Frank Jr.: Oh, no, no. Not Leslie. No, she's, she's the only one that
  knows how to burp the alphabet.

Phoebe: Alright, so that leaves Chandler.

Frank Jr.: Oh, no, no, you can't have Chandler, no. No, no. She's my
  little genius. I got big hopes for her. She's gonna be a doctor or a
  realtor..

Phoebe: Wow, Frank. I think we just ran out of kids.

Frank Jr.: Oh, I think you're right. Oh, wow. Phoebe, I don't think I
  can give one of them up. I mean, you know, they drive me crazy, but
  they're my babies.

Phoebe: I'm sorry, Frank. I didn't realise things were so bad. You
  know, I'll help out more. I can - I can babysit any time you want. You
  name the day, and I'll be there.

Frank Jr.: How about tomorrow?

Phoebe: Well, that's not good. But you know, I can move some stuff
  around, and I'll be there. You and Alice just take the whole day
  together.

Frank Jr.: You'd do that for us?

Phoebe: Are you kidding? That's what sisters are for.

Frank Jr. (looks at the triplets): Look at them! Aw. I love you so
  much. (Strokes Leslie's hair, and she moves a little.) Oh crap, don't
  wake up, don't wake up!

  [Scene: Bill and Colleen's apartment. Chandler comes running into the
  living room. Monica is the only one there.]

Chandler: Where are Bill and Colleen?

Monica: They're in the kitchen getting something to eat. Can you
  believe how nice they are?

Chandler: We have to leave!!

Monica: Why? What did you do in the bathroom?

Chandler: I didn't get to the bathroom. I bumped into Owen on the way,
  and he didn't know he was adopted. And there's a slight chance I may
  have told him.

Monica: Oh my god, where's my purse? No, you know what? I can replace
  everything in there. Get that binder, and let's go!

(Bill and Colleen enter.)

Colleen: Hey.

Bill: Some little snacks for everybody. Oh, you don't have to eat the
  sour worms. Those are for Owen.

Colleen: I'll go get him in a second. By the way, you should know we
  haven't told him he's adopted yet.

Chandler: But kids are so intuitive. Don't you think on some level he
  already knows?

(Owen comes running in.)

Owen: I'm adopted?!

Chandler: See? Intuitive!

Bill: What? Where did you hear that?

Owen: He told me! And he paid me 50 dollars not to tell.

Chandler: Which technically now you should give back!

Colleen: You told him he's adopted?

Chandler: I'm so sorry, but you should have a sign out there or
  something. Or at least whisper it to people when they come in the
  door. "Owen doesn't know he's adopted, and he also thinks that Santa
  is real."

Owen: He isn't?!

Chandler (to Monica): We have to get out of here, baby!

  [Scene: Ross's apartment. Rachel, Joey and Charlie are eating fajitas
  when Ross enters from the kitchen.]

Ross: Everyone? I would like to make a toast to Rachel and Joey.

Rachel: Ooy.

Ross: And to love. Ah, love. L-O-V-E, love. L is for life. And what is
  life without love?

Rachel: Oh my god, are we supposed to answer?

Ross: O is for "oh, wow!" The V is for this very surprising turn
  of
  events, which I'm still fine with by the way. E is for how extremely
  normal I find it. That you two are together. And now one day you might
  get married and have children of your own.

(Ross chokes up and pauses. Rachel and Joey look at him.)

Joey: Dude, are you okay?

Ross: Totally.

Rachel: Ross, you don't seem okay.

Ross (on the verge of tears): I'm sorry, it must be the pressure of
  entertaining. I think everyone would feel better if we had some flan.

Charlie: Wait, Ross. Ross. I - I have to take off.

Ross: No!

Charlie: I'm sorry, I have a really early class in the morning, but
  this has been lovely.

Ross: Wasn't it? And you thought it would be awkward with Joey and
  that you never really liked Rachel.

Charlie: You're on fire! I'll call you in the morning, okay?

Ross: Okay.

Charlie: Alright.

(Ross goes to the kitchen.)

Charlie: God, Rachel, what Ross just said that is just so..

Rachel: Oh, that's okay, girls tend not to like me.

Charlie: Bye.

(Ross enters from the kitchen with three plates with flan.)

Ross: Okay, I guess it's just flan for three! Hey, hey, that rhymed!

Rachel: You know what, Ross? I think we're gonna take off too.

Ross: Oh, oh. Of course. God, I'm so stupid. You guys are a couple
  now. I mean, you probably just want to be alone.

Rachel: No, no, it's just that it's getting late...

Ross: Hey, hey, it's fine. It's totally fine. We've got plenty of
  margaritas. It's all good.

(The oven timer pings again.)

Ross: I don't even know what that's for.

(He goes back to the kitchen.)

Joey: You know what? I think I'm gonna stay here and make sure he's
  okay.

Rachel: Yeah, that's probably a good idea.

Joey: Yeah. I'll see you in the morning.

Rachel: Uh-huh. Okay. You know what, Joey, I don't think he's ever
  gonna be okay with this.

Joey: It doesn't look good, does it?

(They kiss each other on the cheek, and Rachel leaves.)

  [Scene: Ross's apartment the next morning. Ross is very hung-over on
  the couch as Joey enters with a cup of coffee for him.]

Joey: Morning. Here you go.

Ross: Thanks. Did you stay here all night?

Joey: Yeah.

Ross: So you took off my pants and shoes?

Joey: No, no, no. You actually did that when you were dancing to the
  Chicago-soundtrack. Look, Ross, about, about Rachel and I. Listen, you
  don't have to worry about that, okay? Because nothing is gonna happen.

Ross: What do you mean?

Joey: Well, she and I said from the beginning that we weren't gonna do
  anything unless you were okay with it. And clearly..

Ross: Hey, what are you talking about? I'm fine!

Joey: It's okay, Ross, alright? I totally understand. Of course you're
  not fine. You're.. You're Ross and Rachel.

Ross: Except we're not. I mean, we haven't been a couple in like, six
  years. Oh my god, is that right? Has it been that long?

Joey: That's what I hear, yeah.

Ross: This is crazy. I mean, six years? And because of me you guys
  aren't gonna be together? Can I ask you something? Really, what is
  this thing with you and Rachel?

Joey: Come on, I mean, you know me, you know...

Ross: Joey.

Joey: I'm crazy about her.

Ross: And she feels the same way?

Joey: I think so.

Ross: Well, then, maybe it's time we all moved on.

Joey: Yeah, but, Ross, I mean, you're not okay with it.

Ross: No, but I wanna be. Hey, I will be. Besides, I'm with Charlie,
  right? Oh my god, I'm still with Charlie, aren't I? I mean, she didn't
  see the dance, did she?

Joey: No, no, no, no, that was - that was just for me. Are you sure
  about this?

Ross: Yeah, I'm sure.

Joey: And we're okay?

(Ross smiles and holds up his hand for a high-five, but he has
  forgotten about his burnt hands. He gasps in pain as Joey grabs his
  hand.)

  ENDING CREDITS

  [Scene: Central Perk. Monica and Chandler enter to find Phoebe there
  with the triplets.]

Monica: Hey, Phoebs

Phoebe: Hey.

Monica: Uhm, we just wanna give you a heads-up. Bill and Colleen hate
  us.

Chandler: Owen didn't know he was adopted, and Monica told him.

Monica: What?

Phoebe: Still, he had to find out sometime.

Chandler: Yeah, but how would you like it if someone told the triplets
  that you gave birth to them?

(The triplets stare at him.)

Chandler: I'm gonna go tell Emma she was an accident. (Runs off.)

THE END


text:
package main

import "testing"

func testHelper(t *testing.T, message string, command string, directOnly bool, isPersonal bool, argsAllowed bool, expectMatch bool, expectArgs string) {
	config := new(Configuration)
	config.directCommandsOnly = directOnly

	matched, args := checkCommandMatch(message, command, isPersonal, argsAllowed, config)

	if matched != expectMatch {
		t.Errorf("Matched %v != Expected Match %v", matched, expectMatch)
	}

	if args != expectArgs {
		t.Errorf("Returned Args '%v' != Expected Returned Args '%v'", args, expectArgs)
	}
}

func TestDirectOnlyBareCommandPresent(t *testing.T) {
	testHelper(t, "/ho", "/ho", true, false, false, false, "")
}

func TestDirectOnlyBareCommandPresentPersonalMessage(t *testing.T) {
	testHelper(t, "/ho", "/ho", true, true, false, true, "")
}

func TestBareCommandPresent(t *testing.T) {
	testHelper(t, "/ho", "/ho", false, false, false, true, "")
}

func TestBareCommandPresentWithArgs(t *testing.T) {
	testHelper(t, "/nose knows", "/nose", false, false, true, true, "knows")
}

func TestBareCommandPresentWithArgsAndSpace(t *testing.T) {
	testHelper(t, "/nose   knows yo   ", "/nose", false, false, true, true, "knows yo")
}

func TestBareCommandNotPresent(t *testing.T) {
	testHelper(t, "/yo", "/ho", false, false, false, false, "")
}


text:
User List

        <form action="" method="GET" class="form-inline">
        <input type="text" class="form-control" name="keyword" value="<?php echo $search_data['keyword'];?>">

        <button class="btn btn-success" type="submit" >Search</button>
        <a class="btn btn-primary" href="<?php echo site_url('admin/' . $self );?>"/>Reset
        <a class="btn btn-primary" href="<?php echo site_url('admin');?>"/>Return

        <input type="text" id="datetimepicker1" class="form-control" name="start_time" placeholder="Create_time start" value="<?php echo $search_data['start_time'];?>">
          <input type="text" id="datetimepicker2" class="form-control" name="end_time" placeholder="Create_time end" value="<?php echo $search_data['end_time'];?>">
      </form>

        ID
        username
        cellphone
        email
        create time

      <?php
      foreach ($users as $key => $user) {
        echo "
";
        echo "" . $user['id'] . "	";
        echo "" . $user['username'] . "	";
        echo "" . $user['cellphone'] . "	";
        echo "" . $user['email'] . "	";
        echo "" . $user['create_time'] . "	";
        echo "
";
      }
      ?>

      <?php foreach ($link_array as $key => $value) {echo $value;}?>

<link rel="stylesheet" href="/public/css/jquery.datetimepicker.css">


text:
<?php
if(isset($isAjax) && $isAjax == true):
	include $viewDir . $actionFile;
	return;
endif;
?>

<!doctype html>

<?php include($viewDir . 'elements/head.php'); ?>

	<?php include($viewDir . 'elements/header.php'); ?>
	<?php include $viewDir . 'framework/flashmessage.php' ?>

		<?php include $viewDir . $actionFile; ?>

	<?php include($viewDir . 'elements/footer.php'); ?>

	<?php include($viewDir . 'framework/javascript.php'); ?>

	<img style="position: fixed; top: 40px; left: 0; border: 0;" src="<?=$baseUrl;?>images/generic/fork-github.png" alt="Fork me on GitHub">


text:
package com.microsoft.azure.management.recoveryservices.backup.v2017_07_01;

import com.microsoft.azure.arm.model.HasInner;
import com.microsoft.azure.arm.resources.models.HasManager;
import com.microsoft.azure.management.recoveryservices.backup.v2017_07_01.implementation.RecoveryServicesManager;
import com.microsoft.azure.management.recoveryservices.backup.v2017_07_01.implementation.BackupStatusResponseInner;

/**
 * Type representing BackupStatusResponse.
 */
public interface BackupStatusResponse extends HasInner<BackupStatusResponseInner>, HasManager<RecoveryServicesManager> {
    /**
     * @return the containerName value.
     */
    String containerName();

    /**
     * @return the errorCode value.
     */
    String errorCode();

    /**
     * @return the errorMessage value.
     */
    String errorMessage();

    /**
     * @return the fabricName value.
     */
    FabricName fabricName();

    /**
     * @return the policyName value.
     */
    String policyName();

    /**
     * @return the protectedItemName value.
     */
    String protectedItemName();

    /**
     * @return the protectionStatus value.
     */
    ProtectionStatus protectionStatus();

    /**
     * @return the registrationStatus value.
     */
    String registrationStatus();

    /**
     * @return the vaultId value.
     */
    String vaultId();

}


text:
<component name="libraryTable">
  <library name="Maven: com.github.theborakompanioni:thymeleaf-extras-shiro:2.0.0">
    <CLASSES>
      <root url="jar://F:/maven_dep/repository/com/github/theborakompanioni/thymeleaf-extras-shiro/2.0.0/thymeleaf-extras-shiro-2.0.0.jar!/" />
    </CLASSES>
    <JAVADOC>
      <root url="jar://F:/maven_dep/repository/com/github/theborakompanioni/thymeleaf-extras-shiro/2.0.0/thymeleaf-extras-shiro-2.0.0-javadoc.jar!/" />
    </JAVADOC>
    <SOURCES>
      <root url="jar://F:/maven_dep/repository/com/github/theborakompanioni/thymeleaf-extras-shiro/2.0.0/thymeleaf-extras-shiro-2.0.0-sources.jar!/" />
    </SOURCES>
  </library>
</component>


text:
module Nokogiri
  module XML
    ###
    # Parse options for passing to Nokogiri.XML or Nokogiri.HTML
    class ParseOptions
      # Strict parsing
      STRICT      = 0
      # Recover from errors
      RECOVER     = 1 << 0
      # Substitute entities
      NOENT       = 1 << 1
      # Load external subsets
      DTDLOAD     = 1 << 2
      # Default DTD attributes
      DTDATTR     = 1 << 3
      # validate with the DTD
      DTDVALID    = 1 << 4
      # suppress error reports
      NOERROR     = 1 << 5
      # suppress warning reports
      NOWARNING   = 1 << 6
      # pedantic error reporting
      PEDANTIC    = 1 << 7
      # remove blank nodes
      NOBLANKS    = 1 << 8
      # use the SAX1 interface internally
      SAX1        = 1 << 9
      # Implement XInclude substitition
      XINCLUDE    = 1 << 10
      # Forbid network access
      NONET       = 1 << 11
      # Do not reuse the context dictionnary
      NODICT      = 1 << 12
      # remove redundant namespaces declarations
      NSCLEAN     = 1 << 13
      # merge CDATA as text nodes
      NOCDATA     = 1 << 14
      # do not generate XINCLUDE START/END nodes
      NOXINCNODE  = 1 << 15

      # the default options used for parsing XML documents
      DEFAULT_XML  = RECOVER
      # the default options used for parsing HTML documents
      DEFAULT_HTML = RECOVER | NOERROR | NOWARNING | NONET

      attr_accessor :options
      def initialize options = STRICT
        @options = options
      end

      constants.each do |constant|
        next if constant.to_sym == :STRICT
        class_eval %{
          def #{constant.downcase}
            @options |= #{constant}
            self
          end

          def #{constant.downcase}?
            #{constant} & @options == #{constant}
          end
        }
      end

      def strict
        @options &= ~RECOVER
        self
      end

      def strict?
        @options & RECOVER == STRICT
      end

      alias :to_i :options

      def inspect
        options = []
        self.class.constants.each do |k|
          options << k.downcase if send(:"#{k.downcase}?")
        end
        super.sub(/>$/, " " + options.join(', ') + ">")
      end
    end
  end
end


text:
Improved the string representation of objects related to ``Table.indices`` so
they now indicate the object type and relevant attributes.


text:
<?php
namespace ttempleton\nocache\twig;

use Twig\Compiler as TwigCompiler;
use Twig\Node\Node as TwigNode;
use Twig\Node\BodyNode as TwigBodyNode;
use Twig\Node\ModuleNode as TwigModuleNode;
use Twig\Source as TwigSource;

/**
 * Class Node_Module
 *
 * @package ttempleton\nocache\twig
 * @author Benjamin Fleming
 * @author Thomas Templeton
 * @since 2.0.0
 * @deprecated since 2.0.5, will be removed in 3.0
 */
class Node_Module extends TwigModuleNode
{
	protected $id;

	public function __construct(TwigNode $node, string $id, string $fileName)
	{
		// Pass in some empty objects to satisfy the required parameters
		parent::__construct(
			new TwigBodyNode([$node]),
			null,
			new TwigNode(),
			new TwigNode(),
			new TwigNode(),
			[],
			new TwigSource('', $fileName)
		);

		$this->id = $id;
	}

	/**
	 * @param TwigCompiler $compiler
	 */
	protected function compileClassHeader(TwigCompiler $compiler)
	{
		parent::compileClassHeader($compiler);
	}
}


text:
import string
import unittest
import datetime
import collections
from unittest import mock

from flumine.order.order import (
    BaseOrder,
    BetfairOrder,
    ExchangeType,
    OrderTypes,
    OrderStatus,
    VALID_BETFAIR_CUSTOMER_ORDER_REF_CHARACTERS,
    LIVE_STATUS,
    COMPLETE_STATUS,
)
from flumine.exceptions import OrderUpdateError

class BaseOrderTest(unittest.TestCase):
    def setUp(self) -> None:
        mock_client = mock.Mock(paper_trade=False)
        self.mock_trade = mock.Mock(
            client=mock_client, market_id="1.1", selection_id=123, info={}
        )
        self.mock_order_type = mock.Mock(info={})
        self.order = BaseOrder(
            self.mock_trade, "BACK", self.mock_order_type, 1, context={1: 2}
        )

    def test_init(self):
        self.assertIsNotNone(self.order.id)
        self.assertEqual(self.order.trade, self.mock_trade)
        self.assertEqual(self.order.side, "BACK")
        self.assertEqual(self.order.order_type, self.mock_order_type)
        self.assertEqual(self.order.selection_id, self.mock_trade.selection_id)
        self.assertEqual(self.order.handicap, 1)
        self.assertEqual(
            self.order.lookup,
            (self.order.market_id, self.order.selection_id, self.order.handicap),
        )
        self.assertIsNone(self.order.runner_status)
        self.assertIsNone(self.order.market_type)
        self.assertEqual(self.order.each_way_divisor, 1)
        self.assertIsNone(self.order.status)
        self.assertFalse(self.order.complete)
        self.assertEqual(self.order.status_log, [])
        self.assertIsNone(self.order.violation_msg)
        self.assertEqual(self.order.context, {1: 2})
        self.assertEqual(self.order.notes, {})
        self.assertIsNone(self.order.market_notes)
        self.assertIsNone(self.order.bet_id)
        self.assertIsNone(self.order.EXCHANGE)
        self.assertEqual(self.order.update_data, {})
        self.assertIsNone(self.order.publish_time)
        self.assertIsNone(self.order.market_version)
        self.assertIsNone(self.order.async_)
        self.assertIsNotNone(self.order.date_time_created)
        self.assertIsNone(self.order.date_time_execution_complete)
        self.assertFalse(self.order.simulated)
        self.assertFalse(self.order._simulated)
        self.assertEqual(
            LIVE_STATUS,
            [
                OrderStatus.PENDING,
                OrderStatus.CANCELLING,
                OrderStatus.UPDATING,
                OrderStatus.REPLACING,
                OrderStatus.EXECUTABLE,
            ],
        )
        self.assertEqual(
            COMPLETE_STATUS,
            [
                OrderStatus.EXECUTION_COMPLETE,
                OrderStatus.EXPIRED,
                OrderStatus.VIOLATION,
            ],
        )

    @mock.patch("flumine.order.order.BaseOrder._is_complete")
    @mock.patch("flumine.order.order.BaseOrder.info")
    def test__update_status(self, mock_info, mock__is_complete):
        self.mock_trade.complete = True
        self.order._update_status(OrderStatus.EXECUTION_COMPLETE)
        self.assertEqual(self.order.status_log, [OrderStatus.EXECUTION_COMPLETE])
        self.assertEqual(self.order.status, OrderStatus.EXECUTION_COMPLETE)
        self.mock_trade.complete_trade.assert_called()
        mock__is_complete.assert_called()

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_placing(self, mock__update_status):
        self.order.placing()
        mock__update_status.assert_called_with(OrderStatus.PENDING)

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_executable(self, mock__update_status):
        self.order.update_data = {123: 456}
        self.order.executable()
        mock__update_status.assert_called_with(OrderStatus.EXECUTABLE)
        self.assertEqual(self.order.update_data, {})

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_execution_complete(self, mock__update_status):
        self.order.update_data = {123: 456}
        self.order.execution_complete()
        mock__update_status.assert_called_with(OrderStatus.EXECUTION_COMPLETE)
        self.assertIsNotNone(self.order.date_time_execution_complete)
        self.assertEqual(self.order.update_data, {})

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_cancelling(self, mock__update_status):
        self.order.cancelling()
        mock__update_status.assert_called_with(OrderStatus.CANCELLING)

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_updating(self, mock__update_status):
        self.order.updating()
        mock__update_status.assert_called_with(OrderStatus.UPDATING)

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_replacing(self, mock__update_status):
        self.order.replacing()
        mock__update_status.assert_called_with(OrderStatus.REPLACING)

    @mock.patch("flumine.order.order.BaseOrder._update_status")
    def test_violation(self, mock__update_status):
        self.order.update_data = {123: 456}
        self.order.violation("the murder capital")
        mock__update_status.assert_called_with(OrderStatus.VIOLATION)
        self.assertEqual(self.order.update_data, {})
        self.assertEqual(self.order.violation_msg, "the murder capital")

    def test_place(self):
        with self.assertRaises(NotImplementedError):
            self.order.place(123, 456, False)

    def test_cancel(self):
        with self.assertRaises(NotImplementedError):
            self.order.cancel()

    def test_update(self):
        with self.assertRaises(NotImplementedError):
            self.order.update("PERSIST")

    def test_replace(self):
        with self.assertRaises(NotImplementedError):
            self.order.replace(20.0)

    def test_create_place_instruction(self):
        with self.assertRaises(NotImplementedError):
            self.order.create_place_instruction()

    def test_create_cancel_instruction(self):
        with self.assertRaises(NotImplementedError):
            self.order.create_cancel_instruction()

    def test_create_update_instruction(self):
        with self.assertRaises(NotImplementedError):
            self.order.create_update_instruction()

    def test_create_replace_instruction(self):
        with self.assertRaises(NotImplementedError):
            self.order.create_replace_instruction()

    def test_update_current_order(self):
        mock_current_order = mock.Mock()
        self.order.update_current_order(mock_current_order)
        self.assertEqual(self.order.responses.current_order, mock_current_order)

    def test_current_order(self):
        self.assertIsNone(self.order.current_order)
        mock_responses = mock.Mock()
        mock_responses.current_order = None
        self.order.responses = mock_responses
        self.assertEqual(self.order.current_order, mock_responses.place_response)
        mock_responses.current_order = 1
        self.assertEqual(self.order.current_order, 1)

    @mock.patch("flumine.backtest.simulated.config")
    def test_current_order_simulated(self, mock_config):
        mock_config.simulated = True
        order = BaseOrder(mock.Mock(), "", mock.Mock())
        self.assertTrue(order.simulated)
        self.assertTrue(order._simulated)

    def test__is_complete(self):
        self.order.status = None
        self.assertFalse(self.order._is_complete())
        for s in [
            OrderStatus.PENDING,
            OrderStatus.CANCELLING,
            OrderStatus.UPDATING,
            OrderStatus.REPLACING,
            OrderStatus.EXECUTABLE,
        ]:
            self.order.status = s
            self.assertFalse(self.order._is_complete())
        for s in [
            OrderStatus.EXECUTION_COMPLETE,
            OrderStatus.EXPIRED,
            OrderStatus.VIOLATION,
        ]:
            self.order.status = s
            self.assertTrue(self.order._is_complete())

    def test_average_price_matched(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.average_price_matched

    def test_size_matched(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.size_matched

    def test_size_remaining(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.size_remaining

    def test_size_cancelled(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.size_cancelled

    def test_size_lapsed(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.size_lapsed

    def test_size_voided(self):
        with self.assertRaises(NotImplementedError):
            assert self.order.size_voided

    def test_elapsed_seconds(self):
        self.assertIsNone(self.order.elapsed_seconds)
        mock_responses = mock.Mock()
        mock_responses.date_time_placed = datetime.datetime.utcnow()
        self.order.responses = mock_responses
        self.assertGreaterEqual(self.order.elapsed_seconds, 0)

    def elapsed_seconds_created(self):
        self.assertGreaterEqual(self.order.elapsed_seconds_created, 0)

    def test_elapsed_seconds_executable(self):
        self.assertIsNone(self.order.elapsed_seconds_executable)
        mock_responses = mock.Mock()
        mock_responses.date_time_placed = datetime.datetime.utcnow()
        self.order.responses = mock_responses
        self.order.date_time_execution_complete = datetime.datetime.utcnow()
        self.assertGreaterEqual(self.order.elapsed_seconds_executable, 0)

    def test_market_id(self):
        self.assertEqual(self.order.market_id, self.mock_trade.market_id)

    def test_lookup(self):
        self.assertEqual(
            self.order.lookup,
            (self.mock_trade.market_id, self.mock_trade.selection_id, 1),
        )

    def test_repr(self):
        self.assertEqual(repr(self.order), "Order None: None")

    def test_set_and_get_sep(self):
        self.order.sep = "a"
        self.assertEqual("a", self.order.sep)

    def test_customer_order_ref(self):
        self.order.trade.strategy.name_hash = "my_name_hash"
        self.order.id = 1234
        self.assertEqual("my_name_hash-1234", self.order.customer_order_ref)

        self.order.sep = "I"
        self.assertEqual("my_name_hashI1234", self.order.customer_order_ref)

        self.order.sep = "O"
        self.assertEqual("my_name_hashO1234", self.order.customer_order_ref)

    def test_notes_str(self):
        self.order.notes = collections.OrderedDict({"1": 1, 2: "2", 3: 3, 4: "four"})
        self.assertEqual(self.order.notes_str, "1,2,3,four")
        self.order.notes = collections.OrderedDict()
        self.assertEqual(self.order.notes_str, "")

class BetfairOrderTest(unittest.TestCase):
    def setUp(self) -> None:
        mock_client = mock.Mock(paper_trade=False)
        self.mock_trade = mock.Mock(
            client=mock_client, market_id="1.1", selection_id=123, info={}
        )
        self.mock_status = mock.Mock()
        self.mock_order_type = mock.Mock(info={}, size=2.0, liability=2.0)
        self.order = BetfairOrder(self.mock_trade, "BACK", self.mock_order_type)

    def test_init(self):
        self.assertEqual(self.order.EXCHANGE, ExchangeType.BETFAIR)

    @mock.patch("flumine.order.order.BetfairOrder.placing")
    def test_place(self, mock_placing):
        self.order.place(123, 456, False)
        mock_placing.assert_called_with()
        self.assertEqual(self.order.publish_time, 123)
        self.assertEqual(self.order.market_version, 456)
        self.assertFalse(self.order.async_)

    @mock.patch(
        "flumine.order.order.BetfairOrder.size_remaining",
        new_callable=mock.PropertyMock,
    )
    @mock.patch("flumine.order.order.BetfairOrder.cancelling")
    def test_cancel(self, mock_cancelling, mock_size_remaining):
        mock_size_remaining.return_value = 20
        self.order.bet_id = 123
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.cancel(12)

        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.order.cancel(0.01)
        self.assertEqual(self.order.update_data, {"size_reduction": 0.01})
        mock_cancelling.assert_called_with()
        self.order.cancel()
        self.assertEqual(self.order.update_data, {"size_reduction": None})

    def test_cancel_bet_id(self):
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.cancel(12)

    @mock.patch(
        "flumine.order.order.BetfairOrder.size_remaining",
        new_callable=mock.PropertyMock,
    )
    @mock.patch("flumine.order.order.BetfairOrder.cancelling")
    def test_cancel_error_size(self, mock_cancelling, mock_size_remaining):
        mock_size_remaining.return_value = 20
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.cancel(12)

        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        with self.assertRaises(OrderUpdateError):
            self.order.cancel(21)

    @mock.patch(
        "flumine.order.order.BetfairOrder.size_remaining",
        new_callable=mock.PropertyMock,
    )
    def test_cancel_error(self, mock_size_remaining):
        mock_size_remaining.return_value = 20
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.order.status = OrderStatus.PENDING
        with self.assertRaises(OrderUpdateError):
            self.order.cancel(12)

    @mock.patch("flumine.order.order.BetfairOrder.updating")
    def test_update(self, mock_updating):
        self.order.bet_id = 123
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.update("PERSIST")

        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.mock_order_type.persistence_type = "LAPSE"
        self.order.update("PERSIST")
        self.assertEqual(self.mock_order_type.persistence_type, "PERSIST")
        mock_updating.assert_called_with()

        with self.assertRaises(OrderUpdateError):
            self.order.update("PERSIST")

    def test_update_bet_id(self):
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.update("PERSIST")

    def test_update_error(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.mock_order_type.persistence_type = "LAPSE"
        self.order.status = OrderStatus.PENDING
        with self.assertRaises(OrderUpdateError):
            self.order.update("PERSIST")

    @mock.patch("flumine.order.order.BetfairOrder.replacing")
    def test_replace(self, mock_replacing):
        self.order.bet_id = 123
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.replace(1.01)

        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.mock_order_type.price = 2.02
        self.order.replace(1.01)
        self.assertEqual(self.order.update_data, {"new_price": 1.01})
        mock_replacing.assert_called_with()

        with self.assertRaises(OrderUpdateError):
            self.order.replace(2.02)

    def test_replace_bet_id(self):
        self.order.status = OrderStatus.EXECUTABLE
        with self.assertRaises(OrderUpdateError):
            self.order.replace(1.01)

    def test_replace_error(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.order.status = OrderStatus.PENDING
        with self.assertRaises(OrderUpdateError):
            self.order.replace(1.52)

    def test_create_place_instruction(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.assertEqual(
            self.order.create_place_instruction(),
            {
                "customerOrderRef": self.order.customer_order_ref,
                "handicap": 0,
                "limitOrder": self.mock_order_type.place_instruction(),
                "orderType": "LIMIT",
                "selectionId": self.mock_trade.selection_id,
                "side": "BACK",
            },
        )
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT_ON_CLOSE
        self.assertEqual(
            self.order.create_place_instruction(),
            {
                "customerOrderRef": self.order.customer_order_ref,
                "handicap": 0,
                "limitOnCloseOrder": self.mock_order_type.place_instruction(),
                "orderType": "LIMIT_ON_CLOSE",
                "selectionId": self.mock_trade.selection_id,
                "side": "BACK",
            },
        )
        self.mock_order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
        self.assertEqual(
            self.order.create_place_instruction(),
            {
                "customerOrderRef": self.order.customer_order_ref,
                "handicap": 0,
                "marketOnCloseOrder": self.mock_order_type.place_instruction(),
                "orderType": "MARKET_ON_CLOSE",
                "selectionId": self.mock_trade.selection_id,
                "side": "BACK",
            },
        )

    def test_create_cancel_instruction(self):
        self.order.update_data = {"size_reduction": 0.02}
        self.assertEqual(
            self.order.create_cancel_instruction(), {"sizeReduction": 0.02}
        )

    def test_create_update_instruction(self):
        self.mock_order_type.persistence_type = "PERSIST"
        self.assertEqual(
            self.order.create_update_instruction(), {"newPersistenceType": "PERSIST"}
        )

    def test_create_replace_instruction(self):
        self.order.update_data = {"new_price": 2.02}
        self.assertEqual(self.order.create_replace_instruction(), {"newPrice": 2.02})

    def test_average_price_matched(self):
        self.assertEqual(self.order.average_price_matched, 0)
        mock_current_order = mock.Mock(average_price_matched=12.3)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(
            self.order.average_price_matched, mock_current_order.average_price_matched
        )

    def test_size_matched(self):
        self.assertEqual(self.order.size_matched, 0)
        mock_current_order = mock.Mock(size_matched=10)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(self.order.size_matched, mock_current_order.size_matched)

    def test_size_remaining(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.mock_order_type.size = 0
        self.mock_order_type.bet_target_size = 0
        self.assertEqual(self.order.size_remaining, 0)
        self.mock_order_type.size = 10
        mock_current_order = mock.Mock(size_remaining=10)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(self.order.size_remaining, mock_current_order.size_remaining)

    def test_size_remaining_missing(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        self.mock_order_type.size = 2.51
        self.assertEqual(self.order.size_remaining, 2.51)

    @mock.patch(
        "flumine.order.order.BetfairOrder.size_matched",
        new_callable=mock.PropertyMock,
    )
    def test_size_remaining_missing_partial_match(self, mock_size_matched):
        self.mock_order_type.ORDER_TYPE = OrderTypes.LIMIT
        mock_size_matched.return_value = 2
        self.mock_order_type.size = 10
        self.assertEqual(self.order.size_remaining, 8)

    def test_size_remaining_market_on_close(self):
        self.mock_order_type.ORDER_TYPE = OrderTypes.MARKET_ON_CLOSE
        self.mock_order_type.size = ValueError
        self.order.responses.current_order = None
        self.assertEqual(self.order.size_remaining, self.mock_order_type.liability)

    def test_size_cancelled(self):
        self.assertEqual(self.order.size_cancelled, 0)
        mock_current_order = mock.Mock(size_cancelled=10)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(self.order.size_cancelled, mock_current_order.size_cancelled)

    def test_size_lapsed(self):
        self.assertEqual(self.order.size_lapsed, 0)
        mock_current_order = mock.Mock(size_lapsed=10)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(self.order.size_lapsed, mock_current_order.size_lapsed)

    def test_size_voided(self):
        self.assertEqual(self.order.size_voided, 0)
        mock_current_order = mock.Mock(size_voided=10)
        self.order.responses.current_order = mock_current_order
        self.assertEqual(self.order.size_voided, mock_current_order.size_voided)

    def test_info(self):
        self.order.status_log = [OrderStatus.PENDING, OrderStatus.EXECUTION_COMPLETE]
        self.assertEqual(
            self.order.info,
            {
                "bet_id": None,
                "handicap": self.order.handicap,
                "id": self.order.id,
                "date_time_created": str(self.order.date_time_created),
                "market_id": self.mock_trade.market_id,
                "selection_id": self.mock_trade.selection_id,
                "publish_time": None,
                "market_version": None,
                "async": None,
                "status": None,
                "status_log": "Pending, Execution complete",
                "trade": self.mock_trade.info,
                "order_type": self.mock_order_type.info,
                "info": {
                    "side": self.order.side,
                    "size_matched": self.order.size_matched,
                    "size_remaining": self.order.size_remaining,
                    "size_cancelled": self.order.size_cancelled,
                    "size_lapsed": self.order.size_lapsed,
                    "size_voided": self.order.size_voided,
                    "average_price_matched": self.order.average_price_matched,
                },
                "customer_order_ref": self.order.customer_order_ref,
                "simulated": {
                    "profit": 0.0,
                    "piq": 0.0,
                    "matched": [],
                },
                "violation_msg": self.order.violation_msg,
                "responses": {
                    "date_time_placed": None,
                    "elapsed_seconds_executable": None,
                },
                "runner_status": self.order.runner_status,
                "market_notes": None,
                "notes": "",
            },
        )

    def test_json(self):
        self.assertTrue(isinstance(self.order.json(), str))

    def test_set_invalid_sep(self):
        with self.assertRaises(ValueError):
            self.order.sep = "@"

class IsValidCustomerOrderRefTestCase(unittest.TestCase):
    def test_letters_True(self):
        # ascii_letters contains a-z and A-Z
        for c in string.ascii_letters:
            self.assertTrue(BetfairOrder.is_valid_customer_order_ref_character(c))

    def test_2letters_False(self):
        self.assertFalse(BetfairOrder.is_valid_customer_order_ref_character("aB"))
        self.assertFalse(BetfairOrder.is_valid_customer_order_ref_character("CD"))

    def test_digits_True(self):
        # string.digits contains digits 0-9
        for c in string.digits:
            self.assertTrue(BetfairOrder.is_valid_customer_order_ref_character(c))

    def test_special_characters_True(self):
        for c in VALID_BETFAIR_CUSTOMER_ORDER_REF_CHARACTERS:
            self.assertTrue(BetfairOrder.is_valid_customer_order_ref_character((c)))

    def test_special_characters_False(self):
        for c in list('!"£$%'):
            self.assertFalse(BetfairOrder.is_valid_customer_order_ref_character((c)))