text:
import TestCmd

import copy
import fnmatch
import glob
import math
import os
import os.path
import re
import shutil
import StringIO
import subprocess
import sys
import tempfile
import time
import traceback
import tree
import types

from xml.sax.saxutils import escape

class TestEnvironmentError(Exception):
    pass

annotations = []

def print_annotation(name, value, xml):
    """Writes some named bits of information about the current test run."""
    if xml:
        print escape(name) + " {{{"
        print escape(value)
        print "}}}"
    else:
        print name + " {{{"
        print value
        print "}}}"

def flush_annotations(xml=0):
    global annotations
    for ann in annotations:
        print_annotation(ann[0], ann[1], xml)
    annotations = []

def clear_annotations():
    global annotations
    annotations = []

defer_annotations = 0

def set_defer_annotations(n):
    global defer_annotations
    defer_annotations = n

def annotate_stack_trace(tb=None):
    if tb:
        trace = TestCmd.caller(traceback.extract_tb(tb), 0)
    else:
        trace = TestCmd.caller(traceback.extract_stack(), 1)
    annotation("stacktrace", trace)

def annotation(name, value):
    """Records an annotation about the test run."""
    annotations.append((name, value))
    if not defer_annotations:
        flush_annotations()

def get_toolset():
    toolset = None
    for arg in sys.argv[1:]:
        if not arg.startswith("-"):
            toolset = arg
    return toolset or "gcc"

# Detect the host OS.
cygwin = hasattr(os, "uname") and os.uname()[0].lower().startswith("cygwin")
windows = cygwin or os.environ.get("OS", "").lower().startswith("windows")

def prepare_prefixes_and_suffixes(toolset):
    prepare_suffix_map(toolset)
    prepare_library_prefix(toolset)

def prepare_suffix_map(toolset):
    """
      Set up suffix translation performed by the Boost Build testing framework
    to accomodate different toolsets generating targets of the same type using
    different filename extensions (suffixes).

    """
    global suffixes
    suffixes = {}
    if windows:
        if toolset == "gcc":
            suffixes[".lib"] = ".a"  # mingw static libs use suffix ".a".
            suffixes[".obj"] = ".o"
        if cygwin:
            suffixes[".implib"] = ".lib.a"
        else:
            suffixes[".implib"] = ".lib"
    else:
        suffixes[".exe"] = ""
        suffixes[".dll"] = ".so"
        suffixes[".lib"] = ".a"
        suffixes[".obj"] = ".o"
        suffixes[".implib"] = ".no_implib_files_on_this_platform"

        if hasattr(os, "uname") and os.uname()[0] == "Darwin":
            suffixes[".dll"] = ".dylib"

def prepare_library_prefix(toolset):
    """
      Setup whether Boost Build is expected to automatically prepend prefixes
    to its built library targets.

    """
    global lib_prefix
    lib_prefix = "lib"

    global dll_prefix
    if cygwin:
        dll_prefix = "cyg"
    elif windows and toolset != "gcc":
        dll_prefix = None
    else:
        dll_prefix = "lib"

def re_remove(sequence, regex):
    me = re.compile(regex)
    result = filter(lambda x: me.match(x), sequence)
    if not result:
        raise ValueError()
    for r in result:
        sequence.remove(r)

def glob_remove(sequence, pattern):
    result = fnmatch.filter(sequence, pattern)
    if not result:
        raise ValueError()
    for r in result:
        sequence.remove(r)

class Tester(TestCmd.TestCmd):
    """Main tester class for Boost Build.

    Optional arguments:

    `arguments`                   - Arguments passed to the run executable.
    `executable`                  - Name of the executable to invoke.
    `match`                       - Function to use for compating actual and
                                    expected file contents.
    `boost_build_path`            - Boost build path to be passed to the run
                                    executable.
    `translate_suffixes`          - Whether to update suffixes on the the file
                                    names passed from the test script so they
                                    match those actually created by the current
                                    toolset. For example, static library files
                                    are specified by using the .lib suffix but
                                    when the "gcc" toolset is used it actually
                                    creates them using the .a suffix.
    `pass_toolset`                - Whether the test system should pass the
                                    specified toolset to the run executable.
    `use_test_config`             - Whether the test system should tell the run
                                    executable to read in the test_config.jam
                                    configuration file.
    `ignore_toolset_requirements` - Whether the test system should tell the run
                                    executable to ignore toolset requirements.
    `workdir`                     - Absolute directory where the test will be
                                    run from.
    `pass_d0`                     - If set, when tests are not explicitly run
                                    in verbose mode, they are run as silent
                                    (-d0 & --quiet Boost Jam options).

    Optional arguments inherited from the base class:

    `description`                 - Test description string displayed in case
                                    of a failed test.
    `subdir`                      - List of subdirectories to automatically
                                    create under the working directory. Each
                                    subdirectory needs to be specified
                                    separately, parent coming before its child.
    `verbose`                     - Flag that may be used to enable more
                                    verbose test system output. Note that it
                                    does not also enable more verbose build
                                    system output like the --verbose command
                                    line option does.
    """
    def __init__(self, arguments=None, executable="bjam",
        match=TestCmd.match_exact, boost_build_path=None,
        translate_suffixes=True, pass_toolset=True, use_test_config=True,
        ignore_toolset_requirements=True, workdir="", pass_d0=True,
        **keywords):

        assert arguments.__class__ is not str
        self.original_workdir = os.getcwd()
        if workdir and not os.path.isabs(workdir):
            raise ("Parameter workdir <%s> must point to an absolute "
                "directory: " % workdir)

        self.last_build_timestamp = 0
        self.translate_suffixes = translate_suffixes
        self.use_test_config = use_test_config

        self.toolset = get_toolset()
        self.pass_toolset = pass_toolset
        self.ignore_toolset_requirements = ignore_toolset_requirements

        prepare_prefixes_and_suffixes(pass_toolset and self.toolset or "gcc")

        use_default_bjam = "--default-bjam" in sys.argv

        if not use_default_bjam:
            jam_build_dir = ""
            if os.name == "nt":
                jam_build_dir = "bin.ntx86"
            elif (os.name == "posix") and os.__dict__.has_key("uname"):
                if os.uname()[0].lower().startswith("cygwin"):
                    jam_build_dir = "bin.cygwinx86"
                    if ("TMP" in os.environ and
                        os.environ["TMP"].find("~") != -1):
                        print("Setting $TMP to /tmp to get around problem "
                            "with short path names")
                        os.environ["TMP"] = "/tmp"
                elif os.uname()[0] == "Linux":
                    cpu = os.uname()[4]
                    if re.match("i.86", cpu):
                        jam_build_dir = "bin.linuxx86"
                    else:
                        jam_build_dir = "bin.linux" + os.uname()[4]
                elif os.uname()[0] == "SunOS":
                    jam_build_dir = "bin.solaris"
                elif os.uname()[0] == "Darwin":
                    if os.uname()[4] == "i386":
                        jam_build_dir = "bin.macosxx86"
                    else:
                        jam_build_dir = "bin.macosxppc"
                elif os.uname()[0] == "AIX":
                    jam_build_dir = "bin.aix"
                elif os.uname()[0] == "IRIX64":
                    jam_build_dir = "bin.irix"
                elif os.uname()[0] == "FreeBSD":
                    jam_build_dir = "bin.freebsd"
                elif os.uname()[0] == "OSF1":
                    jam_build_dir = "bin.osf"
                else:
                    raise ("Do not know directory where Jam is built for this "
                        "system: %s/%s" % (os.name, os.uname()[0]))
            else:
                raise ("Do not know directory where Jam is built for this "
                    "system: %s" % os.name)

            # Find where jam_src is located. Try for the debug version if it is
            # lying around.
            dirs = [os.path.join("..", "engine", jam_build_dir + ".debug"),
                    os.path.join("..", "engine", jam_build_dir)]
            for d in dirs:
                if os.path.exists(d):
                    jam_build_dir = d
                    break
            else:
                print("Cannot find built Boost.Jam")
                sys.exit(1)

        verbosity = ["-d0", "--quiet"]
        if not pass_d0:
            verbosity = []
        if "--verbose" in sys.argv:
            keywords["verbose"] = True
            verbosity = ["-d+2"]

        if boost_build_path is None:
            boost_build_path = self.original_workdir + "/.."

        program_list = []
        if use_default_bjam:
            program_list.append(executable)
        else:
            program_list.append(os.path.join(jam_build_dir, executable))
        program_list.append('-sBOOST_BUILD_PATH="' + boost_build_path + '"')
        if verbosity:
            program_list += verbosity
        if arguments:
            program_list += arguments

        TestCmd.TestCmd.__init__(self, program=program_list, match=match,
            workdir=workdir, inpath=use_default_bjam, **keywords)

        os.chdir(self.workdir)

    def cleanup(self):
        try:
            TestCmd.TestCmd.cleanup(self)
            os.chdir(self.original_workdir)
        except AttributeError:
            # When this is called during TestCmd.TestCmd.__del__ we can have
            # both 'TestCmd' and 'os' unavailable in our scope. Do nothing in
            # this case.
            pass

    #
    # Methods that change the working directory's content.
    #
    def set_tree(self, tree_location):
        # It is not possible to remove the current directory.
        d = os.getcwd()
        os.chdir(os.path.dirname(self.workdir))
        shutil.rmtree(self.workdir, ignore_errors=False)

        if not os.path.isabs(tree_location):
            tree_location = os.path.join(self.original_workdir, tree_location)
        shutil.copytree(tree_location, self.workdir)

        os.chdir(d)
        def make_writable(unused, dir, entries):
            for e in entries:
                name = os.path.join(dir, e)
                os.chmod(name, os.stat(name).st_mode | 0222)
        os.path.walk(".", make_writable, None)

    def write(self, file, content, wait=True):
        nfile = self.native_file_name(file)
        self.__makedirs(os.path.dirname(nfile), wait)
        f = open(nfile, "wb")
        try:
            f.write(content)
        finally:
            f.close()
        self.__ensure_newer_than_last_build(nfile)

    def copy(self, src, dst):
        try:
            self.write(dst, self.read(src, 1))
        except:
            self.fail_test(1)

    def copy_preserving_timestamp(self, src, dst):
        src_name = self.native_file_name(src)
        dst_name = self.native_file_name(dst)
        stats = os.stat(src_name)
        self.write(dst, self.read(src, 1))
        os.utime(dst_name, (stats.st_atime, stats.st_mtime))

    def touch(self, names, wait=True):
        if names.__class__ is str:
            names = [names]
        for name in names:
            path = self.native_file_name(name)
            if wait:
                self.__ensure_newer_than_last_build(path)
            else:
                os.utime(path, None)

    def rm(self, names):
        if not type(names) == types.ListType:
            names = [names]

        if names == ["."]:
            # If we are deleting the entire workspace, there is no need to wait
            # for a clock tick.
            self.last_build_timestamp = 0

        # Avoid attempts to remove the current directory.
        os.chdir(self.original_workdir)
        for name in names:
            n = glob.glob(self.native_file_name(name))
            if n: n = n[0]
            if not n:
                n = self.glob_file(name.replace("$toolset", self.toolset + "*")
                    )
            if n:
                if os.path.isdir(n):
                    shutil.rmtree(n, ignore_errors=False)
                else:
                    os.unlink(n)

        # Create working dir root again in case we removed it.
        if not os.path.exists(self.workdir):
            os.mkdir(self.workdir)
        os.chdir(self.workdir)

    def expand_toolset(self, name):
        """
          Expands $toolset placeholder in the given file to the name of the
        toolset currently being tested.

        """
        self.write(name, self.read(name).replace("$toolset", self.toolset))

    def dump_stdio(self):
        annotation("STDOUT", self.stdout())
        annotation("STDERR", self.stderr())

    def run_build_system(self, extra_args=None, subdir="", stdout=None,
        stderr="", status=0, match=None, pass_toolset=None,
        use_test_config=None, ignore_toolset_requirements=None,
        expected_duration=None, **kw):

        assert extra_args.__class__ is not str

        if os.path.isabs(subdir):
            print("You must pass a relative directory to subdir <%s>." % subdir
                )
            return

        self.previous_tree, dummy = tree.build_tree(self.workdir)

        if match is None:
            match = self.match

        if pass_toolset is None:
            pass_toolset = self.pass_toolset

        if use_test_config is None:
            use_test_config = self.use_test_config

        if ignore_toolset_requirements is None:
            ignore_toolset_requirements = self.ignore_toolset_requirements

        try:
            kw["program"] = []
            kw["program"] += self.program
            if extra_args:
                kw["program"] += extra_args
            if pass_toolset:
                kw["program"].append("toolset=" + self.toolset)
            if use_test_config:
                kw["program"].append('--test-config="%s"' % os.path.join(
                    self.original_workdir, "test-config.jam"))
            if ignore_toolset_requirements:
                kw["program"].append("--ignore-toolset-requirements")
            if "--python" in sys.argv:
                kw["program"].append("--python")
            kw["chdir"] = subdir
            self.last_program_invocation = kw["program"]
            build_time_start = time.time()
            apply(TestCmd.TestCmd.run, [self], kw)
            build_time_finish = time.time()
        except:
            self.dump_stdio()
            raise

        old_last_build_timestamp = self.last_build_timestamp
        self.tree, self.last_build_timestamp = tree.build_tree(self.workdir)
        self.difference = tree.tree_difference(self.previous_tree, self.tree)
        if self.difference.empty():
            # If nothing has been changed by this build and sufficient time has
            # passed since the last build that actually changed something,
            # there is no need to wait for touched or newly created files to
            # start getting newer timestamps than the currently existing ones.
            self.last_build_timestamp = old_last_build_timestamp

        self.difference.ignore_directories()
        self.unexpected_difference = copy.deepcopy(self.difference)

        if (status and self.status) is not None and self.status != status:
            expect = ""
            if status != 0:
                expect = " (expected %d)" % status

            annotation("failure", '"%s" returned %d%s' % (kw["program"],
                self.status, expect))

            annotation("reason", "unexpected status returned by bjam")
            self.fail_test(1)

        if stdout is not None and not match(self.stdout(), stdout):
            annotation("failure", "Unexpected stdout")
            annotation("Expected STDOUT", stdout)
            annotation("Actual STDOUT", self.stdout())
            stderr = self.stderr()
            if stderr:
                annotation("STDERR", stderr)
            self.maybe_do_diff(self.stdout(), stdout)
            self.fail_test(1, dump_stdio=False)

        # Intel tends to produce some messages to stderr which make tests fail.
        intel_workaround = re.compile("^xi(link|lib): executing.*\n", re.M)
        actual_stderr = re.sub(intel_workaround, "", self.stderr())

        if stderr is not None and not match(actual_stderr, stderr):
            annotation("failure", "Unexpected stderr")
            annotation("Expected STDERR", stderr)
            annotation("Actual STDERR", self.stderr())
            annotation("STDOUT", self.stdout())
            self.maybe_do_diff(actual_stderr, stderr)
            self.fail_test(1, dump_stdio=False)

        if expected_duration is not None:
            actual_duration = build_time_finish - build_time_start
            if actual_duration > expected_duration:
                print("Test run lasted %f seconds while it was expected to "
                    "finish in under %f seconds." % (actual_duration,
                    expected_duration))
                self.fail_test(1, dump_stdio=False)

    def glob_file(self, name):
        result = None
        if hasattr(self, "difference"):
            for f in (self.difference.added_files +
                self.difference.modified_files +
                self.difference.touched_files):
                if fnmatch.fnmatch(f, name):
                    result = self.native_file_name(f)
                    break
        if not result:
            result = glob.glob(self.native_file_name(name))
            if result:
                result = result[0]
        return result

    def read(self, name, binary=False):
        try:
            if self.toolset:
                name = name.replace("$toolset", self.toolset + "*")
            name = self.glob_file(name)
            openMode = "r"
            if binary:
                openMode += "b"
            else:
                openMode += "U"
            f = open(name, openMode)
            result = f.read()
            f.close()
            return result
        except:
            annotation("failure", "Could not open '%s'" % name)
            self.fail_test(1)
            return ""

    def read_and_strip(self, name):
        if not self.glob_file(name):
            return ""
        f = open(self.glob_file(name), "rb")
        lines = f.readlines()
        f.close()
        result = "\n".join(x.rstrip() for x in lines)
        if lines and lines[-1][-1] != "\n":
            return result + "\n"
        return result

    def fail_test(self, condition, dump_difference=True, dump_stdio=True,
        dump_stack=True):
        if not condition:
            return

        if dump_difference and hasattr(self, "difference"):
            f = StringIO.StringIO()
            self.difference.pprint(f)
            annotation("changes caused by the last build command",
                f.getvalue())

        if dump_stdio:
            self.dump_stdio()

        if "--preserve" in sys.argv:
            print
            print "*** Copying the state of working dir into 'failed_test' ***"
            print
            path = os.path.join(self.original_workdir, "failed_test")
            if os.path.isdir(path):
                shutil.rmtree(path, ignore_errors=False)
            elif os.path.exists(path):
                raise "Path " + path + " already exists and is not a directory"
            shutil.copytree(self.workdir, path)
            print "The failed command was:"
            print " ".join(self.last_program_invocation)

        if dump_stack:
            annotate_stack_trace()
        sys.exit(1)

    # A number of methods below check expectations with actual difference
    # between directory trees before and after a build. All the 'expect*'
    # methods require exact names to be passed. All the 'ignore*' methods allow
    # wildcards.

    # All names can be either a string or a list of strings.
    def expect_addition(self, names):
        for name in self.adjust_names(names):
            try:
                glob_remove(self.unexpected_difference.added_files, name)
            except:
                annotation("failure", "File %s not added as expected" % name)
                self.fail_test(1)

    def ignore_addition(self, wildcard):
        self.__ignore_elements(self.unexpected_difference.added_files,
            wildcard)

    def expect_removal(self, names):
        for name in self.adjust_names(names):
            try:
                glob_remove(self.unexpected_difference.removed_files, name)
            except:
                annotation("failure", "File %s not removed as expected" % name)
                self.fail_test(1)

    def ignore_removal(self, wildcard):
        self.__ignore_elements(self.unexpected_difference.removed_files,
            wildcard)

    def expect_modification(self, names):
        for name in self.adjust_names(names):
            try:
                glob_remove(self.unexpected_difference.modified_files, name)
            except:
                annotation("failure", "File %s not modified as expected" %
                    name)
                self.fail_test(1)

    def ignore_modification(self, wildcard):
        self.__ignore_elements(self.unexpected_difference.modified_files,
            wildcard)

    def expect_touch(self, names):
        d = self.unexpected_difference
        for name in self.adjust_names(names):
            # We need to check both touched and modified files. The reason is
            # that:
            #   (1) Windows binaries such as obj, exe or dll files have slight
            #       differences even with identical inputs due to Windows PE
            #       format headers containing an internal timestamp.
            #   (2) Intel's compiler for Linux has the same behaviour.
            filesets = [d.modified_files, d.touched_files]

            while filesets:
                try:
                    glob_remove(filesets[-1], name)
                    break
                except ValueError:
                    filesets.pop()

            if not filesets:
                annotation("failure", "File %s not touched as expected" % name)
                self.fail_test(1)

    def ignore_touch(self, wildcard):
        self.__ignore_elements(self.unexpected_difference.touched_files,
            wildcard)

    def ignore(self, wildcard):
        self.ignore_addition(wildcard)
        self.ignore_removal(wildcard)
        self.ignore_modification(wildcard)
        self.ignore_touch(wildcard)

    def expect_nothing(self, names):
        for name in self.adjust_names(names):
            if name in self.difference.added_files:
                annotation("failure",
                    "File %s added, but no action was expected" % name)
                self.fail_test(1)
            if name in self.difference.removed_files:
                annotation("failure",
                    "File %s removed, but no action was expected" % name)
                self.fail_test(1)
                pass
            if name in self.difference.modified_files:
                annotation("failure",
                    "File %s modified, but no action was expected" % name)
                self.fail_test(1)
            if name in self.difference.touched_files:
                annotation("failure",
                    "File %s touched, but no action was expected" % name)
                self.fail_test(1)

    def expect_nothing_more(self):
        # Not totally sure about this change, but I do not see a good
        # alternative.
        if windows:
            self.ignore("*.ilk")       # MSVC incremental linking files.
            self.ignore("*.pdb")       # MSVC program database files.
            self.ignore("*.rsp")       # Response files.
            self.ignore("*.tds")       # Borland debug symbols.
            self.ignore("*.manifest")  # MSVC DLL manifests.

        # Debug builds of bjam built with gcc produce this profiling data.
        self.ignore("gmon.out")
        self.ignore("*/gmon.out")

        # Boost Build's 'configure' functionality (unfinished at the time)
        # produces this file.
        self.ignore("bin/config.log")
        self.ignore("bin/project-cache.jam")

        # Compiled Python files created when running Python based Boost Build.
        self.ignore("*.pyc")

        if not self.unexpected_difference.empty():
            annotation("failure", "Unexpected changes found")
            output = StringIO.StringIO()
            self.unexpected_difference.pprint(output)
            annotation("unexpected changes", output.getvalue())
            self.fail_test(1)

    def expect_output_lines(self, lines, expected=True):
        self.__expect_lines(self.stdout(), lines, expected)

    def expect_content_lines(self, filename, line, expected=True):
        self.__expect_lines(self.__read_file(filename), line, expected)

    def expect_content(self, name, content, exact=False):
        actual = self.__read_file(name, exact)
        content = content.replace("$toolset", self.toolset + "*")

        matched = False
        if exact:
            matched = fnmatch.fnmatch(actual, content)
        else:
            def sorted_(x):
                x.sort()
                return x
            actual_ = map(lambda x: sorted_(x.split()), actual.splitlines())
            content_ = map(lambda x: sorted_(x.split()), content.splitlines())
            if len(actual_) == len(content_):
                matched = map(
                    lambda x, y: map(lambda n, p: fnmatch.fnmatch(n, p), x, y),
                    actual_, content_)
                matched = reduce(
                    lambda x, y: x and reduce(
                        lambda a, b: a and b,
                    y),
                    matched)

        if not matched:
            print "Expected:\n"
            print content
            print "Got:\n"
            print actual
            self.fail_test(1)

    def maybe_do_diff(self, actual, expected):
        if os.environ.get("DO_DIFF"):
            e = tempfile.mktemp("expected")
            a = tempfile.mktemp("actual")
            f = open(e, "w")
            f.write(expected)
            f.close()
            f = open(a, "w")
            f.write(actual)
            f.close()
            print("DIFFERENCE")
            # Current diff should return 1 to indicate 'different input files'
            # but some older diff versions may return 0 and depending on the
            # exact Python/OS platform version, os.system() call may gobble up
            # the external process's return code and return 0 itself.
            if os.system('diff -u "%s" "%s"' % (e, a)) not in [0, 1]:
                print('Unable to compute difference: diff -u "%s" "%s"' % (e, a
                    ))
            os.unlink(e)
            os.unlink(a)
        else:
            print("Set environmental variable 'DO_DIFF' to examine the "
                "difference.")

    # Internal methods.
    def adjust_lib_name(self, name):
        global lib_prefix
        global dll_prefix
        result = name

        pos = name.rfind(".")
        if pos != -1:
            suffix = name[pos:]
            if suffix == ".lib":
                (head, tail) = os.path.split(name)
                if lib_prefix:
                    tail = lib_prefix + tail
                    result = os.path.join(head, tail)
            elif suffix == ".dll":
                (head, tail) = os.path.split(name)
                if dll_prefix:
                    tail = dll_prefix + tail
                    result = os.path.join(head, tail)
        # If we want to use this name in a Jamfile, we better convert \ to /,
        # as otherwise we would have to quote \.
        result = result.replace("\\", "/")
        return result

    def adjust_suffix(self, name):
        if not self.translate_suffixes:
            return name
        pos = name.rfind(".")
        if pos == -1:
            return name
        suffix = name[pos:]
        return name[:pos] + suffixes.get(suffix, suffix)

    # Acceps either a string or a list of strings and returns a list of
    # strings. Adjusts suffixes on all names.
    def adjust_names(self, names):
        if names.__class__ is str:
            names = [names]
        r = map(self.adjust_lib_name, names)
        r = map(self.adjust_suffix, r)
        r = map(lambda x, t=self.toolset: x.replace("$toolset", t + "*"), r)
        return r

    def native_file_name(self, name):
        name = self.adjust_names(name)[0]
        return os.path.normpath(os.path.join(self.workdir, *name.split("/")))

    def wait_for_time_change(self, path, touch):
        """
          Wait for newly assigned file system modification timestamps for the
        given path to become large enough for the timestamp difference to be
        correctly recognized by both this Python based testing framework and
        the Boost Jam executable being tested. May optionally touch the given
        path to set its modification timestamp to the new value.

        """
        self.__wait_for_time_change(path, touch, last_build_time=False)

    def __build_timestamp_resolution(self):
        """
          Returns the minimum path modification timestamp resolution supported
        by the used Boost Jam executable.

        """
        dir = tempfile.mkdtemp("bjam_version_info")
        try:
            jam_script = "timestamp_resolution.jam"
            f = open(os.path.join(dir, jam_script), "w")
            try:
                f.write("EXIT $(JAM_TIMESTAMP_RESOLUTION) : 0 ;")
            finally:
                f.close()
            p = subprocess.Popen([self.program[0], "-d0", "-f%s" % jam_script],
                stdout=subprocess.PIPE, cwd=dir, universal_newlines=True)
            out, err = p.communicate()
        finally:
            shutil.rmtree(dir, ignore_errors=False)

        if p.returncode != 0:
            raise TestEnvironmentError("Unexpected return code (%s) when "
                "detecting Boost Jam's minimum supported path modification "
                "timestamp resolution version information." % p.returncode)
        if err:
            raise TestEnvironmentError("Unexpected error output (%s) when "
                "detecting Boost Jam's minimum supported path modification "
                "timestamp resolution version information." % err)

        r = re.match("([0-9]{2}):([0-9]{2}):([0-9]{2}\\.[0-9]{9})$", out)
        if not r:
            # Older Boost Jam versions did not report their minimum supported
            # path modification timestamp resolution and did not actually
            # support path modification timestamp resolutions finer than 1
            # second.
            # TODO: Phase this support out to avoid such fallback code from
            # possibly covering up other problems.
            return 1
        if r.group(1) != "00" or r.group(2) != "00":  # hours, minutes
            raise TestEnvironmentError("Boost Jam with too coarse minimum "
                "supported path modification timestamp resolution (%s:%s:%s)."
                % (r.group(1), r.group(2), r.group(3)))
        return float(r.group(3))  # seconds.nanoseconds

    def __ensure_newer_than_last_build(self, path):
        """
          Updates the given path's modification timestamp after waiting for the
        newly assigned file system modification timestamp to become large
        enough for the timestamp difference between it and the last build
        timestamp to be correctly recognized by both this Python based testing
        framework and the Boost Jam executable being tested. Does nothing if
        there is no 'last build' information available.

        """
        if self.last_build_timestamp:
            self.__wait_for_time_change(path, touch=True, last_build_time=True)

    def __expect_lines(self, data, lines, expected):
        """
          Checks whether the given data contains the given lines.

          Data may be specified as a single string containing text lines
        separated by newline characters.

          Lines may be specified in any of the following forms:
            * Single string containing text lines separated by newlines - the
              given lines are searched for in the given data without any extra
              data lines between them.
            * Container of strings containing text lines separated by newlines
              - the given lines are searched for in the given data with extra
              data lines allowed between lines belonging to different strings.
            * Container of strings containing text lines separated by newlines
              and containers containing strings - the same as above with the
              internal containers containing strings being interpreted as if
              all their content was joined together into a single string
              separated by newlines.

          A newline at the end of any multi-line lines string is interpreted as
        an expected extra trailig empty line.
        """
        # str.splitlines() trims at most one trailing newline while we want the
        # trailing newline to indicate that there should be an extra empty line
        # at the end.
        splitlines = lambda x : (x + "\n").splitlines()

        if data is None:
            data = []
        elif data.__class__ is str:
            data = splitlines(data)

        if lines.__class__ is str:
            lines = [splitlines(lines)]
        else:
            expanded = []
            for x in lines:
                if x.__class__ is str:
                    x = splitlines(x)
                expanded.append(x)
            lines = expanded

        if _contains_lines(data, lines) != bool(expected):
            output = []
            if expected:
                output = ["Did not find expected lines:"]
            else:
                output = ["Found unexpected lines:"]
            first = True
            for line_sequence in lines:
                if line_sequence:
                    if first:
                        first = False
                    else:
                        output.append("...")
                    output.extend("  > " + line for line in line_sequence)
            output.append("in output:")
            output.extend("  > " + line for line in data)
            annotation("failure", "\n".join(output))
            self.fail_test(1)

    def __ignore_elements(self, list, wildcard):
        """Removes in-place 'list' elements matching the given 'wildcard'."""
        list[:] = filter(lambda x, w=wildcard: not fnmatch.fnmatch(x, w), list)

    def __makedirs(self, path, wait):
        """
          Creates a folder with the given path, together with any missing
        parent folders. If WAIT is set, makes sure any newly created folders
        have modification timestamps newer than the ones left behind by the
        last build run.

        """
        try:
            if wait:
                stack = []
                while path and path not in stack and not os.path.isdir(path):
                    stack.append(path)
                    path = os.path.dirname(path)
                while stack:
                    path = stack.pop()
                    os.mkdir(path)
                    self.__ensure_newer_than_last_build(path)
            else:
                os.makedirs(path)
        except Exception:
            pass

    def __python_timestamp_resolution(self, path, minimum_resolution):
        """
          Returns the modification timestamp resolution for the given path
        supported by the used Python interpreter/OS/filesystem combination.
        Will not check for resolutions less than the given minimum value. Will
        change the path's modification timestamp in the process.

          Return values:
            0                - nanosecond resolution supported
            positive decimal - timestamp resolution in seconds

        """
        # Note on Python's floating point timestamp support:
        #   Python interpreter versions prior to Python 2.3 did not support
        # floating point timestamps. Versions 2.3 through 3.3 may or may not
        # support it depending on the configuration (may be toggled by calling
        # os.stat_float_times(True/False) at program startup, disabled by
        # default prior to Python 2.5 and enabled by default since). Python 3.3
        # deprecated this configuration and 3.4 removed support for it after
        # which floating point timestamps are always supported.
        ver = sys.version_info[0:2]
        python_nanosecond_support = ver >= (3, 4) or (ver >= (2, 3) and
            os.stat_float_times())

        # Minimal expected floating point difference used to account for
        # possible imprecise floating point number representations. We want
        # this number to be small (at least smaller than 0.0001) but still
        # large enough that we can be sure that increasing a floating point
        # value by 2 * eta guarantees the value read back will be increased by
        # at least eta.
        eta = 0.00005

        stats_orig = os.stat(path)
        def test_time(diff):
            """Returns whether a timestamp difference is detectable."""
            os.utime(path, (stats_orig.st_atime, stats_orig.st_mtime + diff))
            return os.stat(path).st_mtime > stats_orig.st_mtime + eta

        # Test for nanosecond timestamp resolution support.
        if not minimum_resolution and python_nanosecond_support:
            if test_time(2 * eta):
                return 0

        # Detect the filesystem timestamp resolution. Note that there is no
        # need to make this code 'as fast as possible' as, this function gets
        # called before having to sleep until the next detectable modification
        # timestamp value and that, since we already know nanosecond resolution
        # is not supported, will surely take longer than whatever we do here to
        # detect this minimal detectable modification timestamp resolution.
        step = 0.1
        if not python_nanosecond_support:
            # If Python does not support nanosecond timestamp resolution we
            # know the minimum possible supported timestamp resolution is 1
            # second.
            minimum_resolution = max(1, minimum_resolution)
        index = max(1, int(minimum_resolution / step))
        while step * index < minimum_resolution:
            # Floating point number representation errors may cause our
            # initially calculated start index to be too small if calculated
            # directly.
            index += 1
        while True:
            # Do not simply add up the steps to avoid cumulative floating point
            # number representation errors.
            next = step * index
            if next > 10:
                raise TestEnvironmentError("File systems with too coarse "
                    "modification timestamp resolutions not supported.")
            if test_time(next):
                return next
            index += 1

    def __read_file(self, name, exact=False):
        name = self.adjust_names(name)[0]
        result = ""
        try:
            if exact:
                result = self.read(name)
            else:
                result = self.read_and_strip(name).replace("\\", "/")
        except (IOError, IndexError):
            print "Note: could not open file", name
            self.fail_test(1)
        return result

    def __wait_for_time_change(self, path, touch, last_build_time):
        """
          Wait until a newly assigned file system modification timestamp for
        the given path is large enough for the timestamp difference between it
        and the last build timestamp or the path's original file system
        modification timestamp (depending on the last_build_time flag) to be
        correctly recognized by both this Python based testing framework and
        the Boost Jam executable being tested. May optionally touch the given
        path to set its modification timestamp to the new value.

        """
        assert self.last_build_timestamp or not last_build_time
        stats_orig = os.stat(path)

        if last_build_time:
            start_time = self.last_build_timestamp
        else:
            start_time = stats_orig.st_mtime

        build_resolution = self.__build_timestamp_resolution()
        assert build_resolution >= 0

        # Check whether the current timestamp is already new enough.
        if stats_orig.st_mtime > start_time and (not build_resolution or
            stats_orig.st_mtime >= start_time + build_resolution):
            return

        resolution = self.__python_timestamp_resolution(path, build_resolution)
        assert resolution >= build_resolution

        # Implementation notes:
        #  * Theoretically time.sleep() API might get interrupted too soon
        #    (never actually encountered).
        #  * We encountered cases where we sleep just long enough for the
        #    filesystem's modifiction timestamp to change to the desired value,
        #    but after waking up, the read timestamp is still just a tiny bit
        #    too small (encountered on Windows). This is most likely caused by
        #    imprecise floating point timestamp & sleep interval representation
        #    used by Python. Note though that we never encountered a case where
        #    more than one additional tiny sleep() call was needed to remedy
        #    the situation.
        #  * We try to wait long enough for the timestamp to change, but do not
        #    want to waste processing time by waiting too long. The main
        #    problem is that when we have a coarse resolution, the actual times
        #    get rounded and we do not know the exact sleep time needed for the
        #    difference between two such times to pass. E.g. if we have a 1
        #    second resolution and the original and the current file timestamps
        #    are both 10 seconds then it could be that the current time is
        #    10.99 seconds and that we can wait for just one hundredth of a
        #    second for the current file timestamp to reach its next value, and
        #    using a longer sleep interval than that would just be wasting
        #    time.
        while True:
            os.utime(path, None)
            c = os.stat(path).st_mtime
            if resolution:
                if c > start_time and (not build_resolution or c >= start_time
                    + build_resolution):
                    break
                if c <= start_time - resolution:
                    # Move close to the desired timestamp in one sleep, but not
                    # close enough for timestamp rounding to potentially cause
                    # us to wait too long.
                    if start_time - c > 5:
                        if last_build_time:
                            error_message = ("Last build time recorded as "
                                "being a future event, causing a too long "
                                "wait period. Something must have played "
                                "around with the system clock.")
                        else:
                            error_message = ("Original path modification "
                                "timestamp set to far into the future or "
                                "something must have played around with the "
                                "system clock, causing a too long wait "
                                "period.\nPath: '%s'" % path)
                        raise TestEnvironmentError(message)
                    _sleep(start_time - c)
                else:
                    # We are close to the desired timestamp so take baby sleeps
                    # to avoid sleeping too long.
                    _sleep(max(0.01, resolution / 10))
            else:
                if c > start_time:
                    break
                _sleep(max(0.01, start_time - c))

        if not touch:
            os.utime(path, (stats_orig.st_atime, stats_orig.st_mtime))

class List:
    def __init__(self, s=""):
        elements = []
        if s.__class__ is str:
            # Have to handle escaped spaces correctly.
            elements = s.replace("\ ", "\001").split()
        else:
            elements = s
        self.l = [e.replace("\001", " ") for e in elements]

    def __len__(self):
        return len(self.l)

    def __getitem__(self, key):
        return self.l[key]

    def __setitem__(self, key, value):
        self.l[key] = value

    def __delitem__(self, key):
        del self.l[key]

    def __str__(self):
        return str(self.l)

    def __repr__(self):
        return "%s.List(%r)" % (self.__module__, " ".join(self.l))

    def __mul__(self, other):
        result = List()
        if not isinstance(other, List):
            other = List(other)
        for f in self:
            for s in other:
                result.l.append(f + s)
        return result

    def __rmul__(self, other):
        if not isinstance(other, List):
            other = List(other)
        return List.__mul__(other, self)

    def __add__(self, other):
        result = List()
        result.l = self.l[:] + other.l[:]
        return result

def _contains_lines(data, lines):
    data_line_count = len(data)
    expected_line_count = reduce(lambda x, y: x + len(y), lines, 0)
    index = 0
    for expected in lines:
        if expected_line_count > data_line_count - index:
            return False
        expected_line_count -= len(expected)
        index = _match_line_sequence(data, index, data_line_count -
            expected_line_count, expected)
        if index < 0:
            return False
    return True

def _match_line_sequence(data, start, end, lines):
    if not lines:
        return start
    for index in xrange(start, end - len(lines) + 1):
        data_index = index
        for expected in lines:
            if not fnmatch.fnmatch(data[data_index], expected):
                break;
            data_index += 1
        else:
            return data_index
    return -1

def _sleep(delay):
    if delay > 5:
        raise TestEnvironmentError("Test environment error: sleep period of "
            "more than 5 seconds requested. Most likely caused by a file with "
            "its modification timestamp set to sometime in the future.")
    time.sleep(delay)

###############################################################################
#
# Initialization.
#
###############################################################################

# Make os.stat() return file modification times as floats instead of integers
# to get the best possible file timestamp resolution available. The exact
# resolution depends on the underlying file system and the Python os.stat()
# implementation. The better the resolution we achieve, the shorter we need to
# wait for files we create to start getting new timestamps.
#
# Additional notes:
#  * os.stat_float_times() function first introduced in Python 2.3. and
#    suggested for deprecation in Python 3.3.
#  * On Python versions 2.5+ we do not need to do this as there os.stat()
#    returns floating point file modification times by default.
#  * Windows CPython implementations prior to version 2.5 do not support file
#    modification timestamp resolutions of less than 1 second no matter whether
#    these timestamps are returned as integer or floating point values.
#  * Python documentation states that this should be set in a program's
#    __main__ module to avoid affecting other libraries that might not be ready
#    to support floating point timestamps. Since we use no such external
#    libraries, we ignore this warning to make it easier to enable this feature
#    in both our single & multiple-test scripts.
if (2, 3) <= sys.version_info < (2, 5) and not os.stat_float_times():
    os.stat_float_times(True)

# Quickie tests. Should use doctest instead.
if __name__ == "__main__":
    assert str(List("foo bar") * "/baz") == "['foo/baz', 'bar/baz']"
    assert repr("foo/" * List("bar baz")) == "__main__.List('foo/bar foo/baz')"

    assert _contains_lines([], [])
    assert _contains_lines([], [[]])
    assert _contains_lines([], [[], []])
    assert _contains_lines([], [[], [], []])
    assert not _contains_lines([], [[""]])
    assert not _contains_lines([], [["a"]])

    assert _contains_lines([""], [])
    assert _contains_lines(["a"], [])
    assert _contains_lines(["a", "b"], [])
    assert _contains_lines(["a", "b"], [[], [], []])

    assert _contains_lines([""], [[""]])
    assert not _contains_lines([""], [["a"]])
    assert not _contains_lines(["a"], [[""]])
    assert _contains_lines(["a", "", "b", ""], [["a"]])
    assert _contains_lines(["a", "", "b", ""], [[""]])
    assert _contains_lines(["a", "", "b"], [["b"]])
    assert not _contains_lines(["a", "b"], [[""]])
    assert not _contains_lines(["a", "", "b", ""], [["c"]])
    assert _contains_lines(["a", "", "b", "x"], [["x"]])

    data = ["1", "2", "3", "4", "5", "6", "7", "8", "9"]
    assert _contains_lines(data, [["1", "2"]])
    assert not _contains_lines(data, [["2", "1"]])
    assert not _contains_lines(data, [["1", "3"]])
    assert not _contains_lines(data, [["1", "3"]])
    assert _contains_lines(data, [["1"], ["2"]])
    assert _contains_lines(data, [["1"], [], [], [], ["2"]])
    assert _contains_lines(data, [["1"], ["3"]])
    assert not _contains_lines(data, [["3"], ["1"]])
    assert _contains_lines(data, [["3"], ["7"], ["8"]])
    assert not _contains_lines(data, [["1"], ["3", "5"]])
    assert not _contains_lines(data, [["1"], [""], ["5"]])
    assert not _contains_lines(data, [["1"], ["5"], ["3"]])
    assert not _contains_lines(data, [["1"], ["5", "3"]])

    assert not _contains_lines(data, [[" 3"]])
    assert not _contains_lines(data, [["3 "]])
    assert not _contains_lines(data, [["3", ""]])
    assert not _contains_lines(data, [["", "3"]])

    print("tests passed")


text:
package com.alibaba.fastjson.parser.deserializer;

import java.lang.reflect.Type;
import java.text.SimpleDateFormat;

import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.parser.DefaultJSONParser;
import com.alibaba.fastjson.parser.JSONToken;

public class DateFormatDeserializer extends AbstractDateDeserializer implements ObjectDeserializer {

    public final static DateFormatDeserializer instance = new DateFormatDeserializer();

    @SuppressWarnings("unchecked")
    protected <T> T cast(DefaultJSONParser parser, Type clazz, Object fieldName, Object val) {

        if (val instanceof String) {
            String strVal = (String) val;
            if (strVal.length() == 0) {
                return null;
            }

            return (T) new SimpleDateFormat(strVal);
        }

        throw new JSONException("parse error");
    }

    public int getFastMatchToken() {
        return JSONToken.LITERAL_INT;
    }
}

redpajama_set_name: RedPajamaGithub


text:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.brixen</groupId>
    <artifactId>brixen</artifactId>
    <version>1.0.1-SNAPSHOT</version>
    <packaging>pom</packaging>

    <modules>
        <module>brixen-java</module>
    </modules>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-surefire-plugin</artifactId>
                <version>2.18.1</version>
                <configuration>
                    <workingDirectory>target/surefire-reports</workingDirectory>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-eclipse-plugin</artifactId>
                <version>2.10</version>
                <configuration>
                    <!-- Without this, the maven-eclipse plugin will generate bad .classpath
                             files, or at least files that expect a project called account-api to exist
                             at the same leve. -->
                    <useProjectReferences>false</useProjectReferences>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>


text:
layout: post
title: "How to Pimp Your Rprofile"
categories: r
tags: [r]
---

After you've been using R for a little bit, you start to notice people talking about their .Rprofile as if it's some mythical being. Nothing magical about it, but it can be a big time-saver if you find yourself typing things like, `summary()` or, the ever-hated, `stringasfactors=FALSE`, ad nauseam.

### Where is my .Rprofile?

The simple answer is, if you don't know, then you probably don't have one. R-Studio doesn't include one unless you tell it to. In Mac and Linux the .Rprofile is usually a hidden file in your user's home directory. In Windows the most common place is `C:\Program Files\R\Rx.x\etc`.

### Check to see if I have an .Rprofile

Before creating a new profile, fire up R and check to see if you have an existing .Rprofile lying around. Like I said, it's usually a hidden file.

{% highlight r %}
c(Sys.getenv("R_PROFILE_USER"), file.path(getwd(),".Rprofile"))
{% endhighlight %}

### How to create an .Rprofile

Assuming you don't already have one, these files are easy to create. Open a text editor and name your blank file .Rprofile with no trailing extension and place it in the appropriate directory. After populating the file, you'll have to restart R for the settings to take affect.

### Sample .Rprofile

Below is a snapshot of mine. Of coarse, you can make this as simple or as complex as you like.

{% highlight r %}
## Print this on start so I know it's loaded.
cat("Loading custom .Rprofile")

## A little gem from Hadley Wickam that will set your CRAN mirror and automatically load devtools in interactive sessions.
.First <- function() {
  options(
    repos = c(CRAN = "https://cran.rstudio.com/"),
    setwd("~/Documents/R"),
    deparse.max.lines = 2)
}

if (interactive()) {
  suppressMessages(require(devtools))
}

## Nice option for local work. I keep it commented out so my code can remain portable.
## options(stringsAsFactors=FALSE)

## Increase the size of my Rhistory file, becasue I like to use the up arrow!
Sys.setenv(R_HISTSIZE='100000')

## Create invisible environment ot hold all your custom functions
.env <- new.env()

## Single character shortcuts for summary() and head().
.env$s <- base::summary
.env$h <- utils::head

#ht==headtail, i.e., show the first and last 10 items of an object.
.env$ht <- function(d) rbind(head(d,10),tail(d,10))

## Read data on clipboard.
.env$read.cb <- function(...) {
  ismac <- Sys.info()[1]=="Darwin"
  if (!ismac) read.table(file="clipboard", ...)
  else read.table(pipe("pbpaste"), ...)
}

## List objects and classes.
.env$lsa <- function() {
{
    obj_type <- function(x) class(get(x, envir = .GlobalEnv)) # define environment
    foo = data.frame(sapply(ls(envir = .GlobalEnv), obj_type))
    foo$object_name = rownames(foo)
    names(foo)[1] = "class"
    names(foo)[2] = "object"
    return(unrowname(foo))
}

## List all functions in a package.
.env$lsp <-function(package, all.names = FALSE, pattern) {
    package <- deparse(substitute(package))
    ls(
        pos = paste("package", package, sep = ":"),
        all.names = all.names,
        pattern = pattern
    )
}

## Open Finder to the current directory. Mac Only!
.env$macopen <- function(...) if(Sys.info()[1]=="Darwin") system("open .")
.env$o       <- function(...) if(Sys.info()[1]=="Darwin") system("open .")

## Attach all the variables above
attach(.env)

## Finally, a function to print out all the functions you have defined in the .Rprofile.
print.functions <- function(){
	cat("s() - shortcut for summary\n",sep="")
	cat("h() - shortcut for head\n",sep="")
	cat("read.cb() - read from clipboard\n",sep="")
	cat("lsa() - list objects and classes\n",sep="")
	cat("lsp() - list all functions in a package\n",sep="")
	cat("macopen() - open finder to current working directory\n",sep="")
}
{% endhighlight %}

### Limitations and gotchas

The major disadvantage to all this is code portability. For example, if you set your .Rprofile to load `dplyr` on every session, when someone else tries to run your code, it won't work. For this reason, I'm a little picky about my settings, opting for functions that will only be used in interactive sessions.


text:
import { test } from "./file";

it("should hide import by local var", function() {
	var test = "ok";
	test.should.be.eql("ok");
});

it("should hide import by object pattern", function() {
	var { test } = { test: "ok" };
	test.should.be.eql("ok");
});

it("should hide import by array pattern", function() {
	var [test] = ["ok"];
	test.should.be.eql("ok");
});

it("should hide import by array pattern (nested)", function() {
	var [[test]] = [["ok"]];
	test.should.be.eql("ok");
});

it("should hide import by pattern in function", function() {
	(function({test}) {
		test.should.be.eql("ok");
	}({ test: "ok" }));
});

it("should allow import in default", function() {
	var { other = test, test } = { test: "ok" };
	test.should.be.eql("ok");
	other.should.be.eql("test");
});


text:
using Microsoft.Owin;
using Owin;
using XSockets.Owin.Host;

[assembly: OwinStartup(typeof(MoveShape.Startup))]

namespace MoveShape
{
    public class Startup
    {
        public void Configuration(IAppBuilder app)
        {
            app.UseXSockets();
        }
    }
}


text:
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<type language="gradle" name="org.gradle.api.internal.TypedDomainObjectContainerWrapper" version="3.0">
    <method name="add" returnType="boolean">
        <parameter type="U" name="e"/>
    </method>
    <method name="addAll" returnType="boolean">
        <parameter type="java.util.Collection" name="c"/>
    </method>
    <method name="addRule" returnType="org.gradle.api.Rule">
        <parameter type="org.gradle.api.Rule" name="rule"/>
    </method>
    <method name="addRule" returnType="org.gradle.api.Rule">
        <parameter type="java.lang.String" name="description"/>
        <parameter type="groovy.lang.Closure" name="ruleAction"/>
    </method>
    <method name="all" returnType="void">
        <parameter type="groovy.lang.Closure" name="action"/>
    </method>
    <method name="all" returnType="void">
        <parameter type="org.gradle.api.Action" name="action"/>
    </method>
    <method name="clear" returnType="void"/>
    <method name="configure" returnType="org.gradle.api.NamedDomainObjectContainer">
        <parameter type="groovy.lang.Closure" name="configureClosure"/>
    </method>
    <method name="contains" returnType="boolean">
        <parameter type="java.lang.Object" name="o"/>
    </method>
    <method name="containsAll" returnType="boolean">
        <parameter type="java.util.Collection" name="c"/>
    </method>
    <method name="create" returnType="U">
        <parameter type="java.lang.String" name="name"/>
    </method>
    <method name="create" returnType="U">
        <parameter type="java.lang.String" name="name"/>
        <parameter type="groovy.lang.Closure" name="configureClosure"/>
    </method>
    <method name="create" returnType="U">
        <parameter type="java.lang.String" name="name"/>
        <parameter type="org.gradle.api.Action" name="configureAction"/>
    </method>
    <method name="equals" returnType="boolean">
        <parameter type="java.lang.Object" name="o"/>
    </method>
    <method name="findAll" returnType="java.util.Set">
        <parameter type="groovy.lang.Closure" name="spec"/>
    </method>
    <method name="findByName" returnType="U">
        <parameter type="java.lang.String" name="name"/>
    </method>
    <method name="getAsDynamicObject" returnType="org.gradle.internal.metaobject.DynamicObject"/>
    <method name="getAsMap" returnType="java.util.SortedMap"/>
    <method name="getAt" returnType="U">
        <parameter type="java.lang.String" name="name"/>
    </method>
    <method name="getByName" returnType="U">
        <parameter type="java.lang.String" name="name"/>
    </method>
    <method name="getByName" returnType="U">
        <parameter type="java.lang.String" name="name"/>
        <parameter type="groovy.lang.Closure" name="configureClosure"/>
    </method>
    <method name="getNamer" returnType="org.gradle.api.Namer"/>
    <method name="getNames" returnType="java.util.SortedSet"/>
    <method name="getRules" returnType="java.util.List"/>
    <method name="hashCode" returnType="int"/>
    <method name="isEmpty" returnType="boolean"/>
    <method name="iterator" returnType="java.util.Iterator"/>
    <method name="matching" returnType="org.gradle.api.NamedDomainObjectSet">
        <parameter type="groovy.lang.Closure" name="spec"/>
    </method>
    <method name="matching" returnType="org.gradle.api.NamedDomainObjectSet">
        <parameter type="org.gradle.api.specs.Spec" name="spec"/>
    </method>
    <method name="maybeCreate" returnType="U">
        <parameter type="java.lang.String" name="name"/>
    </method>
    <method name="remove" returnType="boolean">
        <parameter type="java.lang.Object" name="o"/>
    </method>
    <method name="removeAll" returnType="boolean">
        <parameter type="java.util.Collection" name="c"/>
    </method>
    <method name="retainAll" returnType="boolean">
        <parameter type="java.util.Collection" name="c"/>
    </method>
    <method name="size" returnType="int"/>
    <method name="toArray" returnType="java.lang.Object"/>
    <method name="toArray" returnType="T">
        <parameter type="T" name="a"/>
    </method>
    <method name="whenObjectAdded" returnType="void">
        <parameter type="groovy.lang.Closure" name="action"/>
    </method>
    <method name="whenObjectAdded" returnType="org.gradle.api.Action">
        <parameter type="org.gradle.api.Action" name="action"/>
    </method>
    <method name="whenObjectRemoved" returnType="void">
        <parameter type="groovy.lang.Closure" name="action"/>
    </method>
    <method name="whenObjectRemoved" returnType="org.gradle.api.Action">
        <parameter type="org.gradle.api.Action" name="action"/>
    </method>
    <method name="withType" returnType="org.gradle.api.NamedDomainObjectSet">
        <parameter type="java.lang.Class" name="type"/>
    </method>
    <method name="withType" returnType="org.gradle.api.DomainObjectCollection" delegationTarget="withType">
        <parameter type="java.lang.Class" name="type"/>
        <parameter type="groovy.lang.Closure" name="configureClosure"/>
    </method>
    <method name="withType" returnType="org.gradle.api.DomainObjectCollection">
        <parameter type="java.lang.Class" name="type"/>
        <parameter type="org.gradle.api.Action" name="configureAction"/>
    </method>
    <interface name="org.gradle.api.NamedDomainObjectContainer"/>
    <interface name="org.gradle.api.internal.DynamicObjectAware"/>
    <property name="asDynamicObject" type="org.gradle.internal.metaobject.DynamicObject"/>
    <property name="asMap" type="java.util.SortedMap"/>
    <property name="at" type="U"/>
    <property name="byName" type="U"/>
    <property name="empty" type="boolean"/>
    <property name="namer" type="org.gradle.api.Namer"/>
    <property name="names" type="java.util.SortedSet"/>
    <property name="rules" type="java.util.List"/>
</type>


text:
/*
 * test_thread_equal.cc
 * uv_thread_t is pthread_t, and is type unsigned long int.
 * Created on: 2015年2月5日
 */

#include <stdlib.h>
#include <stdio.h>
#include <uv.h>

uv_thread_t main_thread_id;	//主线程id
uv_thread_t subthreads[2];	//传递给子线程的数据

/**
 * this method will run in subthread.
 * user can translate data from arg.
 * @param arg data from main thread or you can pass subthread's data to main thread.
 */
static void check_thread(void* arg) {
	uv_thread_t *thread_id = (uv_thread_t *)arg;
	uv_thread_t self_id = uv_thread_self();
	printf("main_thread_id:%lu\n", main_thread_id);
	printf("sub_thread_id:%lu\n", self_id);
	*thread_id = uv_thread_self();
}

int main()
{
	uv_thread_t threads[2];
	main_thread_id = uv_thread_self();	//获得当前线程的id

	uv_thread_create(threads + 0, check_thread, subthreads + 0);	//创建子线程
	uv_thread_create(threads + 1, check_thread, subthreads + 1);

	uv_thread_join(threads + 0);	//thread join,阻塞,直到目标子线程执行完成
	uv_thread_join(threads + 1);

	return 0;
}


text:
export const FORCED: string;
export const DISABLED: string;
export const ONLY_LABEL_CHARS: string;


text:
export default (req, res, next) => {
  res.locals.apiBaseUrl = process.env.API_BASE_PATH

  return next()
}


text:
class CreateConductorDailyExperiments < ActiveRecord::Migration
  def change
    create_table "conductor_daily_experiments", :force => true do |t|
      t.date    "activity_date"
      t.string  "group_name"
      t.string  "alternative"
      t.decimal "conversion_value", :precision => 8, :scale => 2
      t.integer "views"
      t.integer "conversions"
    end

    add_index "conductor_daily_experiments", ["activity_date"], :name => "index_conductor_daily_experiments_on_activity_date"
    add_index "conductor_daily_experiments", ["group_name"], :name => "index_conductor_daily_experiments_on_group_name"

  end
end


text:
<!DOCTYPE html>

    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta name="generator" content="Hugo 0.20.7">
    <meta name="theme" content="Tranquilpeak 0.3.1-BETA">
    <title>Web scraping fantasy football data using R</title>
    <meta name="author" content="Paul Burgoine">
    <meta name="keywords" content="">

    <link rel="icon" href="/favicon.png">

    <meta name="description" content="Now the amount of data on the FPL website is good; everyone's scores are public and you can see your own points week by week. However, clicking from week to week is a bit clunky and the individual player scores, located below the player's jersey are rather small and don't take centre stage. The page has a pitch layout as it is user friendly; it makes setting a formation, transferring players and making substitutions intuitive but it isn't helpful for data analysis.">
    <meta property="og:description" content="Now the amount of data on the FPL website is good; everyone's scores are public and you can see your own points week by week. However, clicking from week to week is a bit clunky and the individual player scores, located below the player's jersey are rather small and don't take centre stage. The page has a pitch layout as it is user friendly; it makes setting a formation, transferring players and making substitutions intuitive but it isn't helpful for data analysis.">
    <meta property="og:type" content="blog">
    <meta property="og:title" content="Web scraping fantasy football data using R">
    <meta property="og:url" content="/2017/06/web-scraping-fantasy-football-data-using-r/">
    <meta property="og:site_name" content="Paul Burgoine">
    <meta name="twitter:card" content="summary">
    <meta name="twitter:title" content="Paul Burgoine">
    <meta name="twitter:description" content="Now the amount of data on the FPL website is good; everyone's scores are public and you can see your own points week by week. However, clicking from week to week is a bit clunky and the individual player scores, located below the player's jersey are rather small and don't take centre stage. The page has a pitch layout as it is user friendly; it makes setting a formation, transferring players and making substitutions intuitive but it isn't helpful for data analysis.">

      <meta property="og:image" content="/images/me.png">

    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" />
    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/jquery.fancybox.css" />
    <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/helpers/jquery.fancybox-thumbs.min.css" />

    <link rel="stylesheet" href="/css/style-u6mk0ojoywresbx8iepslrmmhl4stuhrsxuwhkpwrkrx7mryjcaimasnk4pi.min.css" />

    Paul Burgoine

Paul Burgoine

Interested in data, visualisations, machine learning

      Categories

      Tags

      Archives

      About

      Home

      Stack Overflow

      GitHub

      RSS

      Web scraping fantasy football data using R

    <time itemprop="datePublished" datetime="2017-06-01T00:00:00Z">

  June 1, 2017

    </time>

    in

      R,

      Web scraping

Now the amount of data on the FPL website is good; everyone's scores are public and you can see your own points week by week. However, clicking from week to week is a bit clunky and the individual player scores, located below the player's jersey are rather small and don't take centre stage. The page has a pitch layout as it is user friendly; it makes setting a formation, transferring players and making substitutions intuitive but it isn't helpful for data analysis.

At first I thought about simply copying and pasting numbers week by week into a spreadsheet, but for 38 weeks this would have been incredibly error prone and time consuming. So that's when I had the idea of web scraping the data by writing a script to do the tedious work for me. I had never web scraped any data before, however I read an article on Kaggle about web scraping beer data from a website and it sounded relatively simple and an interesting new skill to learn.

I decided I would try to write it in Python using BeautifulSoup, and it seemed relatively simple to get going; I managed to write a short script that seemed to be working – I was able to scrape data from the menus and static text on the FPL site. This is content everyone would see when visiting the site but when it came to taking player's names, teams, and most importantly points hauls the script drew a blank. After a bit of research I realised it was because player data is JavaScript generated content. Users individual players and points are unique and based on team selection and bench and captaincy choices. Even when visiting a specific users page for a specific gameweek the content was still JS generated. Back to the drawing board.

So I did some research and attempted to write a Python script that could handle JavaScript content (basically the script would have to run as if it were a person visiting the site to get the JavaScript to run). I half-heartedly attempted a few different options that I saw people were recommending on Stack Overflow (DryScape and Selenium were two packages/methods I saw referenced multiple times and subsequently tried and to get working). It just seemed like a lot of effort for a simple process of collecting 15 names and numbers for 38 different pages. I can use Python, however I am not massively experienced with it and I am sure this is partly to blame for the issues I ran into here.

I stepped away from the computer for a while and then when I came back decided I would google R's capabilities to webscrape. I am far more experienced using R and its flexibility in dealing with all sorts of problems have never let me down before. The R package rvest is the only package we need to use within R and has a lot of different functions to help read html and extract relevant bits. The only outstanding issue, again, is the JavaScript content.

To my delight, after a bit of research I came across phantomjs and managed to get it working almost immediately. To use the code below you will need to download phantomjs. It isn’t a R package so you then need to copy and paste the file in the following location on your computer if your using a Mac /usr/local/bin/. I’m sure its similar for other systems. And thats it! R can now scrape JS content. Back to web scraping and I broke the problem down into 3 steps –

Writing a function to access the data and create a local copy on my computer.

Search through the downloaded data and extract all the relevant data.

Clean the data and get into a usable format for data visualisation.

Scraping the data

The first function is below. It takes a number from 1 to 38 if you just want a specific gameweek, or a vector of gameweeks if you would like multiple. I found that the code was inconsistent at retrieving data, it would occasionally return empty and hence I just used a brute force method to get every gameweek, whereby the function will keep attempting a gameweek until successful. Although it will sometimes fail for a gameweek, other times it would succeed for the same week, and hence found that simply retrying fixed any issues. If you would like to use the code below change my user id number (1537030) to whatever yours is - if you go on the points tab for any gameweek it will be located in the address bar.

#install.packages("rvest")
library(rvest)

###############################################

#Function to scrape site
fplscraper <- function(gameweek){
  url_week = paste("http://fantasy.premierleague.com/a/team/1537030/event/", gameweek, sep="")

  # render HTML from the site with phantomjs
 repeat{
  writeLines(sprintf("var page = require('webpage').create();
                     page.open('%s', function () {
                     console.log(page.content); //page source
                     phantom.exit();
                     });", url_week), con="scrape.js")

  system("phantomjs scrape.js > scrape.html")

  # extract the content you need
  html_week = read_html("scrape.html")

  # check extract worked properly
  if(length(html_week %>% html_nodes("#ismr-pos1") %>% html_nodes(".ism-element__name") %>% html_text()>0)) break
  print("Extract failed...retrying")
}

  print("Record extracted")
return(html_week)
}

Extracting information of value

The next function takes the output we generate from the first function, and then returns a list (or a number of lists if more than one gameweek is fed to it) of key metrics for each of the players involved in that gameweek, namely, players name, points, team, whether captain or vice-captain and whether the player was on the bench or field.

#Function to search extract data from specific gameweek
calc_points <- function(fpl_data){

  players = c(1:15)
  Players_Names = c()
  Players_Points = c()
  Players_Team = c()
  Players_Captain = c()
  Players_Played = c()

  for (player in players){

  player_pos <- paste("#ismr-pos", player,sep="")
  player_name <- as.character(fpl_data %>% html_nodes(player_pos) %>% html_nodes(".ism-element__name") %>% html_text())
  player_data <- as.numeric(fpl_data %>% html_nodes(player_pos) %>% html_nodes(".ism-element__data") %>% html_text())
  player_shirt <- fpl_data %>% html_nodes(player_pos) %>% html_nodes(".ism-element__shirt")
  player_team <- as.character(html_attr(player_shirt, "title"))
  player_cap = fpl_data %>% html_nodes(player_pos) %>% html_nodes(".ism-element__control--captain") %>% html_nodes(".ism-element__icon")
  player_captain = as.character(html_attr(player_cap, "title"))
  if (length(player_captain)==0){player_captain = NA}
  player_played = "Blank"
  if (player<12) {player_played <- "Played"} else {player_played <- "Benched"}

  Players_Names = c(Players_Names,player_name)
  Players_Points = c(Players_Points,player_data)
  Players_Team = c(Players_Team,player_team)
  Players_Captain = c(Players_Captain,player_captain)
  Players_Played = c(Players_Played,player_played)

  }

  df <- list(Players_Names,Players_Points,Players_Team,Players_Captain,Players_Played)
  return(df)
}

Constructing the dataset from our lists

Finally, the last function will spit out a data frame of all our players in the selected gameweeks, with variables for the player name, team and then n variables for gameweeks, whether they played, and if they were captain.

#Function to bind all our gameweeks into one final data set

data_merge <- function(fpl_data){

  all_players = c()
  for (gw in c(1:38)) {
    all_players = c(all_players, paste(all_gw[[gw]][[1]],all_gw[[gw]][[3]],sep=";"))
  }

  unique_players = unique(all_players)
  result = matrix(NA,nrow = length(unique_players), ncol = 116)

  unique_names = strsplit(unique_players,';',fixed=TRUE)

  split_names = gsub(";.*", "", unique_players)
  split_teams = gsub(".*;", "", unique_players)

  final_data = data.frame(result)

  names(final_data) <- c("Player", "Team", paste0("GW", 1:38),paste0("CAP", 1:38),paste0("PLAY", 1:38))
  final_data$Player <- split_names
  final_data$Team <- split_teams

  for(ngw in c(1:38)){

    match_indexes = match(all_gw[[ngw]][[1]],split_names)

    for (matches in c(1:15)){
      final_data[[2+ngw]][match_indexes[matches]] = all_gw[[ngw]][[2]][matches]
      final_data[[40+ngw]][match_indexes[matches]] = all_gw[[ngw]][[4]][matches]
      final_data[[78+ngw]][match_indexes[matches]] = all_gw[[ngw]][[5]][matches]
    }

  }

  return(final_data)

}

Calling our functions & save to csv

Next bit is nice and simple.

#Call our functions
gameweeks <- c(1:38)
gameweek_data<- lapply(gameweeks,fplscraper)
all_gw<-lapply(gameweek_data,calc_points)
final_dataset<-data_merge(all_gw)

#Save to csv
write.csv(final_dataset, file = "FPLdata1617.csv",fileEncoding = "UTF-8")

And thats it! We have a lovely CSV file ready to analyse. In my next post I will be visualising the data. Thanks for reading.

                    TAGGED IN

  R

  Web scraping

            NEXT

            PREVIOUS

    © 2017 Paul Burgoine. All Rights Reserved

            NEXT

            PREVIOUS

        Share on Google Plus

        Share on Facebook

        Share on Twitter

Paul Burgoine

Interested in data, visualisations, machine learning

        Data Enthusiast

        Manchester, UK

        by

      <form id="algolia-search-form">
        <input type="text" id="algolia-search-input" name="search"
          class="form-control input--large search-input" placeholder="Search" />
      </form>

no post found

Last years FPL season Bubble Chart

                  Sep 9, 2017

Linked in this post is my 2016-17 FPL points by week visualised as an interactive bubble chart. The data was collected using a web scraping script I wrote in R and detailed in my previous post.
The larger the bubble the more points the player earned that week. If you are not on a mobile device, you can hover over the bubbles to reveal players points that week, whether they were sat on the bench or in my team and whether they were the captain or not.

Web scraping fantasy football data using R

                  Jun 6, 2017

Now the amount of data on the FPL website is good; everyone's scores are public and you can see your own points week by week. However, clicking from week to week is a bit clunky and the individual player scores, located below the player's jersey are rather small and don't take centre stage. The page has a pitch layout as it is user friendly; it makes setting a formation, transferring players and making substitutions intuitive but it isn't helpful for data analysis.

Posts

                  Sep 9, 2017

         3 posts found


text:
package org.auraframework.impl.root.parser.handler;

import java.util.Collections;
import java.util.List;
import java.util.Set;

import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;

import org.apache.commons.lang3.StringUtils;
import org.auraframework.Aura;
import org.auraframework.def.ClientLibraryDef;
import org.auraframework.def.ComponentDef;
import org.auraframework.def.RootDefinition;
import org.auraframework.impl.clientlibrary.ClientLibraryDefImpl;
import org.auraframework.system.AuraContext;
import org.auraframework.system.Source;
import org.auraframework.throwable.quickfix.DefinitionNotFoundException;
import org.auraframework.throwable.quickfix.InvalidDefinitionException;
import org.auraframework.throwable.quickfix.QuickFixException;
import org.auraframework.util.AuraTextUtil;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;

/**
 * Process client library tags and create {@link ClientLibraryDef} definition
 */
public class ClientLibraryDefHandler<P extends RootDefinition> extends ParentedTagHandler<ClientLibraryDef, P> {

    public static final String TAG = "aura:clientLibrary";

    private static final String ATTRIBUTE_NAME = "name";
    private static final String ATTRIBUTE_TYPE = "type";
    private static final String ATTRIBUTE_MODES = "modes";

    private final static Set<String> ALLOWED_ATTRIBUTES = ImmutableSet.of(ATTRIBUTE_NAME, ATTRIBUTE_TYPE, ATTRIBUTE_MODES);

    private ClientLibraryDefImpl.Builder builder;

    public ClientLibraryDefHandler(RootTagHandler<P> parentHandler, XMLStreamReader xmlReader, Source<?> source) throws DefinitionNotFoundException {
        super(parentHandler, xmlReader, source);

        if (!isInInternalNamespace()) {
            throw new DefinitionNotFoundException(Aura.getDefinitionService().getDefDescriptor(TAG, ComponentDef.class));
        }

        this.builder = new ClientLibraryDefImpl.Builder();
        this.builder.setLocation(getLocation());
        this.builder.setParentDescriptor(parentHandler.getDefDescriptor());
    }

    @Override
    public Set<String> getAllowedAttributes() {
        return ALLOWED_ATTRIBUTES;
    }

    /**
     * Reads name, url, type, and modes attributes.
     */
    @Override
    protected void readAttributes() throws QuickFixException {
        builder.setName(getAttributeValue(ATTRIBUTE_NAME));

        String type = getAttributeValue(ATTRIBUTE_TYPE);
        // JS by default
        builder.setType(ClientLibraryDef.Type.JS);
        if (StringUtils.isNotBlank(type)) {
            try {
                builder.setType(ClientLibraryDef.Type.valueOf(type.trim().toUpperCase()));
            } catch (IllegalArgumentException iae) {
                throw new InvalidDefinitionException("Missing valid type", getLocation());
            }
        }

        String modeStr = getAttributeValue(ATTRIBUTE_MODES);
        Set<AuraContext.Mode> modes = Collections.emptySet();
        if (StringUtils.isNotBlank(modeStr)) {
            modes = Sets.newHashSet();
            List<String> modesList = AuraTextUtil.splitSimpleAndTrim(modeStr, ",", 0);
            for (String m : modesList) {
                try {
                    modes.add(AuraContext.Mode.valueOf(m.toUpperCase()));
                } catch (IllegalArgumentException iae) {
                    throw new InvalidDefinitionException("Invalid mode specified", getLocation());
                }
            }
        }
        builder.setModes(modes);
    }

    @Override
    protected ClientLibraryDef createDefinition() throws QuickFixException {
        return builder.build();
    }

    @Override
    public String getHandledTag() {
        return TAG;
    }

    @Override
    protected void handleChildTag() throws XMLStreamException, QuickFixException {
        error("ClientLibrary cannot have a child tag");
    }

    @Override
    protected void handleChildText() throws XMLStreamException, QuickFixException {
        error("ClientLibrary cannot have child text");
    }
}


text:
<form action="{% url 'workcenters:create_report' job %}" method="get" name="create_report" class="report-and-deatil-buttons">
    <input type="submit" name="create_report" value="Create Report"/>
</form>


text:
<?php

namespace GeniusDesign\Components\NewsBundle\Helper;

use Symfony\Component\DependencyInjection\ContainerInterface;

class NewsHelper {

    /**
     * The container
     * @var ContainerInterface
     */
    private $container = null;

    /**
     * Class constructor
     *
     * @param ContainerInterface $container The container
     * @return void
     */
    public function __construct(ContainerInterface $container) {
        $this->container = $container;
    }

    /**
     * Returns container
     * @return ContainerInterface
     */
    public function getContainer() {
        return $this->container;
    }

    /**
     * Returns information whether to show the picture
     * @return boolean
     */
    public function isImageEnabled() {
        return $this->getContainer()
                        ->getParameter('genius_design_components_news.show_image');
    }

    /**
     * Returns information whether to show the autor
     * @return boolean
     */
    public function isAutorEnabled() {
        return $this->getContainer()
                        ->getParameter('genius_design_components_news.show_autor');
    }

    /**
     * Returns information whether to show the date
     * @return boolean
     */
    public function isDateEnabled() {
        return $this->getContainer()
                        ->getParameter('genius_design_components_news.show_date');
    }

}


text:
import * as constants from '../constants';

export const updateOrganization = (formData) => ({
  type: constants.UPDATE_ORGANIZATION,
  payload: formData,
});

export const setOrganizationLocation = (customer) => ({
  type: constants.SET_ORGANIZATION_LOCATION,
  payload: customer,
});


text:
                <button type="button" class="btn btn-default active show-ws">Workspaces</button>
                <button type="button" class="btn btn-default show-objs">Objects ({{selectedObjs.length}})</button>

Selection

Core

Heatmap

Tree

            No Workspace Selected

            Please select a workspace from the left.  You will be able to FBA objects for analysis


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

		<meta http-equiv="content-type" content="text/html; charset=utf-8" />
		<meta name="generator" content="JsDoc Toolkit" />

		<title>JsDoc Reference - AV.Role</title>

Class Index
| File Index

Classes

_global_

AV

AV.ACL

AV.BigQuery

AV.Cloud

AV.Cloud.AfterSaveRequest

AV.Cloud.AfterUpdateRequest

AV.Cloud.BeforeDeleteRequest

AV.Cloud.BeforeDeleteResponse

AV.Cloud.BeforeSaveRequest

AV.Cloud.BeforeSaveResponse

AV.Cloud.FunctionRequest

AV.Cloud.FunctionResponse

AV.Collection

AV.Error

AV.Events

AV.FacebookUtils

AV.File

AV.GeoPoint

AV.History

AV.InboxQuery

AV.Insight

AV.Insight.JobQuery

AV.Object

AV.Op

AV.Op.Add

AV.Op.AddUnique

AV.Op.Increment

AV.Op.Relation

AV.Op.Remove

AV.Op.Set

AV.Op.Unset

AV.Promise

AV.Push

AV.Query

AV.Relation

AV.Role

AV.Router

AV.SearchQuery

AV.SearchSortBuilder

AV.Status

AV.User

AV.View

avosExpressCookieSession

avosExpressHttpsRedirect

				Class AV.Role

				A AV.Role is a local representation of a role persisted to the AV
cloud.

Defined in:  av.js.

Class Summary

						Constructor Attributes
						Constructor Name and Description

								AV.Role()

Represents a Role on the AV server.

Method Summary

							Method Attributes
							Method Name and Description

constructor(name, acl)

Constructs a new AVRole with the given name and ACL.

getName()

Gets the name of the role.

getRoles()

Gets the AV.Relation for the AV.Roles that are direct
children of this role.

getUsers()

Gets the AV.Relation for the AV.Users that are direct
children of this role.

setName(name, options)

Sets the name for a role.

					Class Detail

						AV.Role()

					Represents a Role on the AV server. Roles represent groupings of
Users for the purposes of granting permissions (e.g. specifying an ACL
for an Object). Roles are specified by their sets of child users and
child roles, all of which are granted any permissions that the parent
role has.

Roles must have a name (which cannot be changed after creation of the
role), and must specify an ACL.

					Method Detail

					constructor(name, acl)

						Constructs a new AVRole with the given name and ACL.

Parameters:

									{String} name

The name of the Role to create.

									{AV.ACL} acl

The ACL for this role. Roles must have an ACL.

					{String}
					getName()

						Gets the name of the role.  You can alternatively call role.get("name")

Returns:

{String} the name of the role.

					{AV.Relation}
					getRoles()

						Gets the AV.Relation for the AV.Roles that are direct
children of this role. These roles' users are granted any privileges that
this role has been granted (e.g. read or write access through ACLs). You
can add or remove child roles from this role through this relation.

This is equivalent to calling role.relation("roles")

Returns:

{AV.Relation} the relation for the roles belonging to this
    role.

					{AV.Relation}
					getUsers()

						Gets the AV.Relation for the AV.Users that are direct
children of this role. These users are granted any privileges that this
role has been granted (e.g. read or write access through ACLs). You can
add or remove users from the role through this relation.

This is equivalent to calling role.relation("users")

Returns:

{AV.Relation} the relation for the users belonging to this
    role.

					setName(name, options)

						Sets the name for a role. This value must be set before the role has
been saved to the server, and cannot be set once the role has been
saved.

  A role's name can only contain alphanumeric characters, _, -, and
  spaces.

This is equivalent to calling role.set("name", name)

Parameters:

									{String} name

The name of the role.

									{Object} options

Standard options object with success and error
    callbacks.

			Documentation generated by JsDoc Toolkit 2.4.0 on Wed Aug 05 2015 11:22:59 GMT+0800 (CST)


text:
<!DOCTYPE html>

      <title>Bear Talk Podcast</title>
      <meta name = "viewport" content = "width = device-width, initial-scale = 1.0">

      <link href = "css/bootstrap.css" rel = "stylesheet">

<link href="font-awesome/css/font-awesome.min.css" rel="stylesheet" type="text/css">
<link href="http://fonts.googleapis.com/css?family=Montserrat:400,700" rel="stylesheet" type="text/css">
<link href="http://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic" rel="stylesheet" type="text/css">

               <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
                   Toggle navigation

               </button>

the bear talk podcast

                       home

                       about us

                     subscribe

may episodes

episode #37

Jonathan Taylor ThomasCast

5/31/16 WE LOST CONNOR! If you see him anywhere, let us know! This week, Jared and Kyle talk about a lot of different things, Kyle gets movies confused, and Jared tells us all about the makeup of buttermilk. #FindConnor

episode #36

Riel Talk About New Hampshire

5/24/16 This week, the boys interview Cole Riel, a fellow podcaster and friend of the show. We ask him all types of questions about life, liberty, and the pursuit of knowledge about New Hampshire. We had a real fun time recording this one, we hope you have as good of a time listening to it! Also this might be our least explicit episode yet!

episode #35

Super Food Super Show

5/17/16 The boys talk about super food, read viewer mail, sweet potatoes, terrible people on twitter, animals, soylent, and more! This is a good episode. Listen!

episode #34

New England Meme

5/11/16 The boys come out this week addressing concerns about our respect and like/dislike of various New England States. Connor also tells some stories about weird and rude people, and Kyle hears about how Theme Song creator Matt Farley has gone viral!

episode #33

The One Where Kyle's Mic Works

5/3/16 DISCLAIMER: Kyle fucked up and used his laptop internal microphone to record this episode, so the audio isn't up to bar. It's still a good episode though! The boys take on 20 questions, and talk about graduation!

                         ← Older

                         Newer →

archive

january

february

march

april

may

june

july

august

september

october

november

december

contact us

                              Bear Tweets

                                Connor

                                Jared

                                Kyle

                                beartalkpodcast@gmail.com

Copyright © Bear Talk Podcast


text:
#include <Python.h>

#include "common_types.h"

#include "my_macro.h"
#include "my_error.h"
#include "my_common_types.h"
#include "my_voice_engine.h"
#include "my_channel.h"
#include "my_voe_encryption.h"

using namespace webrtc;

class EncryptionCallback: public Encryption {
public:
    EncryptionCallback(PyObject* encrypt_o, PyObject* decrypt_o, PyObject* encrypt_rtcp_o, PyObject* decrypt_rtcp_o, PyObject* userdata);
    ~EncryptionCallback();
    void encrypt(int channel_no, unsigned char* in_data, unsigned char* out_data, int bytes_in, int* bytes_out);
    void decrypt(int channel_no, unsigned char* in_data, unsigned char* out_data, int bytes_in, int* bytes_out);
    void encrypt_rtcp(int channel_no, unsigned char* in_data, unsigned char* out_data, int bytes_in, int* bytes_out);
    void decrypt_rtcp(int channel_no, unsigned char* in_data, unsigned char* out_data, int bytes_in, int* bytes_out);

private:
    PyObject *encrypt_o;
    PyObject *decrypt_o;
    PyObject *encrypt_rtcp_o;
    PyObject *decrypt_rtcp_o;
    PyObject *userdata;
};

EncryptionCallback::EncryptionCallback(PyObject* encrypt_o, PyObject* decrypt_o, PyObject* encrypt_rtcp_o, PyObject* decrypt_rtcp_o, PyObject* userdata)
{
    Py_XINCREF(encrypt_o);
    this->encrypt_o = encrypt_o;
    Py_XINCREF(decrypt_o);
    this->decrypt_o = decrypt_o;
    Py_XINCREF(encrypt_rtcp_o);
    this->encrypt_rtcp_o = encrypt_rtcp_o;
    Py_XINCREF(decrypt_rtcp_o);
    this->decrypt_rtcp_o = decrypt_rtcp_o;
    Py_XINCREF(userdata);
    this->userdata = userdata;
}

EncryptionCallback::~EncryptionCallback() {
    Py_XDECREF(this->encrypt_o);
    Py_XDECREF(this->decrypt_o);
    Py_XDECREF(this->encrypt_rtcp_o);
    Py_XDECREF(this->decrypt_rtcp_o);
    Py_XDECREF(this->userdata);
}

#define DEFINE_ENCRYPTION_METHOD(name)          \
void EncryptionCallback::name(int channel, unsigned char* in_data, unsigned char* out_data, int bytes_in, int* bytes_out)   \
{                                                                                \
    if (this->name##_o != NULL) {                                                                                \
        PyObject* input = PyString_FromStringAndSize((const char*) in_data, bytes_in);                                                    \
        PyObject* arglist = Py_BuildValue("(OiO)", this->userdata, channel, input);                                         \
        PyObject* output = PyObject_CallObject(this->name##_o, arglist);                                                    \
        Py_DECREF(arglist);                                                                                \
        if (PyString_Check(output)) {                                                                                \
            *bytes_out = PyString_Size(output);                                                                             \
            memcpy(out_data, PyString_AsString(output), *bytes_out);                                                        \
        }                                                                                \
    }                                                                                \
}

DEFINE_ENCRYPTION_METHOD(encrypt)
DEFINE_ENCRYPTION_METHOD(decrypt)
DEFINE_ENCRYPTION_METHOD(encrypt_rtcp)
DEFINE_ENCRYPTION_METHOD(decrypt_rtcp)

My_METHOD_ARGS_KWARGS(WebRtcChannel, RegisterExternalEncryption)
{
    My_NULL_IF_NULL(self->voe->base, VoEBase)
    My_NULL_IF_NULL(self->voe->encryption, VoEEncryption)
    My_NULL_IF_NEG2(self->ch)

    PyObject* encrypt_o = NULL;
    PyObject* decrypt_o = NULL;
    PyObject* encrypt_rtcp_o = NULL;
    PyObject* decrypt_rtcp_o = NULL;
    PyObject* userdata = Py_None;
    static char *kwlist[] = {"encrypt", "decrypt", "encrypt_rtcp", "decrypt_rtcp", "userdata", NULL};

    if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOOO|O", kwlist, &encrypt_o, &decrypt_o, &encrypt_rtcp_o, &decrypt_rtcp_o, &userdata)) {
        return NULL;
    }

    if (encrypt_o != Py_None && !PyCallable_Check(encrypt_o)) {
        PyErr_SetString(WebRtcError, "encrypt must be callable or None");
        return NULL;
    }
    if (decrypt_o != Py_None && !PyCallable_Check(decrypt_o)) {
        PyErr_SetString(WebRtcError, "decrypt must be callable or None");
        return NULL;
    }
    if (encrypt_rtcp_o != Py_None && !PyCallable_Check(encrypt_rtcp_o)) {
        PyErr_SetString(WebRtcError, "encrypt_rtcp must be callable or None");
        return NULL;
    }
    if (decrypt_rtcp_o != Py_None && !PyCallable_Check(decrypt_rtcp_o)) {
        PyErr_SetString(WebRtcError, "decrypt_rtcp must be callable or None");
        return NULL;
    }

    if (self->encryption_callback != NULL) {
        delete self->encryption_callback;
        self->encryption_callback = NULL;
    }

    if (encrypt_o != Py_None && decrypt_o != Py_None && encrypt_rtcp_o != Py_None && decrypt_rtcp_o != Py_None) {
        self->encryption_callback = new EncryptionCallback(encrypt_o, decrypt_o, encrypt_rtcp_o, decrypt_rtcp_o, userdata);
        self->voe->encryption->RegisterExternalEncryption(self->ch, *self->encryption_callback);
    }
    else {
        self->voe->encryption->DeRegisterExternalEncryption(self->ch);
    }

    Py_INCREF(Py_None);
    return Py_None;
}