text:
import sys
import io
import json
import zipfile

from django.contrib.gis.geos import GEOSGeometry
from django.core.exceptions import ImproperlyConfigured
from django.test import SimpleTestCase, TestCase
from greenwich import Raster
from greenwich.io import MemFileIO

from spillway import renderers
from spillway.collections import Feature, FeatureCollection
from .models import Location, _geom
from .test_models import RasterTestBase, RasterStoreTestBase

class GeoJSONRendererTestCase(SimpleTestCase):
    def setUp(self):
        self.data = Feature(id=1, properties={'name': 'San Francisco'},
                            geometry=_geom)
        self.collection = FeatureCollection(features=[self.data])
        self.r = renderers.GeoJSONRenderer()

    def test_render_feature(self):
        data = json.loads(self.r.render(self.data))
        self.assertEqual(data, self.data)
        self.assertEqual(self.r.render({}), str(Feature()))

    def test_render_feature_collection(self):
        data = json.loads(self.r.render(self.collection))
        self.assertEqual(data, self.collection)
        self.assertEqual(self.r.render([]), str(FeatureCollection()))

    def test_render_list(self):
        data = json.loads(self.r.render([self.data]))
        self.assertEqual(data, self.collection)

class KMLRendererTestCase(SimpleTestCase):
    def setUp(self):
        self.data = {'id': 1,
                     'properties': {'name': 'playground',
                                    'notes': 'epic slide'},
                     'geometry': GEOSGeometry(json.dumps(_geom)).kml}

    def test_render(self):
        rkml = renderers.KMLRenderer()
        self.assertIn(self.data['geometry'], rkml.render(self.data))

    def test_render_kmz(self):
        rkmz = renderers.KMZRenderer()
        stream = io.BytesIO(rkmz.render(self.data))
        self.assertTrue(zipfile.is_zipfile(stream))
        zf = zipfile.ZipFile(stream)
        self.assertIn(self.data['geometry'], zf.read('doc.kml'))

class SVGRendererTestCase(TestCase):
    def setUp(self):
        Location.create()
        self.qs = Location.objects.svg()
        self.svg = self.qs[0].svg
        self.data = {'id': 1,
                     'properties': {'name': 'playground',
                                    'notes': 'epic slide'},
                     'geometry': self.svg}

    def test_render(self):
        rsvg = renderers.SVGRenderer()
        svgdoc = rsvg.render(self.data)
        self.assertIn(self.data['geometry'], svgdoc)

        #serializer = FeatureSerializer(self.qs)
        ##serializer.opts.model = self.qs.model
        #svgdoc = rsvg.render(serializer.data)
        #print svgdoc
        #self.assertIn(self.data['geometry'], svgdoc)

class RasterRendererTestCase(RasterTestBase):
    img_header = 'EHFA_HEADER_TAG'

    def _save(self, drivername):
        memio = MemFileIO()
        with Raster(self.data['path']) as r:
            r.save(memio, drivername)
        return memio

    def assert_format(self, data, format):
        im = self._image(data)
        self.assertEqual(im.format, format)

    def assert_member_formats(self, obj, format):
        memio = self._save(format)
        imgs = [{'file': memio, 'path': 'test'}]
        fp = obj.render(imgs)
        self.assertTrue(memio.closed)
        with zipfile.ZipFile(fp) as zf:
            for name in zf.namelist():
                self.assert_format(zf.read(name), format)
            filecount = len(zf.filelist)
        self.assertEqual(filecount, len(imgs))
        fp.close()

    def test_render_geotiff(self):
        fp = renderers.GeoTIFFRenderer().render(self.data)
        self.assertEqual(fp.read(), self.f.read())

    def test_render_hfa(self):
        memio = self._save('HFA')
        data = renderers.HFARenderer().render(
            {'file': memio, 'path': 'test.img'})
        # Read the image header.
        self.assertEqual(data[:15], self.img_header)
        self.assertTrue(memio.closed)

    def test_render_hfazip(self):
        memio = self._save('HFA')
        fp = renderers.HFAZipRenderer().render(
            {'file': memio, 'path': 'test.img'})
        zf = zipfile.ZipFile(fp)
        for name in zf.namelist():
            self.assertRegexpMatches(name, '(?<!\.img)\.img$')
        self.assertEqual(zf.read(zf.namelist()[0])[:15], self.img_header)

    def test_render_jpeg(self):
        memio = self._save('JPEG')
        imgdata = renderers.JPEGRenderer().render(
            {'file': memio, 'path': 'test.jpg'})
        self.assert_format(imgdata, 'JPEG')

    def test_render_jpegzip(self):
        self.assert_member_formats(renderers.JPEGZipRenderer(), 'JPEG')

    def test_render_png(self):
        memio = self._save('PNG')
        imgdata = renderers.JPEGRenderer().render(
            {'file': memio, 'path': 'test.png'})
        self.assert_format(imgdata, 'PNG')

    def test_render_pngzip(self):
        self.assert_member_formats(renderers.PNGZipRenderer(), 'PNG')

    def test_render_tifzip(self):
        tifs = [self.data, self.data]
        fp = renderers.GeoTIFFZipRenderer().render(tifs)
        zf = zipfile.ZipFile(fp)
        self.assertEqual(len(zf.filelist), len(tifs))
        self.assertTrue(all(name.endswith('.tif') for name in zf.namelist()))

class MapnikRendererTestCase(RasterStoreTestBase):
    def test_render(self):
        ctx = {'bbox': self.object.geom}
        imgdata = renderers.MapnikRenderer().render(
            self.object, renderer_context=ctx)
        im = self._image(imgdata)
        self.assertEqual(im.size, (256, 256))
        self.assertNotEqual(im.getpixel((100, 100)), (0, 0, 0, 0))

    def test_compat(self):
        from spillway import compat
        sys.modules.pop('mapnik')
        sys.path = []
        reload(compat)
        with self.assertRaises(ImproperlyConfigured):
            m = compat.mapnik.Map(128, 128)


text:
#include "unicode/unifunct.h"

U_NAMESPACE_BEGIN

UOBJECT_DEFINE_ABSTRACT_RTTI_IMPLEMENTATION(UnicodeFunctor)

UnicodeFunctor::~UnicodeFunctor() {}

UnicodeMatcher* UnicodeFunctor::toMatcher() const {
    return 0;
}

UnicodeReplacer* UnicodeFunctor::toReplacer() const {
    return 0;
}

U_NAMESPACE_END

//eof

redpajama_set_name: RedPajamaGithub


text:
"""Models for assessing primal feasibility"""
from .set import ConstraintSet
from ..nomials import Variable, VectorVariable, parse_subs, NomialArray
from ..keydict import KeyDict
from .. import NamedVariables, SignomialsEnabled
from ..small_scripts import appendsolwarning, initsolwarning, mag

class ConstraintsRelaxedEqually(ConstraintSet):
    """Relax constraints the same amount, as in Eqn. 10 of [Boyd2007].

    Arguments
    ---------
    constraints : iterable
        Constraints which will be relaxed (made easier).

    Attributes
    ----------
    relaxvar : Variable
        The variable controlling the relaxation. A solved value of 1 means no
        relaxation. Higher values indicate the amount by which all constraints
        have been made easier: e.g., a value of 1.5 means all constraints were
        50 percent easier in the final solution than in the original problem.

    [Boyd2007] : "A tutorial on geometric programming", Optim Eng 8:67-122

    """

    def __init__(self, original_constraints):
        if not isinstance(original_constraints, ConstraintSet):
            original_constraints = ConstraintSet(original_constraints)
        self.original_constraints = original_constraints
        original_substitutions = original_constraints.substitutions

        with NamedVariables("Relax"):
            self.relaxvar = Variable("C")
        with SignomialsEnabled():
            relaxed_constraints = [c.relaxed(self.relaxvar)
                                   for c in original_constraints.flat()]

        ConstraintSet.__init__(self, {
            "minimum relaxation": self.relaxvar >= 1,
            "relaxed constraints": relaxed_constraints}, original_substitutions)

    def process_result(self, result):
        "Warns if any constraints were relaxed"
        super().process_result(result)
        self.check_relaxed(result)

    def check_relaxed(self, result):
        "Adds relaxation warnings to the result"
        initsolwarning(result, "Relaxed Constraints")
        for val, msg in get_relaxed([result["freevariables"][self.relaxvar]],
                                    ["All constraints relaxed by %i%%"]):
            appendsolwarning(msg % (0.9+(val-1)*100), self, result,
                             "Relaxed Constraints")

class ConstraintsRelaxed(ConstraintSet):
    """Relax constraints, as in Eqn. 11 of [Boyd2007].

    Arguments
    ---------
    constraints : iterable
        Constraints which will be relaxed (made easier).

    Attributes
    ----------
    relaxvars : Variable
        The variables controlling the relaxation. A solved value of 1 means no
        relaxation was necessary or optimal for a particular constraint.
        Higher values indicate the amount by which that constraint has been
        made easier: e.g., a value of 1.5 means it was made 50 percent easier
        in the final solution than in the original problem.

    [Boyd2007] : "A tutorial on geometric programming", Optim Eng 8:67-122

    """

    def __init__(self, original_constraints):
        if not isinstance(original_constraints, ConstraintSet):
            original_constraints = ConstraintSet(original_constraints)
        self.original_constraints = original_constraints
        original_substitutions = original_constraints.substitutions
        with NamedVariables("Relax"):
            self.relaxvars = VectorVariable(len(original_constraints), "C")

        with SignomialsEnabled():
            relaxed_constraints = [
                c.relaxed(self.relaxvars[i])
                for i, c in enumerate(original_constraints.flat())]

        ConstraintSet.__init__(self, {
            "minimum relaxation": self.relaxvars >= 1,
            "relaxed constraints": relaxed_constraints}, original_substitutions)

    def process_result(self, result):
        "Warns if any constraints were relaxed"
        super().process_result(result)
        self.check_relaxed(result)

    def check_relaxed(self, result):
        "Adds relaxation warnings to the result"
        relaxed = get_relaxed(result["freevariables"][self.relaxvars],
                              range(len(self["relaxed constraints"])))
        initsolwarning(result, "Relaxed Constraints")
        for relaxval, i in relaxed:
            relax_percent = "%i%%" % (0.5+(relaxval-1)*100)
            oldconstraint = self.original_constraints[i]
            newconstraint = self["relaxed constraints"][i][0]
            subs = {self.relaxvars[i]: relaxval}
            relaxdleft = newconstraint.left.sub(subs)
            relaxdright = newconstraint.right.sub(subs)
            oldleftstr = str(oldconstraint.left)
            relaxedleftstr = str(relaxdleft)
            padding = len(relaxedleftstr) - len(oldleftstr)
            if padding > 0:
                oldleftstr = " " * padding + oldleftstr
            elif padding < 0:
                relaxedleftstr = " " * padding + relaxedleftstr
            msg = (" %3i: %5s relaxed, from %s %s %s\n"
                   "                       to %s %s %s"
                   % (i, relax_percent, oldleftstr,
                      oldconstraint.oper, oldconstraint.right,
                      relaxedleftstr, newconstraint.oper, relaxdright))
            appendsolwarning(msg, oldconstraint, result, "Relaxed Constraints")

class ConstantsRelaxed(ConstraintSet):
    """Relax constants in a constraintset.

    Arguments
    ---------
    constraints : iterable
        Constraints which will be relaxed (made easier).

    include_only : set (optional)
        variable names must be in this set to be relaxed

    exclude : set (optional)
        variable names in this set will never be relaxed

    Attributes
    ----------
    relaxvars : Variable
        The variables controlling the relaxation. A solved value of 1 means no
        relaxation was necessary or optimal for a particular constant.
        Higher values indicate the amount by which that constant has been
        made easier: e.g., a value of 1.5 means it was made 50 percent easier
        in the final solution than in the original problem. Of course, this
        can also be determined by looking at the constant's new value directly.
    """
    # pylint:disable=too-many-locals
    def __init__(self, constraints, *, include_only=None, exclude=None):
        exclude = frozenset(exclude) if exclude else frozenset()
        include_only = frozenset(include_only) if include_only else frozenset()
        with NamedVariables("Relax") as (self.lineage, _):
            pass  # gives this model the correct lineage.

        if not isinstance(constraints, ConstraintSet):
            constraints = ConstraintSet(constraints)
        substitutions = KeyDict(constraints.substitutions)
        constants, _, linked = parse_subs(constraints.vks, substitutions)
        if linked:
            kdc = KeyDict(constants)
            constrained_varkeys = constraints.constrained_varkeys()
            constants.update({k: f(kdc) for k, f in linked.items()
                              if k in constrained_varkeys})

        self._derelax_map = {}
        relaxvars, self.freedvars, relaxation_constraints = [], [], {}
        for const, val in sorted(constants.items(), key=lambda i: i[0].eqstr):
            if val == 0:
                substitutions[const] = 0
                continue
            if include_only and const.name not in include_only:
                continue
            if const.name in exclude:
                continue
            # set up the lineage
            const.descr.pop("gradients", None)  # nothing wants an old gradient
            newconstd = const.descr.copy()
            newconstd.pop("veckey", None)  # only const wants an old veckey
            # everything but const wants a new lineage, to distinguish them
            newconstd["lineage"] = (newconstd.pop("lineage", ())
                                    + (self.lineage[-1],))
            # make the relaxation variable, free to get as large as it needs
            relaxedd = newconstd.copy()
            relaxedd["unitrepr"] = "-"  # and unitless, importantly
            relaxvar = Variable(**relaxedd)
            relaxvars.append(relaxvar)
            # the newly freed const can acquire a new value
            del substitutions[const]
            freed = Variable(**const.descr)
            self.freedvars.append(freed)
            # becuase the make the newconst will take its old value
            newconstd["lineage"] += (("OriginalValues", 0),)
            newconst = Variable(**newconstd)
            substitutions[newconst] = val
            self._derelax_map[newconst.key] = const
            # add constraints so the newly freed's wiggle room
            # is proportional to the value relaxvar, and it can't antirelax
            relaxation_constraints[str(const)] = [relaxvar >= 1,
                                                  newconst/relaxvar <= freed,
                                                  freed <= newconst*relaxvar]
        ConstraintSet.__init__(self, {
            "original constraints": constraints,
            "relaxation constraints": relaxation_constraints})
        self.relaxvars = NomialArray(relaxvars)  # so they can be .prod()'d
        self.substitutions = substitutions
        self.constants = constants

    def process_result(self, result):
        "Transfers constant sensitivities back to the original constants"
        super().process_result(result)
        constant_senss = result["sensitivities"]["variables"]
        for new_constant, former_constant in self._derelax_map.items():
            constant_senss[former_constant] = constant_senss[new_constant]
            del constant_senss[new_constant]
        self.check_relaxed(result)

    def check_relaxed(self, result):
        "Adds relaxation warnings to the result"
        relaxed = get_relaxed([result["freevariables"][r]
                               for r in self.relaxvars], self.freedvars)
        initsolwarning(result, "Relaxed Constants")
        for (_, freed) in relaxed:
            msg = ("  %s: relaxed from %-.4g to %-.4g"
                   % (freed,
                      mag(self.constants[freed.key]),
                      mag(result["freevariables"][freed])))
            appendsolwarning(msg, freed, result, "Relaxed Constants")

def get_relaxed(relaxvals, mapped_list):
    "Returns 'relaxed' vals (those above an arbitrary threshold of 1.01)."
    sortrelaxed = sorted(zip(relaxvals, mapped_list), key=lambda x: -x[0])
    mostrelaxed = max(sortrelaxed[0][0], 1.01)
    for i, (val, _) in enumerate(sortrelaxed):
        if val <= 1.01 and (val-1) <= (mostrelaxed-1)/10:
            return sortrelaxed[:i]
    return sortrelaxed


text:
<?php

namespace Application\Controller;

use Zend\Mvc\Controller\AbstractRestfulController;
use Zend\View\Model\JsonModel;

use Application\Entity\Album;

/**
 * Class IndexController
 *
 * @package Application\Controller
 *
 *              -Create-            -Read-              -Update-                    -Delete-
 *              POST	            GET	                PUT	                        DELETE
 * /app	        create record	    Album list  	    Update Album data   	    delete all
 * /app/42	    error	            Album Data  	    Update Album    	        delete album
 */

class IndexController extends AbstractRestfulController
{
    const ALBUM_ENTITY = 'Application\Entity\Album';

    protected $mapper;

    /**
     * @return JsonModel
     */
    public function getList()
    {
        $results = $this->mapper->findAll();

        $albums = $this->getArrayResult($results);

        return new JsonModel([
            'albums' => $albums
        ]);
    }

    /**
     * @param mixed $id
     *
     * @return JsonModel
     */
    public function get($id)
    {
        $album = $this->mapper->findById($id);

        if (!is_null($album)) {
            $album = $album->getArrayCopy();
        }

        return new JsonModel([
            'album' => $album
        ]);
    }

    /**
     * @param mixed $data
     *
     * @return JsonModel
     */
    public function create($data)
    {
        $newAlbum = new Album();

        if (!isset($data['artist'], $data['title'])) {
            return new JsonModel([
                'error' => 'missing params'
            ]);
        }

        $newAlbum->setArtist($data['artist']);
        $newAlbum->setTitle($data['title']);

        $this->mapper->persist($newAlbum);

        return new JsonModel([
            'id' => $newAlbum->getId()
        ]);
    }

    /**
     * @param mixed $id
     * @param mixed $data (data={"artist":"new artist", "title":"new title"})
     *
     * @return JsonModel
     */
    public function update($id, $data)
    {
        $album = $this->mapper->findById($id);

        if (is_null($album)) {
            return new JsonModel([
                'error' => 'no such album'
            ]);
        }

        /**
         * @todo get data from PUT request
         */
        $data = (array)json_decode($data['data']);

        $album->setArtist($data['artist']);
        $album->setTitle($data['title']);

        $this->mapper->persist($album);

        return new JsonModel([
            'id' => $album->getId()
        ]);
    }

    /**
     * @param mixed $id
     *
     * @return JsonModel
     */
    public function delete($id)
    {
        $album = $this->mapper->findById($id);

        $this->mapper->remove($album);

        return new JsonModel([
            'data' => 'deleted',
        ]);
    }

    /**
     * @param mixed $mapper
     */
    public function setMapper($mapper)
    {
        $this->mapper = $mapper;
    }

    /**
     * @param $results
     *
     * @return array
     */
    protected function getArrayResult($results)
    {
        $albums = []; $count = 0;
        foreach($results as $result) {
            $albums[$count]['id']       = $result->getId();
            $albums[$count]['artist']   = $result->getArtist();
            $albums[$count]['title']    = $result->getTitle();

            $count++;
        }

        return $albums;
    }
}


text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
// Code generated by Microsoft (R) AutoRest Code Generator.

package com.azure.ai.formrecognizer.documentanalysis.implementation.models;

import com.azure.core.annotation.Fluent;
import com.azure.core.util.CoreUtils;
import com.fasterxml.jackson.annotation.JsonProperty;

/** Document analysis parameters. */
@Fluent
public final class AnalyzeDocumentRequest {
    /*
     * Document URL to analyze
     */
    @JsonProperty(value = "urlSource")
    private String urlSource;

    /*
     * Base64 encoding of the document to analyze
     */
    @JsonProperty(value = "base64Source")
    private byte[] base64Source;

    /** Creates an instance of AnalyzeDocumentRequest class. */
    public AnalyzeDocumentRequest() {}

    /**
     * Get the urlSource property: Document URL to analyze.
     *
     * @return the urlSource value.
     */
    public String getUrlSource() {
        return this.urlSource;
    }

    /**
     * Set the urlSource property: Document URL to analyze.
     *
     * @param urlSource the urlSource value to set.
     * @return the AnalyzeDocumentRequest object itself.
     */
    public AnalyzeDocumentRequest setUrlSource(String urlSource) {
        this.urlSource = urlSource;
        return this;
    }

    /**
     * Get the base64Source property: Base64 encoding of the document to analyze.
     *
     * @return the base64Source value.
     */
    public byte[] getBase64Source() {
        return CoreUtils.clone(this.base64Source);
    }

    /**
     * Set the base64Source property: Base64 encoding of the document to analyze.
     *
     * @param base64Source the base64Source value to set.
     * @return the AnalyzeDocumentRequest object itself.
     */
    public AnalyzeDocumentRequest setBase64Source(byte[] base64Source) {
        this.base64Source = CoreUtils.clone(base64Source);
        return this;
    }
}


text:
import csv
import os
import math
import argparse
# Contains methods used to build and parse XML
import xml.etree.ElementTree as ET
import requests  # Contains methods used to make HTTP requests
import sys

# The following packages are used to build a multi-part/mixed request.
# They are contained in the 'requests' library.
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata

# The code extracts values from the XML response by using
# ElementTree. This requires using a namespace when searching the XML.
# For details, see:
#      https://docs.python.org/3/library/xml.etree.elementtree.html has more details

# The namespace for the REST API is 'http://tableausoftware.com/api' for Tableau Server 9.0 or 'http://tableau.com/api' for Tableau Server 9.1 or later
xmlns = {'t': 'http://tableau.com/api'}

API_ROOT = "/api/2.3"

####
# Functions for constructing HTTP multi-part requests and dealing with errors
####

def _make_multipart(parts):
    """
    Creates one "chunk" for a multi-part upload.

    'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).

    Returns the post body and the content type string.

    For more information, see this post:
        http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests
    """

    mime_multipart_parts = []

    for name, (filename, blob, content_type) in parts.items():
        multipart_part = RequestField(name=name, data=blob, filename=filename)
        multipart_part.make_multipart(content_type=content_type)
        mime_multipart_parts.append(multipart_part)

    post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
    content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
    return post_body, content_type

def _handle_error(server_response):
    """
    Parses an error response for the error subcode and detail message
    and then displays them.

    Returns the error code and error message.
    """
    print("An error occurred")
    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    error_code, error_detail = None, None
    if xml_response.find('t:error', namespaces=xmlns):
        error_code = xml_response.find('t:error', namespaces=xmlns).attrib.get('code')
        error_detail = xml_response.find('.//t:detail', namespaces=xmlns).text
        print("\tError code: " + str(error_code))
        print("\tError detail: " + str(error_detail))
    else:
        print(xml_response)
    return error_code, error_detail

def _exit_if_error(server_response, expected_status = 200):
    if server_response.status_code != expected_status:
        _handle_error(server_response)
        exit(1)

def _encode_for_display(text):
    """
    Encodes strings so they can display as ASCII in a Windows terminal window.
    This function also encodes strings for processing by xml.etree.ElementTree functions.

    Returns an ASCII-encoded version of the text. Unicode characters are converted to ASCII placeholders (for example, "?").
    """
    return text.encode('ascii', errors="backslashreplace").decode('utf-8')

####
# Functions for authentication (sign in and sign out)
####

def sign_in(name, password, site=""):
    """
    Signs in to the server specified in the global SERVER variable.

    'name'     is the name (not ID) of the user to sign in as.
               Note that most of the functions in this example require that the user
               have server administrator permissions.
    'password' is the password for the user.
    'site'     is the ID (as a string) of the site on the server to sign in to. The
               default is "", which signs in to the default site.

    Returns the authentication token and the site ID.
    """
    url = SERVER + API_ROOT + "/auth/signin"

    # Builds the request
    xml_payload_for_request = ET.Element('tsRequest')
    credentials_element = ET.SubElement(xml_payload_for_request, 'credentials', name=name, password=password)
    site_element = ET.SubElement(credentials_element, 'site', contentUrl=site)
    xml_payload_for_request = ET.tostring(xml_payload_for_request)

    # Makes the request to Tableau Server
    server_response = requests.post(url, data=xml_payload_for_request)
    _exit_if_error(server_response)

    # Reads and parses the response
    xml_response = ET.fromstring(_encode_for_display(server_response.text))

    # Gets the token and site ID
    token = xml_response.find('t:credentials', namespaces=xmlns).attrib.get('token')
    site_id = xml_response.find('.//t:site', namespaces=xmlns).attrib.get('id')
    user_id = xml_response.find('.//t:user', namespaces=xmlns).attrib.get('id')
    return token, site_id, user_id

def sign_out():
    """
    Destroys the active session
    """
    global TOKEN
    url = SERVER + API_ROOT + "/auth/signout"
    server_response = requests.post(url, headers={'x-tableau-auth': TOKEN})
    TOKEN = None
    return

####
# Functions for making queries to get information about content on the server
####

def query_projects():
    """
    Returns a list of projects on the site (a list of <project> elements).

    The function paginates over results (if required) using a page size of 100.
    """
    pageNum, pageSize = 1, 100
    url = SERVER + API_ROOT + "/sites/{0}/projects".format(SITE_ID)
    paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, pageNum)

    server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
    server_response.encoding = "utf-8";
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))

    total_count_of_projects = int(xml_response.find('t:pagination', namespaces=xmlns).attrib.get('totalAvailable'))

    if total_count_of_projects > pageSize:
        projects = []
        projects.extend(xml_response.findall('.//t:project', namespaces=xmlns))
        number_of_pages = int(math.ceil(total_count_of_projects / pageSize))

        # Starts from page 2 because page 1 has already been returned
        for page in range(2, number_of_pages + 1):
            paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, page)
            server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
            _exit_if_error(server_response)

            projects_from_page = ET.fromstring(_encode_for_display(server_response.text)).findall('.//t:project', namespaces=xmlns)
            projects.extend(projects_from_page)
    else:
        projects = xml_response.findall('.//t:project', namespaces=xmlns)
    return projects

def _query_users(query_url):
    pageNum, pageSize = 1, 100
    paged_url = query_url + "?pageSize={}&pageNumber={}".format(pageSize, pageNum)
    server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    total_count_of_users = int(xml_response.find('t:pagination', namespaces=xmlns).attrib.get('totalAvailable'))
    if total_count_of_users > pageSize:
        users = []  # A list to hold the users returned from the server
        users.extend(xml_response.findall('.//t:user', namespaces=xmlns))
        number_of_pages = int(math.ceil(total_count_of_users / pageSize))
        # Starts from page 2 because page 1 has already been returned
        for page in range(2, number_of_pages + 1):
            paged_url = query_url + "?pageSize={}&pageNumber={}".format(pageSize, page)
            server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
            _exit_if_error(server_response)

            users_from_page = ET.fromstring(_encode_for_display(server_response.text)).findall('.//t:user', namespaces=xmlns)
            # Adds the new page of users to the list
            users.extend(users_from_page)
    else:
        users = xml_response.findall('.//t:user', namespaces=xmlns)
    return users

def query_site_users():
    """
    Returns a list of users on the site (a list of <user> elements).
    """
    url = SERVER + API_ROOT + "/sites/{0}/users".format(SITE_ID)
    return _query_users(url)

def query_groups():
    """
    Returns a list of groups on the site (a list of <group> elements).

    The function paginates over the results (if required) using a page size of 100.
    """
    pageNum, pageSize = 1, 100
    url = SERVER + API_ROOT + "/sites/{0}/groups".format(SITE_ID)
    paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, pageNum)
    server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    total_count_of_groups = int(xml_response.find('t:pagination', namespaces=xmlns).attrib.get('totalAvailable'))
    if total_count_of_groups > pageSize:
        groups = []  # A list to hold the groups returned from the server
        groups.extend(xml_response.findall('.//t:group', namespaces=xmlns))
        number_of_pages = int(math.ceil(total_count_of_groups / pageSize))
        # Starts from page 2 because page 1 has already been returned
        for page in range(2, number_of_pages + 1):
            paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, page)
            server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
            _exit_if_error(server_response)

            groups_from_page = ET.fromstring(_encode_for_display(server_response.text)).findall('.//t:group', namespaces=xmlns)
            # Adds the new page of groups to the list
            groups.extend(groups_from_page)
    else:
        groups = xml_response.findall('.//t:group', namespaces=xmlns)
    return groups

def query_group_users(site_id, group):
    url = SERVER + API_ROOT + "/sites/{0}/groups/{1}/users".format(site_id, group.get('id'))
    return _query_users(url)

def query_workbooks(site_id):
    """
    Returns a list of workbooks on the site (a list of <workbook> elements).

    The function paginates over the results (if required) using a page size of 100.
    """
    pageNum, pageSize = 1, 100
    url = SERVER + API_ROOT + "/sites/{0}/workbooks".format(site_id)
    paged_url = url + "?pageSize={0}&pageNumber={1}".format(pageSize, pageNum)
    server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    total_count_of_workbooks = int(xml_response.find('t:pagination', namespaces=xmlns).attrib.get('totalAvailable'))
    if total_count_of_workbooks > pageSize:
        workbooks = []  # This list wil hold the users returned from Server
        workbooks.extend(xml_response.findall('.//t:workbook', namespaces=xmlns))
        number_of_pages = int(math.ceil(total_count_of_workbooks / pageSize))
        # Starts from page 2 since page 1 has already been returned
        for page in range(2, number_of_pages + 1):
            paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, page)
            server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
            _exit_if_error(server_response)

            workbooks_from_page = ET.fromstring(_encode_for_display(server_response.text)).findall('.//t:workbook', namespaces=xmlns)
            # Adds the new page of workbooks to the list
            workbooks.extend(workbooks_from_page)
    else:
        workbooks = xml_response.findall('.//t:workbook', namespaces=xmlns)
    return workbooks

def query_datasources(site_id):
    """
    Returns a list of datasources on the site (a list of <datasource> elements).

    The function paginates over the results (if required) using a page size of 100.
    """
    pageNum, pageSize = 1, 100
    url = SERVER + API_ROOT + "/sites/{0}/datasources".format(site_id)
    paged_url = url + "?pageSize={0}&pageNumber={1}".format(pageSize, pageNum)
    server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    total_count_of_datasources = int(xml_response.find('t:pagination', namespaces=xmlns).attrib.get('totalAvailable'))
    if total_count_of_datasources > pageSize:
        datasources = []  # This list wil hold the users returned from Server
        datasources.extend(xml_response.findall('.//t:datasource', namespaces=xmlns))
        number_of_pages = int(math.ceil(total_count_of_datasources / pageSize))
        # Starts from page 2 since page 1 has already been returned
        for page in range(2, number_of_pages + 1):
            paged_url = url + "?pageSize={}&pageNumber={}".format(pageSize, page)
            server_response = requests.get(paged_url, headers={"x-tableau-auth": TOKEN})
            _exit_if_error(server_response)

            datasources_from_page = ET.fromstring(_encode_for_display(server_response.text)).findall('.//t:datasource', namespaces=xmlns)
            # Adds the new page of datasources to the list
            datasources.extend(datasources_from_page)
    else:
        datasources = xml_response.findall('.//t:datasource', namespaces=xmlns)
    return datasources

def download_datasource(site_id, datasource, target_file):
    url = SERVER + API_ROOT + "/sites/{0}/datasources/{1}/content".format(site_id, datasource.get('id'))
    server_response = requests.get(url, headers={"x-tableau-auth": TOKEN})
    with open(target_file, 'wb+') as f:
        for chunk in server_response.iter_content(chunk_size = 1024 * 1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)

def download_workbook(site_id, workbook, target_file):
    url = SERVER + API_ROOT + "/sites/{0}/workbooks/{1}/content".format(site_id, workbook.get('id'))
    server_response = requests.get(url, headers={"x-tableau-auth": TOKEN})
    with open(target_file, 'wb') as f:
        for chunk in server_response.iter_content(chunk_size = 1024 * 1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)

####
# Functions for publishing a workbook
####

def start_upload_session():
    """
    Creates a POST request that initiates a file upload session.

    Returns a session ID that is used by subsequent functions to identify the upload session.
    """
    url = SERVER + "/api/2.3/sites/{0}/fileUploads".format(SITE_ID)
    server_response = requests.post(url, headers={"x-tableau-auth": TOKEN})
    if server_response.status_code != 201:
        print(_encode_for_display(server_response.text))
        sys.exit(1)
    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    uploadSessionId = xml_response.find('t:fileUpload', namespaces=xmlns).attrib.get('uploadSessionId')
    return uploadSessionId

def _read_in_chunks(file_name, chunk_size=None):
    """
    Reads a file piece by piece.

    'file_name'  is the name of the file to read.
    'chunk_size' is the amount of the file (in bytes) to read at once. If no
                 value is provided, the function reads 4K chunks.
    """
    if chunk_size is None:
        chunk_size = 4096
    with open(file_name, 'rb') as file_object:
        while True:
            data = file_object.read(chunk_size)
            if not data:
                break
            yield data

def put_chunk(chunk, session):
    """
    Makes a PUT request to upload a "chunk" of a file.

    'chunk'    is the portion of the file to publish.
    'session'  is the file upload session (returned from start_upload_session()).

    The request payload is empty in API v2.0, so an empty bytestring is generated in this function.
    """
    url = SERVER + "/api/2.3/sites/{0}/fileUploads/{1}".format(SITE_ID, session)
    payload, content_type = _make_multipart({'request_payload': ('', b'', 'text/xml'),
                                             'tableau_file': ('file', chunk, 'application/octet-stream')})

    print('\tPublishing a chunk ...')
    server_response = requests.put(url, data=payload, headers={"x-tableau-auth": TOKEN, "content-type": content_type})

def publish_with_chunk(fn, name, project_id):
    """
    Publishes a file to Tableau Server in chunks.

    'fn'         is the name of the file to publish.
    'name'       is the workbook name as a string.
    'project_id' is the ID (as a string) of the project to publish to.

    Returns a list of <workbook> elements.
    """
    # Initiates an upload session
    uploadID = start_upload_session()
    # Reads the source file in chunks (default is 100KB)
    chunks = _read_in_chunks(fn, 102400)
    # Loops through the chunks, builds a request payload, and makes a PUT request
    # The information should NOT be encoded (e.g. base-64 or UTF-8)
    for chunk in chunks:
        put_chunk(chunk, uploadID)

    # Builds a request for publishing
    xml_payload_for_request = ET.Element('tsRequest')
    workbook_element = ET.SubElement(xml_payload_for_request, 'workbook', name=name)
    p = ET.SubElement(workbook_element, 'project', id=project_id)
    xml_payload_for_request = ET.tostring(xml_payload_for_request)
    payload, content_type = _make_multipart({'request_payload': ('', xml_payload_for_request, 'text/xml')})

    url = SERVER + "/api/2.3/sites/{0}/workbooks".format(SITE_ID)
    url += "?uploadSessionId={}".format(uploadID)
    url += "&workbookType={}".format(fn.split('.')[1])  # Get extension
    url += "&overwrite=true"

    server_response = requests.post(url, data=payload, headers={'x-tableau-auth': TOKEN, 'content-type': content_type})
    if server_response.status_code != 201:
        print(_encode_for_display(server_response.text))
        sys.exit(1)
    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    return xml_response.find('t:workbook', namespaces=xmlns)

def publish_simple(fn, name, project_id):
    """
    Publishes a file to Tableau Server in a single request.

    'fn'         is the name of the file to publish.
    'name'       is the workbook name as a string.
    'project_id' is the ID (as a string) of the project to publish to.

    Returns a list of <workbook> elements.
    """
    with open(fn, 'rb') as f:
        workbook_bytes = f.read()

    # Builds the publish request
    xml_payload_for_request = ET.Element('tsRequest')
    workbook_element = ET.SubElement(xml_payload_for_request, 'workbook', name=name)
    p = ET.SubElement(workbook_element, 'project', id=project_id)
    xml_payload_for_request = ET.tostring(xml_payload_for_request)

    payload, content_type = _make_multipart(
        {'request_payload': ('', xml_payload_for_request, 'text/xml'),
         'tableau_workbook': (fn, workbook_bytes, 'application/octet-stream')})

    url = SERVER + "/api/2.3/sites/{0}/workbooks".format(SITE_ID)
    url += "?workbookType={}".format(fn.split('.')[1])  # Get extension
    url += "&overwrite=true"
    server_response = requests.post(url, data=payload, headers={'x-tableau-auth': TOKEN, 'content-type': content_type})
    if server_response.status_code != 201:
        print(_encode_for_displayserver_response.text)
        sys.exit(1)
    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    return xml_response.find('t:workbook', namespaces=xmlns)

####
# Functions for setting permissions
####

def create_group(name):
    """
    Creates a group on Tableau Server (local, not Active Directory).
    If the group already exists, the function finds the ID of the
    specified group.

    Note: This function illustrates how to check for error codes
    in the response. The pattern shown here can be used in other
    functions.

    'name'  is the name of the group tp create.

    Returns a <group> element with information about the specified group.
    """
    url = SERVER + "/api/2.3/sites/{0}/groups".format(SITE_ID)
    xml_payload_for_request = ET.Element('tsRequest')
    group = ET.SubElement(xml_payload_for_request, 'group', name=name)
    xml_payload_for_request = ET.tostring(xml_payload_for_request)
    server_response = requests.post(url, data=xml_payload_for_request, headers={'x-tableau-auth': TOKEN})

    # Checks HTTP status code. If the code is anything _except_ success (here, 201),
    # the code reads the <error> block from the response. The error code
    # in the block indicates the specific issue. The documentation for each method
    # provides a list of error codes that might be returned for that method.
    if server_response.status_code != 201:
        error, detail = _handle_error(server_response)
        # 409009 is the error code when the group already exists.
        if error == "409009":
            # A group with the specified name already exists. Therefore, gets a list
            # of existing groups and finds the ID of the group with the specified name.
            groups = query_groups()
            for group in groups:
                if group.get('name') == name:
                    return group
            else:
                print(detail)
                sys.exit(1) # Exit the program altogether
    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    return xml_response.find('t:group', namespaces=xmlns)

def set_group_permissions_for_workbook(workbook_id, group_id, permissions_map):
    """
    Adds a set of permissions for a single group on a single workbook

    'workbook_id'      is the ID (as a string) of the workbook to set permissions on.
    'group_id'         is the ID (as a string) of the group to give permissions to.
    'permissions_map'  is a dictionary that provides key-value pairs of the
                       format "permissionName: Allow/Deny ".

    Returns the <granteeCapabilities> element from the response.
    """
    url = SERVER + "/api/2.3/sites/{0}/workbooks/{1}/permissions".format(SITE_ID, workbook_id)
    xml_payload_for_request = ET.Element('tsRequest')
    permissions = ET.SubElement(xml_payload_for_request, 'permissions')
    granteeCapabilities = ET.SubElement(permissions, 'granteeCapabilities')
    user_xml = ET.SubElement(granteeCapabilities, 'group', id=group_id)
    capabilities = ET.SubElement(granteeCapabilities, 'capabilities')
    for perm, mode in permissions_map.items():
        capabilities.append(ET.Element('capability', name=perm, mode=mode))
    xml_payload_for_request = ET.tostring(xml_payload_for_request)
    server_response = requests.put(url, data=xml_payload_for_request, headers={'x-tableau-auth': TOKEN})
    _exit_if_error(server_response)

    xml_response = ET.fromstring(_encode_for_display(server_response.text))
    return xml_response.findall('.//t:granteeCapabilities', namespaces=xmlns)

def get_project_and_owner(obj):
    project = obj.find('t:project', namespaces=xmlns).get('name')
    username = users_map[obj.find('t:owner', namespaces=xmlns).get('id')].get('name')

    return project, username

def create_dir(obj, obj_type, users_map):
    project, username = get_project_and_owner(obj)

    target_dir = '{0}/{1}/{2}/{3}'.format(SITE_URL, project, username, obj_type)
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    return target_dir

####
# Main function. This executes when the script is run from a shell
####

if __name__ == '__main__':
    # TODO: make these not global vars
    SERVER = "https://online.tableau.com"           # Set to the server URL without a trailing slash (/).
    SITE_ADMIN = None                               # Your Tableau Online site admin username.
    PASSWORD = None                                 # Your Tableau Online site admin password
    SITE_URL = None                                 # Yout Tableau Online site url
    CHUNKED = True

    parser = argparse.ArgumentParser(description='Tableau Online Site Content Backup')
    parser.add_argument('--server', help='Tableau Online server, defaults to online.tableau.com')
    parser.add_argument('--siteAdminEmail', help='Email id of site admin')
    parser.add_argument('--siteAdminPassword', help='Password for site admin')
    parser.add_argument('--siteUrl', help='Site Content Url')

    args = parser.parse_args()
    SITE_ADMIN = SITE_ADMIN if not args.siteAdminEmail else args.siteAdminEmail
    PASSWORD = PASSWORD if not args.siteAdminPassword else args.siteAdminPassword
    SITE_URL = SITE_URL if not args.siteUrl else args.siteUrl
    SERVER = SERVER if not args.server else args.server

    if (SITE_ADMIN == None or PASSWORD == None or SITE_URL == None or SERVER == None):
        print("\nMissing arguments")
        parser.print_help()
        sys.exit()

    print('\nUsing the following:')
    print('\nServer     : {}'.format(SERVER))
    print('Site Admin : {}'.format(SITE_ADMIN))
    print('Site Url   : {}'.format(SITE_URL))
    print('Password   : "{}"'.format(PASSWORD))

    # Signs in to get an authentication token and site ID to use later
    print("\nSigning in")

    TOKEN, SITE_ID, MY_USER_ID = sign_in(SITE_ADMIN, PASSWORD, SITE_URL)

    # TODO: check if user is a site admin

    if not os.path.exists(SITE_URL):
        os.makedirs(SITE_URL)

    # Lists all projects on the server
    projects = query_projects()
    with open('{0}/project_mapping.csv'.format(SITE_URL), 'w') as f:
        writer = csv.writer(f)
        for project in projects:
            project_name = project.get('name')
            writer.writerow([project_name, project_name])

    print"\nDownloading users & groups"
    users = query_site_users()
    users_map = {}
    with open('{0}/users.csv'.format(SITE_URL), 'w') as f:
        writer = csv.writer(f)
        for user in users:
            users_map[user.get('id')] = user
            writer.writerow([user.get('name'), user.get('authSetting'), user.get('siteRole')])

    with open('{0}/user_mapping.csv'.format(SITE_URL), 'w') as f:
        writer = csv.writer(f)
        for user in users:
            username = user.get('name')
            writer.writerow([username, username])

    groups = query_groups()
    with open('{0}/groups.csv'.format(SITE_URL), 'w') as f:
        writer = csv.writer(f)
        for group in groups:
            for user in query_group_users(SITE_ID, group):
                writer.writerow([group.get('name'), user.get('name')])

    print("\nDownloading datasources")
    with open('{0}/datasources.csv'.format(SITE_URL), 'w') as datasources_file:
        writer = csv.writer(datasources_file)
        datasources = query_datasources(SITE_ID)
        for datasource in datasources:
            datasource_name = _encode_for_display(datasource.get('name'))
            target_dir = create_dir(datasource, 'datasources', users_map)
            filename = '{0}/{1}.tdsx'.format(target_dir, datasource_name)

            print('\tDownloading datasource: {0} to {1}'.format(datasource_name, filename))
            download_datasource(SITE_ID, datasource, filename)

            project, username = get_project_and_owner(datasource)
            writer.writerow([filename, datasource_name, username, project, datasource.get('id')])

    print("\nDownloading Workbooks")
    with open('{0}/workbooks.csv'.format(SITE_URL), 'w') as workbooks_file:
        writer = csv.writer(workbooks_file)

        workbooks = query_workbooks(SITE_ID)
        for workbook in workbooks:
            workbook_name = _encode_for_display(workbook.get('name'))

            target_dir = create_dir(workbook, 'workbooks', users_map)
            if not os.path.exists(target_dir):
                os.makedirs(target_dir)
            filename = '{0}/{1}.twbx'.format(target_dir, workbook_name)

            # TODO: save the association
            print('\tDownloading workbook: {0} to {1}'.format(workbook_name, filename))
            download_workbook(SITE_ID, workbook, filename)

            project, username = get_project_and_owner(workbook)
            writer.writerow([filename, workbook_name, username, project, workbook.get('id')])

    # Signs out (destroys the current session)
    sign_out()
    print("\nSigned out, and the authentication token has been invalidated")


text:
package org.apache.drill.exec.store.sys.local;

import static org.apache.drill.exec.ExecConstants.DRILL_SYS_FILE_SUFFIX;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;
import org.apache.drill.exec.store.sys.PStore;
import org.apache.drill.exec.store.sys.PStoreConfig;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;

public class LocalPStore<V> implements PStore<V> {
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LocalPStore.class);

  private static final String BLOB_QUALIFIER = "blob";

  private final File basePath;
  private final File blobPath;
  private final PStoreConfig<V> config;
  public LocalPStore(File base, PStoreConfig<V> config) {
    super();
    this.basePath = new File(base, config.getName());
    this.blobPath = new File(basePath, BLOB_QUALIFIER);
    if (!blobPath.exists()) {
      blobPath.mkdirs();
    }
    this.config = config;
  }

  @Override
  public Iterator<Entry<String, V>> iterator() {
    String[] f = basePath.list();
    if (f == null) {
      return Collections.emptyIterator();
    }
    List<String> files = Lists.newArrayList();
    for (String s : f) {
      if (s.endsWith(DRILL_SYS_FILE_SUFFIX)) {
        files.add(s.substring(0, s.length() - DRILL_SYS_FILE_SUFFIX.length()));
      }
    }

    return new Iter(files.iterator());
  }

  private File p(String name, boolean blob) throws IOException {
    Preconditions.checkArgument(
        !name.contains("/") &&
        !name.contains(":") &&
        !name.contains(".."));

    File f = new File(blob ? blobPath : basePath, name + DRILL_SYS_FILE_SUFFIX);
    // do this to check file name.
    f.getCanonicalPath();
    return f;
  }

  @Override
  public V get(String key) {
    return get(key, false);
  }

  @Override
  public V getBlob(String key) {
    return get(key, true);
  }

  protected V get(String key, boolean blob) {
    try (InputStream is = new FileInputStream(p(key, blob))) {
      return config.getSerializer().deserialize(IOUtils.toByteArray(is));
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void put(String key, V value) {
    put(key, false, value);
  }

  @Override
  public void putBlob(String key, V value) {
    put(key, true, value);
  }

  protected void put(String key, boolean blob, V value) {
    try (OutputStream os = new FileOutputStream(p(key, blob))) {
      IOUtils.write(config.getSerializer().serialize(value), os);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public boolean putIfAbsent(String key, V value) {
    try {
      File f = p(key, false);
      if (f.exists()) {
        return false;
      } else {
        put(key, value);
        return true;
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  @Override
  public void delete(String key) {
    try {
      delete(key, false);
      delete(key, true);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  protected void delete(String key, boolean blob) throws IOException {
    try {
      p(key, blob).delete();
    } catch (FileNotFoundException e) { /* ignored */ }
  }

  private class Iter implements Iterator<Entry<String, V>>{

    private Iterator<String> keys;
    private String current;

    public Iter(Iterator<String> keys) {
      super();
      this.keys = keys;
    }

    @Override
    public boolean hasNext() {
      return keys.hasNext();
    }

    @Override
    public Entry<String, V> next() {
      current = keys.next();
      return new DeferredEntry(current);
    }

    @Override
    public void remove() {
      delete(current);
      keys.remove();
    }

    private class DeferredEntry implements Entry<String, V> {

      private String name;

      public DeferredEntry(String name) {
        super();
        this.name = name;
      }

      @Override
      public String getKey() {
        return name;
      }

      @Override
      public V getValue() {
        return get(name);
      }

      @Override
      public V setValue(V value) {
        throw new UnsupportedOperationException();
      }

    }
  }

}


text:
package com.laytonsmith.core.functions;

import com.laytonsmith.PureUtilities.Common.ReflectionUtils;
import com.laytonsmith.annotations.api;
import com.laytonsmith.annotations.core;
import com.laytonsmith.annotations.seealso;
import com.laytonsmith.core.CHVersion;
import com.laytonsmith.core.Optimizable;
import com.laytonsmith.core.ParseTree;
import com.laytonsmith.core.Static;
import com.laytonsmith.core.compiler.FileOptions;
import com.laytonsmith.core.constructs.CArray;
import com.laytonsmith.core.constructs.CFunction;
import com.laytonsmith.core.constructs.CInt;
import com.laytonsmith.core.constructs.CNull;
import com.laytonsmith.core.constructs.CString;
import com.laytonsmith.core.constructs.Construct;
import com.laytonsmith.core.constructs.Target;
import com.laytonsmith.core.environments.Environment;
import com.laytonsmith.core.exceptions.ConfigCompileException;
import com.laytonsmith.core.exceptions.ConfigRuntimeException;
import com.laytonsmith.core.functions.Exceptions.ExceptionType;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

/**
 *
 */
@core
public class Regex {

    public static String docs(){
        return "This class provides regular expression functions. For more details, please see the page on "
                + "[[CommandHelper/Regex|regular expressions]]. Note that all the functions are just passthroughs"
                + " to the Java regex mechanism. If you need to set a flag on the regex, where the api calls"
                + " for a pattern, instead send array('pattern', 'flags') where flags is any of i, m, or s."
                + " Alternatively, using the embedded flag system that Java provides is also valid. Named captures are"
				+ " also supported if you are using Java 7, otherwise they are not supported.";
    }

    @api public static class reg_match extends AbstractFunction implements Optimizable {

		@Override
        public String getName() {
            return "reg_match";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{2};
        }

		@Override
        public String docs() {
            return "array {pattern, subject} Searches for the given pattern, and returns an array with the results. Captures are supported."
                    + " If the pattern is not found anywhere in the subject, an empty array is returned. The indexes of the array"
                    + " follow typical regex fashion; the 0th element is the whole match, and 1-n are the captures specified in"
                    + " the regex.";
        }

		@Override
        public ExceptionType[] thrown() {
            return new ExceptionType[]{ExceptionType.FormatException};
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_2_0;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment env, Construct... args) throws ConfigRuntimeException {
            Pattern pattern = getPattern(args[0], t);
            String subject = args[1].val();
            CArray ret = CArray.GetAssociativeArray(t);
            Matcher m = pattern.matcher(subject);
            if(m.find()){
                ret.set(0, new CString(m.group(0), t), t);
                for(int i = 1; i <= m.groupCount(); i++){
                    if(m.group(i) == null){
                        ret.set(i, CNull.NULL, t);
                    } else {
                        ret.set(i, Static.resolveConstruct(m.group(i), t), t);
                    }
                }
				//Named groups are only supported in Java 7, but we can
				//dynamically enable this feature if they have it.
				Set<String> namedGroups = getNamedGroups(pattern.pattern());
				try{
					for(String key : namedGroups){
						ret.set(key, (String)ReflectionUtils.invokeMethod(Matcher.class, m, "group", new Class[]{String.class}, new Object[]{key}), t);
					}
				} catch(ReflectionUtils.ReflectionException ex){
					throw new ConfigRuntimeException("Named captures are only supported with Java 7.", ExceptionType.FormatException, t);
				}
            }
            return ret;
        }

        @Override
        public ParseTree optimizeDynamic(Target t, List<ParseTree> children, FileOptions fileOptions) throws ConfigCompileException, ConfigRuntimeException {
            if(!children.get(0).getData().isDynamic()){
                getPattern(children.get(0).getData(), t);
            }
            return null;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CONSTANT_OFFLINE,
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.OPTIMIZE_DYNAMIC,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_match('(\\\\d)(\\\\d)(\\\\d)', 'abc123')"),
				//Java 7 can't be assumed to be working on the system running the doc gen, so we'll hardcode these.
				new ExampleScript("Named captures (Only works if your system is running Java 7)",
					"reg_match('abc(?<foo>\\\\d+)(xyz)', 'abc123xyz')", "{0: abc123xyz, 1: 123, 2: xyz, foo: 123}"),
				new ExampleScript("Named captures with backreferences (Only works if your system is running Java 7)",
					"reg_match('abc(?<foo>\\\\d+)def\\\\k<foo>', 'abc123def123')['foo']", "123")
			};
		}

    }

    @api public static class reg_match_all extends AbstractFunction implements Optimizable {

		@Override
        public String getName() {
            return "reg_match_all";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{2};
        }

		@Override
        public String docs() {
            return "array {pattern, subject} Searches subject for all matches to the regular expression given in pattern, unlike reg_match,"
                    + " which just returns the first match.";
        }

		@Override
        public ExceptionType[] thrown() {
            return new ExceptionType[]{ExceptionType.FormatException};
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_2_0;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment env, Construct... args) throws ConfigRuntimeException {
            Pattern pattern = getPattern(args[0], t);
            String subject = args[1].val();
            CArray fret = new CArray(t);
            Matcher m = pattern.matcher(subject);
			Set<String> namedGroups = getNamedGroups(pattern.pattern());
            while(m.find()){
                CArray ret = CArray.GetAssociativeArray(t);
                ret.set(0, new CString(m.group(0), t), t);

                for(int i = 1; i <= m.groupCount(); i++){
                    ret.set(i, new CString(m.group(i), t), t);
                }
				//Named groups are only supported in Java 7, but we can
				//dynamically enable this feature if they have it.
				try{
					for(String key : namedGroups){
						ret.set(key, (String)ReflectionUtils.invokeMethod(Matcher.class, m, "group", new Class[]{String.class}, new Object[]{key}), t);
					}
				} catch(ReflectionUtils.ReflectionException e){
					throw new ConfigRuntimeException("Named captures are only supported with Java 7.", ExceptionType.FormatException, t);
				}
                fret.push(ret);
            }
            return fret;
        }

        @Override
        public ParseTree optimizeDynamic(Target t, List<ParseTree> children, FileOptions fileOptions) throws ConfigCompileException, ConfigRuntimeException {
            if(!children.get(0).getData().isDynamic()){
                getPattern(children.get(0).getData(), t);
            }
            return null;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CONSTANT_OFFLINE,
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.OPTIMIZE_DYNAMIC,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_match_all('(\\\\d{3})', 'abc123456')"),
				//Same thing here, can't guarantee we're running Java 7 when these are generated.
				new ExampleScript("Named captures (Only works if your system is running Java 7)",
					"reg_match_all('abc(?<foo>\\\\d+)(xyz)', 'abc123xyz')[0]['foo']", "123"),
				new ExampleScript("Named captures with backreferences (Only works if your system is running Java 7)",
					"reg_match_all('abc(?<foo>\\\\d+)def\\\\k<foo>', 'abc123def123')[0]['foo']", "123")
			};
		}

    }

    @api public static class reg_replace extends AbstractFunction implements Optimizable {

		@Override
        public String getName() {
            return "reg_replace";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{3};
        }

		@Override
        public String docs() {
            return "string {pattern, replacement, subject} Replaces any occurances of pattern with the replacement in subject."
                    + " Back references are allowed.";
        }

		@Override
        public ExceptionType[] thrown() {
            return new ExceptionType[]{ExceptionType.FormatException};
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_2_0;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment env, Construct... args) throws ConfigRuntimeException {
            Pattern pattern = getPattern(args[0], t);
            String replacement = args[1].val();
            String subject = args[2].val();
            String ret = "";

            try {
            	ret = pattern.matcher(subject).replaceAll(replacement);
            } catch (IndexOutOfBoundsException e) {
            	throw new Exceptions.FormatException("Expecting a regex group at parameter 1 of reg_replace", t);
            } catch(IllegalArgumentException e){
				throw new Exceptions.FormatException(e.getMessage(), t);
			}

            return new CString(ret, t);
        }

        @Override
        public ParseTree optimizeDynamic(Target t, List<ParseTree> children, FileOptions fileOptions) throws ConfigCompileException, ConfigRuntimeException {
			ParseTree data = children.get(0);
            if(!data.getData().isDynamic()){
				String pattern = data.getData().val();
				if(isLiteralRegex(pattern)){
					//We want to replace this with replace()
					//Note the alternative order of arguments
					ParseTree replace = new ParseTree(new CFunction("replace", t), data.getFileOptions());
					replace.addChildAt(0, children.get(2)); //subject -> main
					replace.addChildAt(1, new ParseTree(new CString(getLiteralRegex(pattern), t), replace.getFileOptions())); //pattern -> what
					replace.addChildAt(2, children.get(1)); //replacement -> that
					return replace;
				} else {
					getPattern(data.getData(), t);
				}
            }
            return null;
//            if(!children.get(0).getData().isDynamic()){
//                getPattern(children.get(0).getData(), t);
//            }
//            return null;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CONSTANT_OFFLINE,
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.OPTIMIZE_DYNAMIC,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_replace('\\\\d', 'Z', '123abc')"),
				new ExampleScript("Using backreferences", "reg_replace('abc(\\\\d+)', '$1', 'abc123'"),
				new ExampleScript("Using backreferences with named captures (Only works if your system is running Java 7)",
					"reg_replace('abc(?<foo>\\\\d+)', '${foo}', 'abc123')", "123")
			};
		}

    }

    @api
	@seealso({StringHandling.split.class, ArrayHandling.array_implode.class})
	public static class reg_split extends AbstractFunction implements Optimizable{

		private final static String split = new StringHandling.split().getName();

		@Override
        public String getName() {
            return "reg_split";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{2, 3};
        }

		@Override
        public String docs() {
            return "array {pattern, subject, [limit]} Splits a string on the given regex, and returns an array of the parts. If"
                    + " nothing matched, an array with one element, namely the original subject, is returned."
					+ " Limit defaults to infinity, but if set, only"
					+ " that number of splits will occur.";
        }

		@Override
        public ExceptionType[] thrown() {
            return new ExceptionType[]{ExceptionType.FormatException, ExceptionType.CastException};
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_2_0;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment env, Construct... args) throws ConfigRuntimeException {
            Pattern pattern = getPattern(args[0], t);
            String subject = args[1].val();
			/**
			 * We use a different indexing notation than Java's regex split. In the case of
			 * 0 for the limit, we will still return an array of length 1, assuming there are actual
			 * splits available. In Java, a split of 0 will return the same as length 1. In our method
			 * though, the limit is the number of splits themselves, so 1 means that the array will be
			 * length 2, as in, there were 1 splits performed. This matches the behavior of split().
			 */
			int limit = Integer.MAX_VALUE - 1;
			if(args.length >= 3){
				limit = Static.getInt32(args[2], t);
			}
            String [] rsplit = pattern.split(subject, limit + 1);
            CArray ret = new CArray(t);
            for(String split : rsplit){
                ret.push(new CString(split, t));
            }
            return ret;
        }

        @Override
        public ParseTree optimizeDynamic(Target t, List<ParseTree> children, FileOptions fileOptions) throws ConfigCompileException, ConfigRuntimeException {
            ParseTree data = children.get(0);
            if(!data.getData().isDynamic()){
				String pattern = data.getData().val();
				if(isLiteralRegex(pattern)){
					//We want to replace this with split()
					ParseTree splitNode = new ParseTree(new CFunction(split, t), data.getFileOptions());
					splitNode.addChildAt(0, new ParseTree(new CString(getLiteralRegex(pattern), t), splitNode.getFileOptions()));
					splitNode.addChildAt(1, children.get(1));
					return splitNode;
				} else {
					getPattern(data.getData(), t);
				}
            }
            return null;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.OPTIMIZE_DYNAMIC,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_split('\\\\d', 'a1b2c3')")
			};
		}

    }

    @api public static class reg_count extends AbstractFunction implements Optimizable {

		@Override
        public String getName() {
            return "reg_count";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{2};
        }

		@Override
        public String docs() {
            return "int {pattern, subject} Counts the number of occurances in the subject.";
        }

		@Override
        public ExceptionType[] thrown() {
            return new ExceptionType[]{ExceptionType.FormatException};
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_2_0;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment env, Construct... args) throws ConfigRuntimeException {
            Pattern pattern = getPattern(args[0], t);
            String subject = args[1].val();
            long ret = 0;
            Matcher m = pattern.matcher(subject);
            while(m.find()){
                ret++;
            }
            return new CInt(ret, t);
        }

        @Override
        public ParseTree optimizeDynamic(Target t, List<ParseTree> children, FileOptions fileOptions) throws ConfigCompileException, ConfigRuntimeException {
            if(!children.get(0).getData().isDynamic()){
                getPattern(children.get(0).getData(), t);
            }
            return null;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CONSTANT_OFFLINE,
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.OPTIMIZE_DYNAMIC,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_count('\\\\d', '123abc')")
			};
		}
    }

    @api
    public static class reg_escape extends AbstractFunction implements Optimizable{

		@Override
        public ExceptionType[] thrown() {
            return null;
        }

		@Override
        public boolean isRestricted() {
            return false;
        }

		@Override
        public Boolean runAsync() {
            return null;
        }

		@Override
        public Construct exec(Target t, Environment environment, Construct... args) throws ConfigRuntimeException {
            return new CString(java.util.regex.Pattern.quote(args[0].val()), t);
        }

		@Override
        public String getName() {
            return "reg_escape";
        }

		@Override
        public Integer[] numArgs() {
            return new Integer[]{1};
        }

		@Override
        public String docs() {
            return "string {arg} Escapes arg so that it may be used directly in a regular expression, without fear that"
                    + " it will have special meaning; that is, it escapes all special characters. Use this if you need"
                    + " to use user input or similar as a literal search index.";
        }

		@Override
        public CHVersion since() {
            return CHVersion.V3_3_1;
        }

		@Override
		public Set<OptimizationOption> optimizationOptions() {
			return EnumSet.of(
						OptimizationOption.CONSTANT_OFFLINE,
						OptimizationOption.CACHE_RETURN,
						OptimizationOption.NO_SIDE_EFFECTS
			);
		}

		@Override
		public ExampleScript[] examples() throws ConfigCompileException {
			return new ExampleScript[]{
				new ExampleScript("Basic usage", "reg_escape('\\\\d+')")
			};
		}

    }

    private static Pattern getPattern(Construct c, Target t) throws ConfigRuntimeException{
        String regex = "";
        int flags = 0;
        String sflags = "";
        if(c instanceof CArray){
            CArray ca = (CArray)c;
            regex = ca.get(0, t).val();
            sflags = ca.get(1, t).val();
            for(int i = 0; i < sflags.length(); i++){
                if(sflags.toLowerCase().charAt(i) == 'i'){
                    flags |= java.util.regex.Pattern.CASE_INSENSITIVE;
                } else if(sflags.toLowerCase().charAt(i) == 'm'){
                    flags |= java.util.regex.Pattern.MULTILINE;
                } else if(sflags.toLowerCase().charAt(i) == 's'){
                    flags |= java.util.regex.Pattern.DOTALL;
                } else {
                    throw new ConfigRuntimeException("Unrecognized flag: " + sflags.toLowerCase().charAt(i), ExceptionType.FormatException, t);
                }
            }
        } else {
            regex = c.val();
        }
        try{
            return Pattern.compile(regex, flags);
        } catch(PatternSyntaxException e){
            throw new ConfigRuntimeException(e.getMessage(), ExceptionType.FormatException, t);
        }
    }

	private static boolean isLiteralRegex(String regex){
		//These are the special characters in a regex. If a regex does not contain any of these
		//characters, we can use a faster method in many cases, though the extra overhead of doing
		//this check only makes sense during optimization, not runtime.

		//We also are going to check for the special case where the whole regex starts with \Q and ends with \E, which
		//indicates that they did something like: reg_split(reg_escape('literal string'), '') which is an easily
		//optimizable case, but we will have to transform the regex to get the actual split index, but that's up
		//to the function to call getLiteralRegex. If the internal of the regex further contains more \Q or \E identifiers,
		//they are doing something more complex, so we're just gonna forgo optimizing that.
		if(regex.startsWith("\\Q") && regex.endsWith("\\E")
				&& !regex.substring(2, regex.length() - 2).contains("\\Q")
				&& !regex.substring(2, regex.length() - 2).contains("\\E")
				){
			return true;
		}
		String chars = "[\\^$.|?*+()";
		for(int i = 0; i < chars.length(); i++){
			if(regex.contains(Character.toString(chars.charAt(i)))){
				return false;
			}
		}
		return true;
	}

	private static String getLiteralRegex(String regex){
		if(regex.startsWith("\\Q") && regex.endsWith("\\E")
				&& !regex.substring(2, regex.length() - 2).contains("\\Q")
				&& !regex.substring(2, regex.length() - 2).contains("\\E")
				){
			return regex.substring(2, regex.length() - 2);
		} else {
			return regex;
		}
	}

	private static final Pattern NAMED_GROUP = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>");
	private static Set<String> getNamedGroups(String regex){
		Matcher m = NAMED_GROUP.matcher(regex);
		Set<String> ret = new HashSet<String>();
		while(m.find()){
			ret.add(m.group(1));
		}
		return ret;
	}

}


text:
//-----------------------------------NOTICE----------------------------------//
// Some of this file is automatically filled in by a Python script.  Only    //
// add custom code in the designated areas or it will be overwritten during  //
// the next update.                                                          //
//-----------------------------------NOTICE----------------------------------//

#ifndef _BHKCOMPRESSEDMESHSHAPE_H_
#define _BHKCOMPRESSEDMESHSHAPE_H_

//--BEGIN FILE HEAD CUSTOM CODE--//
#include "obj/bhkCompressedMeshShapeData.h"
//--END CUSTOM CODE--//

#include "bhkShape.h"

// Include structures
#include "../Ref.h"
namespace Niflib {

// Forward define of referenced NIF objects
class NiAVObject;
class bhkCompressedMeshShapeData;
class bhkCompressedMeshShape;
typedef Ref<bhkCompressedMeshShape> bhkCompressedMeshShapeRef;

/*! Compressed collision mesh. */
class bhkCompressedMeshShape : public bhkShape {
public:
	/*! Constructor */
	NIFLIB_API bhkCompressedMeshShape();

	/*! Destructor */
	NIFLIB_API virtual ~bhkCompressedMeshShape();

	/*!
	 * A constant value which uniquly identifies objects of this type.
	 */
	NIFLIB_API static const Type TYPE;

	/*!
	 * A factory function used during file reading to create an instance of this type of object.
	 * \return A pointer to a newly allocated instance of this type of object.
	 */
	NIFLIB_API static NiObject * Create();

	/*!
	 * Summarizes the information contained in this object in English.
	 * \param[in] verbose Determines whether or not detailed information about large areas of data will be printed out.
	 * \return A string containing a summary of the information within the object in English.  This is the function that Niflyze calls to generate its analysis, so the output is the same.
	 */
	NIFLIB_API virtual string asString( bool verbose = false ) const;

	/*!
	 * Used to determine the type of a particular instance of this object.
	 * \return The type constant for the actual type of the object.
	 */
	NIFLIB_API virtual const Type & GetType() const;

	/***Begin Example Naive Implementation****

	// Points to root node?
	// \return The current value.
	NiAVObject * GetTarget() const;

	// Points to root node?
	// \param[in] value The new value.
	void SetTarget( NiAVObject * value );

	// The shape's material.
	// \return The current value.
	SkyrimHavokMaterial GetSkyrimMaterial() const;

	// The shape's material.
	// \param[in] value The new value.
	void SetSkyrimMaterial( const SkyrimHavokMaterial & value );

	// A shell with that radius is added around the shape.
	// \return The current value.
	float GetRadius() const;

	// A shell with that radius is added around the shape.
	// \param[in] value The new value.
	void SetRadius( float value );

	// Scale
	// \return The current value.
	float GetScale() const;

	// Scale
	// \param[in] value The new value.
	void SetScale( float value );

	// The collision mesh data.
	// \return The current value.
	Ref<bhkCompressedMeshShapeData > GetData() const;

	// The collision mesh data.
	// \param[in] value The new value.
	void SetData( Ref<bhkCompressedMeshShapeData > value );

	****End Example Naive Implementation***/

	//--BEGIN MISC CUSTOM CODE--//

	// Points to root node?
	// \return The current value.
	NiAVObject * GetTarget() const;

	// Points to root node?
	// \param[in] value The new value.
	void SetTarget(NiAVObject * value);

	// The shape's material.
	// \return The current value.
	SkyrimHavokMaterial GetSkyrimMaterial() const;

	// The shape's material.
	// \param[in] value The new value.
	void SetSkyrimMaterial(SkyrimHavokMaterial value);

	// A shell with that radius is added around the shape.
	// \return The current value.
	float GetRadius() const;

	// A shell with that radius is added around the shape.
	// \param[in] value The new value.
	void SetRadius(float value);

	// Scale
	// \return The current value.
	float GetScale() const;

	// Scale
	// \param[in] value The new value.
	void SetScale(float value);

	// The collision mesh data.
	// \return The current value.
	bhkCompressedMeshShapeDataRef GetData() const;

	// The collision mesh data.
	// \param[in] value The new value.
	void SetData(bhkCompressedMeshShapeDataRef value);

	//--END CUSTOM CODE--//
protected:
	/*! Points to root node? */
	NiAVObject * target;
	/*! The shape's material. */
	SkyrimHavokMaterial skyrimMaterial;
	/*! Possibly radius */
	float unknownFloat1;
	/*! Unknown. */
	int unknownInt1;
	/*! Unknown */
	Vector4 unknownFloats1;
	/*! A shell with that radius is added around the shape. */
	float radius;
	/*! Scale */
	float scale;
	/*! Unknown */
	float unknownFloat3;
	/*! Unknown */
	float unknownFloat4;
	/*! Unknown */
	float unknownFloat5;
	/*! The collision mesh data. */
	Ref<bhkCompressedMeshShapeData > data;
public:
	/*! NIFLIB_HIDDEN function.  For internal use only. */
	NIFLIB_HIDDEN virtual void Read( istream& in, list<unsigned int> & link_stack, const NifInfo & info );
	/*! NIFLIB_HIDDEN function.  For internal use only. */
	NIFLIB_HIDDEN virtual void Write( ostream& out, const map<NiObjectRef,unsigned int> & link_map, list<NiObject *> & missing_link_stack, const NifInfo & info ) const;
	/*! NIFLIB_HIDDEN function.  For internal use only. */
	NIFLIB_HIDDEN virtual void FixLinks( const map<unsigned int,NiObjectRef> & objects, list<unsigned int> & link_stack, list<NiObjectRef> & missing_link_stack, const NifInfo & info );
	/*! NIFLIB_HIDDEN function.  For internal use only. */
	NIFLIB_HIDDEN virtual list<NiObjectRef> GetRefs() const;
	/*! NIFLIB_HIDDEN function.  For internal use only. */
	NIFLIB_HIDDEN virtual list<NiObject *> GetPtrs() const;
};

//--BEGIN FILE FOOT CUSTOM CODE--//

//--END CUSTOM CODE--//

} //End Niflib namespace
#endif


text:
@interface HBCellView : UITableViewCell {
  UITextField *textField;
  UILabel *label;

  HBDocumentDetailViewController *docController;
}

@property (nonatomic, retain) IBOutlet UITextField *textField;
@property (nonatomic, retain) IBOutlet UILabel *label;
@property (nonatomic, retain) HBDocumentDetailViewController *docController;

@end


text:
<?php

/*
 *
 *
 *
 */

namespace pfp\components {
    require 'UserException.php';
    require 'UserStatus.php';

    /**
     * Description of User
     *
     * REQUIRES:
     *      HttpSessionInterface
     *
     * GROUP: System
     *
     * @author Zeakis George <zeageorge@gmail.com>
     */
    class User extends \pfp\base\UserInterface {

        /**
         *
         * @param array $config
         */
        public function __construct(array $config = [])
        {
            $this->properties['state_storage'] = NULL;
            $this->properties['state_storage_key_prefix'] = '';
            $this->properties['logout_to'] = '';

            parent::__construct($config);
        }

        /**
         *
         * @return string
         */
        protected function public__get($key, $defaultValue = NULL)
        {
            return $this->getStateStorage()->getData($this->getStateStorageKeyPrefix() . $key, $defaultValue);
        }

        /**
         *
         * @return \pfp\components\HttpSession
         */
        protected function public__getStateStorage()
        {
            return $this->properties['state_storage'];
        }

        /**
         *
         * @return string
         */
        protected function public__getStateStorageKeyPrefix()
        {
            return $this->properties['state_storage_key_prefix'];
        }

        /**
         * Get the URL to redirect to after logout
         *
         * @return string
         */
        protected function public__getLogoutTo()
        {
            return $this->properties['logout_to'];
        }

        /**
         *
         * @return string|int
         */
        protected function public__getId()
        {
            return $this->getStateStorage()->getData($this->getStateStorageKeyPrefix() . 'user_id', -1);
        }

        /**
         *
         * @return string
         */
        protected function public__getUsername()
        {
            return $this->getStateStorage()->getData($this->getStateStorageKeyPrefix() . 'username', 'guest');
        }

        /**
         * You Must call User->Save() to make persistent
         *
         * @param string|array $key
         * @param string|array|int|boolean|float|NULL $value
         * @return \pfp\components\User
         */
        protected function public__set($key, $value = NULL)
        {
            $ss = $this->getStateStorage();
            $sskp = $this->getStateStorageKeyPrefix();
            if (\is_string($key)) {
                $ss->setData($sskp . $key, $value);
            } elseif (\is_array($key)) {
                foreach ($key as $k => $v) {
                    $ss->setData($sskp . $k, $v);
                }
            }
            return $this;
        }

        /**
         * Removes a key-value from state storage (session storage)
         * You Must call User->Save() to make persistent
         *
         * @param string $key
         * @return mix
         */
        protected function public__remove($key)
        {
            return $this->getStateStorage()->removeData($this->getStateStorageKeyPrefix() . $key);
        }

        /**
         *
         * @param \pfp\base\HttpSessionInterface $stateStorage
         * @return \pfp\components\User
         */
        protected function public__setStateStorage(\pfp\base\HttpSessionInterface $stateStorage)
        {
            $this->properties['state_storage'] = $stateStorage;
            return $this;
        }

        /**
         *
         * @param string $keyPrefix
         * @return \pfp\components\User
         */
        protected function public__setStateStorageKeyPrefix($keyPrefix)
        {
            $this->properties['state_storage_key_prefix'] = $keyPrefix;
            return $this;
        }

        /**
         *
         * @param string $url
         * @return \pfp\components\User
         */
        protected function public__setLogoutTo($url)
        {
            $this->properties['logout_to'] = $url;
            return $this;
        }

        /**
         *
         * @return boolean|int integer number on success or FALSE on failure.
         */
        protected function public__save()
        {
            return $this->getStateStorage()->save();
        }

        /**
         *
         * @return boolean TRUE if user is guest or FALSE if user is not guest.
         */
        protected function public__isGuest()
        {
            $id = $this->getStateStorage()->getData($this->getStateStorageKeyPrefix() . 'user_id');
            if ($id != NULL) {
                return $id < 1;
            }
            return TRUE;
        }

        /**
         *
         */
        protected function public__login()
        {
            // @todo:
            // regenerate session_id
            // change session state data: user_id, username
            //
            //

        }

        /**
         *
         * @param boolean $redirect Redirect or not
         * @param string $url The URL to redirect to if $redirect == TRUE
         */
        protected function public__logout($redirect = TRUE, $url = NULL)
        {
            // todo:
            // destroy/delete all session data
            // regenerate session_id
            // change session state data: user_id, username
            //
            //

            if ($redirect == TRUE) {
                if ($url == NULL) {
                    // redirect to $this->getLogoutTo()
                } else {
                    // redirect to $url
                }
            } else {
                // ??
            }
        }

    }

}


text:
<?php

// TODO: Document!
class Livesite_Public
  implements Prack_I_MiddlewareApp
{
	// TODO: Document!
	public function call( &$env )
	{
		ob_start();
		  include( join( DIRECTORY_SEPARATOR, array( $env[ 'livesite.templates' ], 'public.html.php' ) ) );
		$output = ob_get_clean();

		return array( 200, array(), $output );
	}
}


text:
title: Resume improvement case study
slug: resume-improvement-case-study
author: Yangshun Tay
author_title: Engineering Lead at Facebook
author_url: https://github.com/yangshun
author_image_url: https://github.com/yangshun.png
tags: [career, resume]
hide_table_of_contents: true
---

import TeikjunResumeOldURL from '@site/static/img/teik-jun-resume-old.png'; import TeikjunResumeNewURL from '@site/static/img/teik-jun-resume-new.png';

As an example of putting the Resume tips to practice, let's reviewing [Teik Jun's](https://github.com/teikjun) resume, critiquing it and improving it. At the time of writing, Teik Jun is a Year 4 undergraduate at National University of Singapore and is looking for a Software Engineering (SWE) internship for the summer of 2022.

Here's the initial version of Teik Jun's resume I received from him.

Teik Jun's initial resume

On first glance, it looks like a solid resume, and it is. However, I've known Teik Jun for a few years now since the [Major League Hacking (MLH) Fellowship program](https://fellowship.mlh.io/) where he [contributed a ton of features to Docusaurus v2](https://github.com/facebook/docusaurus/commits?author=teikjun), and know that he has done even greater things in the past. This version of his resume doesn't do his skills justice. Let's see how we can improve it by applying the points learnt in the previous section.

## Resume critique

### Good

- Demonstrates diverse and full stack experience - Has experience in front end, building web applications, JavaScript tooling, payments infrastructure and blockchain
- Diverse interests and abilities - Open Source, AI and Machine Learning, Front End, Algorithms TA, Hackathons
- Clean formatting which is easily readable
- Reverse chronological order
- Straightforward, bullet point format

### Points of improvement

- **Irrelevant experience** - It's honestly super cool to be "Best Shot" of Sniper Course and to get other impressive accolades as an NSF, but it's hardly relevant to a SWE job. If I were running out of space I'd remove it.
- **Not sufficiently conveying the impressiveness of his experience** - Teik Jun's work on the MLH program was for a popular Facebook Open Source project, Docusaurus (the same project that powers this website) and he did very very good work as part of the program. I know because I reviewed his Pull Requests and he did a damn good job. Docusaurus is an important project to Facebook and widely used by popular Facebook projects like React Native, Jest, and Relay.
- **Not explaining what the project is about** - In most cases, recruiters are the ones reading a resume and they would not know what a project is about, especially when most of them aren't technical. Hence resumes also have to explain the complexity and impact of a project. I know what Docusaurus and TEAMMATES are, but that's because I worked on Docusaurus and I am aware of NUS' projects. Most recruiters would not, so we have to explain what they are. TEAMMATES was also a Google Summer of Code project, so mentioning that would be nice.
- **Trivial details** - Under TEAMMATES, there was a point "Added a progress indicator component for expensive operations". This sounds quite small and can be removed if there's a lack of space.
- **Obscure projects** - The "Mentorship Management Web Application" doesn't sufficiently relay the complexity. The details are also obvious and doesn't sound impressive. I'd remove this project because it doesn't add much value.
- **Contributions to Open Source projects** - Any project can be made open source and contributing to your own open source homework doesn't mean much. What makes open source contributions impressive would be the complexity of the project and the contributions made. The latter might be hard to fit into a resume, so we should do the former at the very least.

### Bonus

I know that Teik Jun will be interning with ByteDance in the fall, so it could be added as a signal of Teik Jun's abilities, telling recruiters that he's good enough to work at ByteDance.

## Improved version

Here's an improved version of Teik Jun's resume which I feel which better highlights Teik Jun's abilities. Main changes made were:

- Added more important keywords - ByteDance (future internship), Facebook (MLH collaboration), Google (TEAMMATES was a Google Summer of Code project for a few years).
- Removed redundant and unimportant details.
- Added links to certain projects to make it more convenient for recruiters who want to gain more context of the projects listed.

In my opinion, this improved resume is sure to capture the eyes of any recruiter in the world!

Teik Jun's improved resume


text:
package polynomial;

import java.io.File;

public class Main {
	public static void main(String[] args) {

		InputOutput read = new InputOutput();
		read.readAndWrite("D:\\Eclipse Workspace\\Assignment 2\\src\\polynomial\\pol.txt");

		try {
			File file = new File("D:\\Eclipse Workspace\\Assignment 2\\src\\polynomial\\output.txt");
			if (file.createNewFile()) {
				System.out.println("File created.");
			} else {
				System.out.println("The file exists already.");
			}
		} catch (Exception e) {
			System.out.println("Failed.");
		}
	}
}