text:
/* SPDX-License-Identifier: BSD-2-Clause */
/*******************************************************************************
 * Copyright 2018-2019, Fraunhofer SIT sponsored by Infineon Technologies AG
 * All rights reserved.
 *******************************************************************************/

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include "tss2_common.h"

#include "ifapi_profiles.h"

#include "tpm_json_deserialize.h"
#include "ifapi_policy_json_deserialize.h"
#include "ifapi_json_deserialize.h"
#include "ifapi_helpers.h"
#define LOGMODULE fapi
#include "util/log.h"
#include "util/aux_util.h"
#include "ifapi_macros.h"

#define PROFILES_EXTENSION ".json"
#define PROFILES_PREFIX "P_"

static TSS2_RC
ifapi_profile_json_deserialize(
    json_object *jso,
    IFAPI_PROFILE *profile);

static TSS2_RC
ifapi_profile_checkpcrs(const TPML_PCR_SELECTION *pcr_profile);

/** Initialize the profiles information in the context in an asynchronous way
 *
 * Load the profile information from disk, fill the dictionary of loaded profiles and fill
 * the default profile information into the context.
 *
 * Call ifapi_profiles_initialize_finish to complete the operation.
 *
 * @param[in,out] profiles The context for the profiles information.
 * @param[in,out] io The input/output context being used for file I/O.
 * @param[in] profilesdir The directory to load profile information from.
 * @param[in] defaultprofile The name of the default profile to use.
 * @retval TSS2_RC_SUCCESS on success.
 * @retval TSS2_FAPI_RC_BAD_REFERENCE if NULL pointers were passed in.
 * @retval TSS2_FAPI_RC_BAD_VALUE if the profilesdir does not exist or is empty.
 * @retval TSS2_FAPI_RC_IO_ERROR if creation of log_dir failed or log_dir is not writable.
 * @retval TSS2_FAPI_RC_MEMORY if memory allocation failed.
 */
TSS2_RC
ifapi_profiles_initialize_async(
    IFAPI_PROFILES *profiles,
    IFAPI_IO *io,
    const char *profilesdir,
    const char *defaultprofile)
{
    TSS2_RC r;
    char *tmp;
    size_t i, j;
    check_not_null(profiles);
    check_not_null(profilesdir);

    memset(profiles, 0, sizeof(*profiles));

    profiles->default_name = strdup(defaultprofile);
    check_oom(profiles->default_name);

    r = ifapi_io_dirfiles(profilesdir, &profiles->filenames, &profiles->num_profiles);
    return_if_error(r, "Reading profiles from profiles dir");

    profiles->profiles = calloc(profiles->num_profiles, sizeof(profiles->profiles[0]));
    check_oom(profiles->profiles);

    /* Clean up list of files to only include those that match our filename pattern
       Expand the filenames with the directory
       Set the names in the dictionary */
    for (i = 0; i < profiles->num_profiles; ) {
        char *ext = strstr(profiles->filenames[i], PROFILES_EXTENSION);
        /* Path the filename with the expected pattern */
        if (ext != NULL && strlen(ext) == strlen(PROFILES_EXTENSION) &&
                strncmp(profiles->filenames[i], PROFILES_PREFIX, strlen(PROFILES_PREFIX)) == 0)
        {
            LOG_TRACE("Using file %s in profiles directory", profiles->filenames[i]);
            /* Add the profile name */
            profiles->profiles[i].name = strndup(profiles->filenames[i],
                                                 ext - profiles->filenames[i]);
            check_oom(profiles->profiles[i].name);
            /* Expand the filename with the directory */
            tmp = profiles->filenames[i];
            r = ifapi_asprintf(&profiles->filenames[i], "%s/%s", profilesdir, tmp);
            return_if_error(r, "Out of memory");

            LOG_TRACE("Added profile-entry %s for file %s based on direntry %s",
                      profiles->profiles[i].name, profiles->filenames[i], tmp);

            /* Cleanup and continue */
            free(tmp);
            i++;
        } else {
            LOG_TRACE("Skipping file %s in profiles directory", profiles->filenames[i]);
            free(profiles->filenames[i]);
            profiles->num_profiles -= 1;
            for (j = i; j < profiles->num_profiles; j++) {
                profiles->filenames[j] = profiles->filenames[j + 1];
            }
        }
    }

    if (profiles->num_profiles == 0) {
        LOG_ERROR("No files found in profile dir %s that match the pattern %s*%s",
                  profilesdir, PROFILES_PREFIX, PROFILES_EXTENSION);
        return TSS2_FAPI_RC_BAD_VALUE;
    }
#ifdef HAVE_REALLOCARRAY
    profiles->profiles = reallocarray(profiles->profiles, profiles->num_profiles,
                                      sizeof(profiles->profiles[0]));
    profiles->filenames = reallocarray(profiles->filenames, profiles->num_profiles,
                                      sizeof(profiles->filenames[0]));
#else /* HAVE_REALLOCARRAY */
    profiles->profiles = realloc(profiles->profiles, profiles->num_profiles *
                                      sizeof(profiles->profiles[0]));
    profiles->filenames = realloc(profiles->filenames, profiles->num_profiles *
                                      sizeof(profiles->filenames[0]));
#endif /* HAVE_REALLOCARRAY */
    /* No need for OOM checks, since num_profiles may only have become smaller */

    r = ifapi_io_read_async(io, profiles->filenames[profiles->profiles_idx]);
    return_if_error2(r, "Reading profile %s", profiles->filenames[profiles->profiles_idx]);

    return TSS2_RC_SUCCESS;
}

/** Initialize the profiles information in the context in an asynchronous way
 *
 * Call after ifapi_profiles_initialize_async to complete the operation.
 *
 * @param[in,out] profiles The context for the profiles information.
 * @param[in,out] io The input/output context being used for file I/O.
 * @retval TSS2_RC_SUCCESS on success.
 * @retval TSS2_FAPI_RC_BAD_REFERENCE if NULL pointers were passed in.
 * @retval TSS2_FAPI_RC_BAD_VALUE if a profile could not be loaded.
 * @retval TSS2_FAPI_RC_IO_ERROR if creation of log_dir failed or log_dir is not writable.
 * @retval TSS2_FAPI_RC_MEMORY if memory allocation failed.
 * @retval TSS2_FAPI_RC_TRY_AGAIN if the I/O operation is not finished yet and this function needs
 *         to be called again.
 */
TSS2_RC
ifapi_profiles_initialize_finish(
    IFAPI_PROFILES *profiles,
    IFAPI_IO *io)
{
    TSS2_RC r;
    uint8_t *buffer;
    size_t i;
    json_object *jso;
    check_not_null(profiles);
    check_not_null(io);

    r = ifapi_io_read_finish(io, &buffer, NULL);
    return_if_error(r, "Reading profile failed");

    jso = ifapi_parse_json((char *) buffer);
    free(buffer);
    if (jso == NULL) {
        LOG_ERROR("Failed to parse profile %s", profiles->filenames[profiles->profiles_idx]);
        r = TSS2_FAPI_RC_BAD_VALUE;
        goto error;
    }

    r = ifapi_profile_json_deserialize(jso,
            &profiles->profiles[profiles->profiles_idx].profile);
    json_object_put(jso);
    goto_if_error2(r, "Parsing profile %s failed", error,
                   profiles->filenames[profiles->profiles_idx]);

    r = ifapi_profile_checkpcrs(&profiles->profiles[profiles->profiles_idx].profile.pcr_selection);
    goto_if_error2(r, "Malformed profile pcr selection for profile %s", error,
                   profiles->filenames[profiles->profiles_idx]);

    profiles->profiles_idx += 1;

    if (profiles->profiles_idx < profiles->num_profiles) {
        r = ifapi_io_read_async(io, profiles->filenames[profiles->profiles_idx]);
        goto_if_error2(r, "Reading profile %s", error,
                       profiles->filenames[profiles->profiles_idx]);

        return TSS2_FAPI_RC_TRY_AGAIN;
    }

    /* Get the data of the default profile into the respective variable */
    for (i = 0; i < profiles->num_profiles; i++) {
        if (strcmp(profiles->default_name, profiles->profiles[i].name) == 0) {
            profiles->default_profile = profiles->profiles[i].profile;
            break;
        }
    }
    if (i == profiles->num_profiles) {
        LOG_ERROR("Default profile %s not in the list of loaded profiles",
                  profiles->default_name);
        r = TSS2_FAPI_RC_BAD_VALUE;
        goto error;
    }

    for (i = 0; i < profiles->num_profiles; i++) {
        free(profiles->filenames[i]);
    }
    SAFE_FREE(profiles->filenames);

    return TSS2_RC_SUCCESS;

 error:
    for (i = 0; i < profiles->num_profiles; i++) {
        SAFE_FREE(profiles->filenames[i]);
    }
    SAFE_FREE(profiles->filenames);
    ifapi_profiles_finalize(profiles);
    return r;
}

/** Return the profile data for a given profile name.
 *
 * Returns a (const, not to be free'd) pointer to the profile data for a requested profile.
 * If a NULL profile is requesten, then the default profile is returned.
 * If a keypath is passed in, then the prefix is analysed. If that keypath starts with a profile
 * then this profile is returned. Otherwise the default profile is returned.
 *
 * @param[in] profiles The profiles context
 * @param[in] name The name of the profile or the keypath
 * @param[out] profile The pointer to the profile data.
 * @retval TSS2_RC_SUCCESS on success.
 * @retval TSS2_FAPI_RC_BAD_REFERENCE if NULL pointers were passed in.
 * @retval TSS2_FAPI_RC_BAD_VALUE if a profile is not found.
 */
TSS2_RC
ifapi_profiles_get(
    const IFAPI_PROFILES *profiles,
    const char *name,
    const IFAPI_PROFILE **profile)
{
    check_not_null(profiles);
    check_not_null(name);
    check_not_null(profile);
    char *split;
    size_t len;

    /* if no name or nor profile prefix is given, use the default profile */
    if (!name || strncmp(name, "P_", 2) != 0 || strncmp(name, "/P_", 2) != 0) {
        *profile = &profiles->default_profile;
        return TSS2_RC_SUCCESS;
    }

    /* Search for path delimiter */
    split = index(name, IFAPI_FILE_DELIM_CHAR);

    /* If the path beging with delimiters, skip over those */
    if (name == split) {
        name += 1;
        split = index(name, IFAPI_FILE_DELIM_CHAR);
    }
    if (split == NULL)
        len = strlen(name);
    else
        len = split - name;

    for (size_t i = 0; i < profiles->num_profiles; i++) {
        if (len == strlen(profiles->profiles[i].name) &&
                strncmp(name, profiles->profiles[i].name, len) == 0) {
            *profile = &profiles->profiles[i].profile;
            return TSS2_RC_SUCCESS;
        }
    }
    LOG_ERROR("Profile %s not in the list of loaded profiles", name);
    return TSS2_FAPI_RC_BAD_VALUE;
}

/** Sanitizes and frees internal data structures of loaded profiles' information.
 *
 * @param[in,out] profiles The context for the profiles information.
 */
void
ifapi_profiles_finalize(
    IFAPI_PROFILES *profiles)
{
    size_t i;
    if (!profiles) {
        LOG_ERROR("Called with bad reference");
        return;
    }

    SAFE_FREE(profiles->default_name);

    for (i = 0; i < profiles->num_profiles; i++) {
        IFAPI_PROFILE_ENTRY * entry = &profiles->profiles[i];
        SAFE_FREE(profiles->profiles[i].name);

        IFAPI_PROFILE * profile = &entry->profile;

        SAFE_FREE(profile->srk_template);
        SAFE_FREE(profile->ek_template);

        SAFE_FREE(profile->srk_description);
        SAFE_FREE(profile->ek_description);

        ifapi_cleanup_policy(profile->eh_policy);
        SAFE_FREE(profile->eh_policy);

        ifapi_cleanup_policy(profile->ek_policy);
        SAFE_FREE(profile->ek_policy);

        ifapi_cleanup_policy(profile->sh_policy);
        SAFE_FREE(profile->sh_policy);
    }
    SAFE_FREE(profiles->profiles);

    memset(profiles, 0, sizeof(*profiles));
}

/** Deserialize a IFAPI_KEY_PROFILE json object.
 *
 * @param[in]  jso the json object to be deserialized.
 * @param[out] out the deserialzed binary object.
 * @retval TSS2_RC_SUCCESS if the function call was a success.
 * @retval TSS2_FAPI_RC_BAD_VALUE if the json object can't be deserialized.
 * @retval TSS2_FAPI_RC_BAD_REFERENCE a invalid null pointer is passed.
 * @retval TSS2_FAPI_RC_IO_ERROR if an error occurred while accessing the
 *         object store.
 * @retval TSS2_FAPI_RC_MEMORY if not enough memory can be allocated.
 */
static TSS2_RC
ifapi_profile_json_deserialize(
    json_object *jso,
    IFAPI_PROFILE *out)
{
    json_object *jso2;
    TSS2_RC r;

    const TPMT_SYM_DEF session_symmetric_default = {
        .algorithm = TPM2_ALG_AES,
        .keyBits = {.aes = 128},
        .mode = {.aes = TPM2_ALG_CFB}
    };

    LOG_TRACE("call");
    memset(out, 0, sizeof(IFAPI_PROFILE));
    return_if_null(out, "Bad reference.", TSS2_FAPI_RC_BAD_REFERENCE);

    if (!ifapi_get_sub_object(jso, "type", &jso2)) {
        LOG_ERROR("Field \"type\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_TPMI_ALG_PUBLIC_deserialize(jso2, &out->type);
    return_if_error(r, "Bad value for field \"type\".");

    if (!ifapi_get_sub_object(jso, "srk_template", &jso2)) {
        LOG_ERROR("Field \"srk_template\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    out->srk_template = strdup(json_object_get_string(jso2));
    return_if_null(out->srk_template, "Out of memory.", TSS2_FAPI_RC_MEMORY);

    if (!ifapi_get_sub_object(jso, "srk_description", &jso2)) {
        out->srk_description = NULL;
    } else {
        out->srk_description = strdup(json_object_get_string(jso2));
        return_if_null(out->srk_description, "Out of memory.", TSS2_FAPI_RC_MEMORY);
    }

    if (!ifapi_get_sub_object(jso, "ek_template", &jso2)) {
        LOG_ERROR("Field \"ek_template\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    out->ek_template = strdup(json_object_get_string(jso2));
    return_if_null(out->ek_template, "Out of memory.", TSS2_FAPI_RC_MEMORY);

    if (!ifapi_get_sub_object(jso, "ek_description", &jso2)) {
        out->ek_description = NULL;
    } else {
        out->ek_description = strdup(json_object_get_string(jso2));
        return_if_null(out->ek_description, "Out of memory.", TSS2_FAPI_RC_MEMORY);
    }

    if (!ifapi_get_sub_object(jso, "ecc_signing_scheme", &jso2)) {
        memset(&out->ecc_signing_scheme, 0, sizeof(TPMT_SIG_SCHEME));
    } else {
        r = ifapi_json_TPMT_SIG_SCHEME_deserialize(jso2, &out->ecc_signing_scheme);
        return_if_error(r, "Bad value for field \"ecc_signing_scheme\".");
    }

    if (!ifapi_get_sub_object(jso, "rsa_signing_scheme", &jso2)) {
        memset(&out->rsa_signing_scheme, 0, sizeof(TPMT_SIG_SCHEME));
    } else {
        r = ifapi_json_TPMT_SIG_SCHEME_deserialize(jso2, &out->rsa_signing_scheme);
        return_if_error(r, "Bad value for field \"rsa_signing_scheme\".");
    }

    if (!ifapi_get_sub_object(jso, "rsa_decrypt_scheme", &jso2)) {
        memset(&out->rsa_decrypt_scheme, 0, sizeof(TPMT_RSA_DECRYPT));
    } else {
        r = ifapi_json_TPMT_RSA_DECRYPT_deserialize(jso2, &out->rsa_decrypt_scheme);
        return_if_error(r, "Bad value for field \"rsa_decrypt_scheme\".");
    }

    if (!ifapi_get_sub_object(jso, "sym_mode", &jso2)) {
        LOG_ERROR("Field \"sym_mode\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_TPMI_ALG_CIPHER_MODE_deserialize(jso2, &out->sym_mode);
    return_if_error(r, "Bad value for field \"sym_mode\".");

    if (!ifapi_get_sub_object(jso, "sym_parameters", &jso2)) {
        LOG_ERROR("Field \"sym_parameters\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_TPMT_SYM_DEF_OBJECT_deserialize(jso2, &out->sym_parameters);
    return_if_error(r, "Bad value for field \"sym_parameters\".");

    if (!ifapi_get_sub_object(jso, "sym_block_size", &jso2)) {
        LOG_ERROR("Field \"sym_block_size\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_UINT16_deserialize(jso2, &out->sym_block_size);
    return_if_error(r, "Bad value for field \"sym_block_size\".");

    if (!ifapi_get_sub_object(jso, "pcr_selection", &jso2)) {
        LOG_ERROR("Field \"pcr_selection\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_TPML_PCR_SELECTION_deserialize(jso2, &out->pcr_selection);
    return_if_error(r, "Bad value for field \"pcr_selection\".");

    if (!ifapi_get_sub_object(jso, "nameAlg", &jso2)) {
        LOG_ERROR("Field \"nameAlg\" not found.");
        return TSS2_FAPI_RC_BAD_VALUE;
    }
    r = ifapi_json_TPMI_ALG_HASH_deserialize(jso2, &out->nameAlg);
    return_if_error(r, "Bad value for field \"nameAlg\".");

    if (out->type == TPM2_ALG_RSA) {
        if (!ifapi_get_sub_object(jso, "exponent", &jso2)) {
            LOG_ERROR("Field \"exponent\" not found.");
            return TSS2_FAPI_RC_BAD_VALUE;
        }
        r = ifapi_json_UINT32_deserialize(jso2, &out->exponent);
        return_if_error(r, "Bad value for field \"exponent\".");
        if (!ifapi_get_sub_object(jso, "keyBits", &jso2)) {
            LOG_ERROR("Field \"keyBits\" not found.");
            return TSS2_FAPI_RC_BAD_VALUE;

        }
        r = ifapi_json_TPMI_RSA_KEY_BITS_deserialize(jso2, &out->keyBits);
        return_if_error(r, "Bad value for field \"keyBits\".");

    } else if (out->type == TPM2_ALG_ECC) {
        if (!ifapi_get_sub_object(jso, "curveID", &jso2)) {
            LOG_ERROR("Field \"curveID\" not found.");
            return TSS2_FAPI_RC_BAD_VALUE;
        }
        r = ifapi_json_TPMI_ECC_CURVE_deserialize(jso2, &out->curveID);
        return_if_error(r, "Bad value for field \"curveID\".");
    }

    if (!ifapi_get_sub_object(jso, "session_symmetric", &jso2)) {
        out->session_symmetric = session_symmetric_default;
    } else {
        r = ifapi_json_TPMT_SYM_DEF_deserialize(jso2, &out->session_symmetric);
        return_if_error(r, "Bad value for field \"session_symmetric\".");
    }

    if (ifapi_get_sub_object(jso, "eh_policy", &jso2)) {
        out->eh_policy = calloc(1, sizeof(TPMS_POLICY));
        goto_if_null2(out->eh_policy, "Out of memory.", r, TSS2_FAPI_RC_MEMORY,
                      cleanup);

        r = ifapi_json_TPMS_POLICY_deserialize(jso2, out->eh_policy);
        goto_if_error(r, "Deserialize policy.", cleanup);
    }

    if (ifapi_get_sub_object(jso, "sh_policy", &jso2)) {
        out->sh_policy = calloc(1, sizeof(TPMS_POLICY));
        goto_if_null2(out->sh_policy, "Out of memory.", r, TSS2_FAPI_RC_MEMORY,
                      cleanup);

        r = ifapi_json_TPMS_POLICY_deserialize(jso2, out->sh_policy);
        goto_if_error(r, "Deserialize policy.", cleanup);
    }

    if (ifapi_get_sub_object(jso, "ek_policy", &jso2)) {
        out->ek_policy = calloc(1, sizeof(TPMS_POLICY));
        goto_if_null2(out->ek_policy, "Out of memory.", r, TSS2_FAPI_RC_MEMORY,
                      cleanup);

        r = ifapi_json_TPMS_POLICY_deserialize(jso2, out->ek_policy);
        goto_if_error(r, "Deserialize policy.", cleanup);
    }

    if (ifapi_get_sub_object(jso, "srk_policy", &jso2)) {
        out->srk_policy = calloc(1, sizeof(TPMS_POLICY));
        goto_if_null2(out->srk_policy, "Out of memory.", r, TSS2_FAPI_RC_MEMORY,
                      cleanup);

        r = ifapi_json_TPMS_POLICY_deserialize(jso2, out->srk_policy);
        goto_if_error(r, "Deserialize policy.", cleanup);
    }

    if (ifapi_get_sub_object(jso, "lockout_policy", &jso2)) {
        out->lockout_policy = calloc(1, sizeof(TPMS_POLICY));
        goto_if_null2(out->lockout_policy, "Out of memory.", r, TSS2_FAPI_RC_MEMORY,
                      cleanup);

        r = ifapi_json_TPMS_POLICY_deserialize(jso2, out->lockout_policy);
        goto_if_error(r, "Deserialize policy.", cleanup);
    }

    if (!ifapi_get_sub_object(jso, "newMaxTries", &jso2)) {
        out->newMaxTries = 5;
    } else {
        r = ifapi_json_UINT32_deserialize(jso2, &out->newMaxTries);
        return_if_error(r, "Bad value for field \"newMaxTries\".");
    }

    if (!ifapi_get_sub_object(jso, "newRecoveryTime", &jso2)) {
        out->newRecoveryTime = 1000;
    } else {
        r = ifapi_json_UINT32_deserialize(jso2, &out->newRecoveryTime);
        return_if_error(r, "Bad value for field \"newRecoveryTime\".");
    }

    if (!ifapi_get_sub_object(jso, "lockoutRecovery", &jso2)) {
        out->lockoutRecovery = 1000;
    } else {
        r = ifapi_json_UINT32_deserialize(jso2, &out->lockoutRecovery);
        return_if_error(r, "Bad value for field \"lockoutRecovery\".");
    }

    LOG_TRACE("true");
    return TSS2_RC_SUCCESS;

cleanup:
    SAFE_FREE(out->eh_policy);
    return r;
}

/**
  * Check whether PCRs with muliple banks are defined in profile.
  *
  * This case is not allowed by FAPI.
  */
static TSS2_RC
ifapi_profile_checkpcrs(const TPML_PCR_SELECTION *pcr_profile)
{
    size_t i, j, byte_idx;

    for (i = 0; i < pcr_profile->count - 1; i++) {
        for (j = i + 1; j <  pcr_profile->count; j++) {
            for (byte_idx = 0; byte_idx < 3; byte_idx++) {
                /* Check whether a PCR register flag does occur in two different banks. */
                if (pcr_profile->pcrSelections[i].pcrSelect[byte_idx] &
                        pcr_profile->pcrSelections[j].pcrSelect[byte_idx]) {
                    return_error2(TSS2_FAPI_RC_BAD_VALUE,
                                  "More than one bank selected for a PCR register.");
                }
            }
        }
    }
    return TSS2_RC_SUCCESS;
}


text:
from __future__ import absolute_import

from django.conf.urls import patterns, url

from .views import (FeedList, Search, EntryList,
                    NumbersUnread, MarkEntryRead,
                    EditFeeds, ExportOpml, UpdateItem)

urlpatterns = patterns('',
                       url(regex=r'^$',
                           view=FeedList.as_view(),
                           name='feed_list'),
                       url(regex=r'^entry_list/$',
                           view=EntryList.as_view(),
                           name='entry_list'),
                       url(regex=r'^num_unread/$',
                           view=NumbersUnread.as_view(),
                           name='num_unread'),
                       url(regex=r'^mark_entry_read/$',
                           view=MarkEntryRead.as_view(),
                           name='mark_entry_read'),
                       url(regex=r'^search/$',
                           view=Search.as_view(),
                           name='search'),
                       url(regex=r'^edit_feeds/$',
                           view=EditFeeds.as_view(),
                           name='edit_feeds'),
                       url(regex=r'^export_opml/$',
                           view=ExportOpml.as_view(),
                           name='export_opml'),
                       url(regex=r'^update/$',
                           view=UpdateItem.as_view(),
                           name='update_item'),
)

redpajama_set_name: RedPajamaGithub


text:
@implementation ObjLocation
@synthesize idx,lat,log,strName,strColorHex,strDescription,strServerId;
@end


text:
import message from './message';
import user from './user';

import mongoose from 'mongoose';

export default function() {
  const app = this;

  mongoose.connect(app.get('mongodb'));
  mongoose.Promise = global.Promise;

  app.configure(message);
  app.configure(user);
}


text:
package org.apereo.cas.support.saml.mdui;

import lombok.extern.slf4j.Slf4j;
import org.apereo.cas.services.RegisteredService;
import org.opensaml.core.xml.XMLObject;
import org.opensaml.saml.common.xml.SAMLConstants;
import org.opensaml.saml.ext.saml2mdui.UIInfo;
import org.opensaml.saml.saml2.metadata.EntityDescriptor;
import org.opensaml.saml.saml2.metadata.Extensions;
import org.opensaml.saml.saml2.metadata.SPSSODescriptor;
import javax.servlet.http.HttpServletRequest;
import java.util.List;
import lombok.NoArgsConstructor;

/**
 * This is {@link MetadataUIUtils}.
 *
 * @author Misagh Moayyed
 * @since 5.1.0
 */
@Slf4j
@NoArgsConstructor
public class MetadataUIUtils {

    /**
     * Gets SP SSO descriptor.
     *
     * @param entityDescriptor the entity descriptor
     * @return the SP SSO descriptor
     */
    public static SPSSODescriptor getSPSsoDescriptor(final EntityDescriptor entityDescriptor) {
        LOGGER.trace("Locating SP SSO descriptor for SAML2 protocol...");
        SPSSODescriptor spssoDescriptor = entityDescriptor.getSPSSODescriptor(SAMLConstants.SAML20P_NS);
        if (spssoDescriptor == null) {
            LOGGER.trace("Locating SP SSO descriptor for SAML11 protocol...");
            spssoDescriptor = entityDescriptor.getSPSSODescriptor(SAMLConstants.SAML11P_NS);
        }
        if (spssoDescriptor == null) {
            LOGGER.trace("Locating SP SSO descriptor for SAML1 protocol...");
            spssoDescriptor = entityDescriptor.getSPSSODescriptor(SAMLConstants.SAML10P_NS);
        }
        LOGGER.trace("SP SSO descriptor resolved to be [{}]", spssoDescriptor);
        return spssoDescriptor;
    }

    /**
     * Is metadata found for entity id ?
     *
     * @param metadataAdapter the metadata adapter
     * @param entityId        the entity id
     * @return true/false
     */
    public static boolean isMetadataFoundForEntityId(final MetadataResolverAdapter metadataAdapter, final String entityId) {
        return metadataAdapter.getEntityDescriptorForEntityId(entityId) != null;
    }

    /**
     * Locate MDUI for entity id simple metadata ui info.
     *
     * @param metadataAdapter   the metadata adapter
     * @param entityId          the entity id
     * @param registeredService the registered service
     * @param requestContext    the request context
     * @return the simple metadata ui info
     */
    public static SamlMetadataUIInfo locateMetadataUserInterfaceForEntityId(final MetadataResolverAdapter metadataAdapter,
                                                                            final String entityId, final RegisteredService registeredService, final HttpServletRequest requestContext) {
        final EntityDescriptor entityDescriptor = metadataAdapter.getEntityDescriptorForEntityId(entityId);
        return locateMetadataUserInterfaceForEntityId(entityDescriptor, entityId, registeredService, requestContext);
    }

    /**
     * Locate mdui for entity id simple metadata ui info.
     *
     * @param entityDescriptor  the entity descriptor
     * @param entityId          the entity id
     * @param registeredService the registered service
     * @param requestContext    the request context
     * @return the simple metadata ui info
     */
    public static SamlMetadataUIInfo locateMetadataUserInterfaceForEntityId(final EntityDescriptor entityDescriptor, final String entityId,
                                                                            final RegisteredService registeredService, final HttpServletRequest requestContext) {
        final SamlMetadataUIInfo mdui = new SamlMetadataUIInfo(registeredService, requestContext.getLocale().getLanguage());
        if (entityDescriptor == null) {
            LOGGER.trace("Entity descriptor not found for [{}]", entityId);
            return mdui;
        }
        final SPSSODescriptor spssoDescriptor = getSPSsoDescriptor(entityDescriptor);
        if (spssoDescriptor == null) {
            LOGGER.trace("SP SSO descriptor not found for [{}]", entityId);
            return mdui;
        }
        final Extensions extensions = spssoDescriptor.getExtensions();
        if (extensions == null) {
            LOGGER.trace("No extensions in the SP SSO descriptor are found for [{}]", UIInfo.DEFAULT_ELEMENT_NAME.getNamespaceURI());
            return mdui;
        }
        final List<XMLObject> spExtensions = extensions.getUnknownXMLObjects(UIInfo.DEFAULT_ELEMENT_NAME);
        if (spExtensions.isEmpty()) {
            LOGGER.trace("No extensions in the SP SSO descriptor are located for [{}]", UIInfo.DEFAULT_ELEMENT_NAME.getNamespaceURI());
            return mdui;
        }
        spExtensions.stream().filter(UIInfo.class::isInstance).forEach(obj -> {
            final UIInfo uiInfo = (UIInfo) obj;
            LOGGER.trace("Found MDUI info for [{}]", entityId);
            mdui.setUiInfo(uiInfo);
        });
        return mdui;
    }
}


text:
A utility to help parsing YAML files from shell scripts.

## Build

```
go get gopkg.in/yaml.v1
go build yamlsh.go
```

## Usage

```
$ yamlsh get foo bar baz < yaml
```

## History

**0.0.1** (Jul 24, 2014)

* Initial public release.

## License

(The MIT license)

* Copyright (c) 2014 Yamashita, Yuu <<peek824545201@gmail.com>>

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


text:
module Backup
  module Database
    class MySQL < Base

      ##
      # Name of the database that needs to get dumped
      attr_accessor :name

      ##
      # Credentials for the specified database
      attr_accessor :username, :password

      ##
      # Connectivity options
      attr_accessor :host, :port, :socket

      ##
      # Tables to skip while dumping the database
      attr_accessor :skip_tables

      ##
      # Tables to dump, tables that aren't specified won't get dumped
      attr_accessor :only_tables

      ##
      # Additional "mysqldump" options
      attr_accessor :additional_options

      ##
      # Creates a new instance of the MySQL adapter object
      def initialize(&block)
        load_defaults!

        @skip_tables        ||= Array.new
        @only_tables        ||= Array.new
        @additional_options ||= Array.new

        instance_eval(&block)
        prepare!
      end

      ##
      # Builds the MySQL syntax for specifying which tables to skip
      # during the dumping of the database
      def tables_to_skip
        skip_tables.map do |table|
          "--ignore-table='#{name}.#{table}'"
        end.join("\s")
      end

      ##
      # Builds the MySQL syntax for specifying which tables to dump
      # during the dumping of the database
      def tables_to_dump
        only_tables.join("\s")
      end

      ##
      # Builds the credentials MySQL syntax to authenticate the user
      # to perform the database dumping process
      def credential_options
        %w[username password].map do |option|
          next if send(option).nil? or send(option).empty?
          "--#{option}='#{send(option)}'".gsub('--username', '--user')
        end.compact.join("\s")
      end

      ##
      # Builds the MySQL connectivity options syntax to connect the user
      # to perform the database dumping process
      def connectivity_options
        %w[host port socket].map do |option|
          next if send(option).nil? or (send(option).respond_to?(:empty?) and send(option).empty?)
          "--#{option}='#{send(option)}'"
        end.compact.join("\s")
      end

      ##
      # Builds a MySQL compatible string for the additional options
      # specified by the user
      def options
        additional_options.join("\s")
      end

      ##
      # Builds the full mysqldump string based on all attributes
      def mysqldump
        "#{ utility(:mysqldump) } #{ credential_options } #{ connectivity_options } " +
        "#{ options } #{ name } #{ tables_to_dump } #{ tables_to_skip }"
      end

      ##
      # Performs the mysqldump command and outputs the
      # data to the specified path based on the 'trigger'
      def perform!
        log!
        run("#{mysqldump} > '#{File.join(dump_path, name)}.sql'")
      end

    end
  end
end


text:
/**
 * @fileOverview {@link FileSizeValidator}クラスを記述したファイルです。
 */

/**
 * @class FileSizeValidator
 * 最大長バリデータクラス。
 * @extends FieldValidator
 */
FileSizeValidator = createSubclass("FileSizeValidator", {}, "FieldValidator");

/**
 * バリデーションを行ないます。
 * @param {String} v 値。
 * @returns {Boolean} バリデーション結果。
 */
FileSizeValidator.prototype.validate = function(v) {
	if (this.isBlank(v)) {
		return true;
	} else {
		if (this.parent.get()[0].files.length > 0) {
			var size = this.parent.get()[0].files[0].size;
			if (size > this.maxFileSize) {
				return false;
			} else {
				return true;
			}
		} else {
			return true;
		}
	}
};

/**
 * エラーメッセージを取得します。
 * @param dspname 項目名。
 * @returns {String} エラーメッセージ。
 */
FileSizeValidator.prototype.getMessage = function(dspname) {
	return MessagesUtil.getMessage(this.messageKey, dspname, (this.maxFileSize / 1024 / 1024) + "MB");
};


text:
import os.path, re, sys, glob
import platform
import ctypes
import ctypes.util

def _environ_path(name):
    if name in os.environ:
        return os.environ[name].split(":")
    else:
        return []

class LibraryLoader(object):
    def __init__(self):
        self.other_dirs=[]

    def load_library(self,libname):
        """Given the name of a library, load it."""
        paths = self.getpaths(libname)

        for path in paths:
            if os.path.exists(path):
                return self.load(path)

        raise ImportError("%s not found." % libname)

    def load(self,path):
        """Given a path to a library, load it."""
        try:
            # Darwin requires dlopen to be called with mode RTLD_GLOBAL instead
            # of the default RTLD_LOCAL.  Without this, you end up with
            # libraries not being loadable, resulting in "Symbol not found"
            # errors
            if sys.platform == 'darwin':
                return ctypes.CDLL(path, ctypes.RTLD_GLOBAL)
            else:
                return ctypes.cdll.LoadLibrary(path)
        except OSError,e:
            raise ImportError(e)

    def getpaths(self,libname):
        """Return a list of paths where the library might be found."""
        if os.path.isabs(libname):
            yield libname
        else:
            # FIXME / TODO return '.' and os.path.dirname(__file__)
            for path in self.getplatformpaths(libname):
                yield path

            path = ctypes.util.find_library(libname)
            if path: yield path

    def getplatformpaths(self, libname):
        return []

# Darwin (Mac OS X)

class DarwinLibraryLoader(LibraryLoader):
    name_formats = ["lib%s.dylib", "lib%s.so", "lib%s.bundle", "%s.dylib",
                "%s.so", "%s.bundle", "%s"]

    def getplatformpaths(self,libname):
        if os.path.pathsep in libname:
            names = [libname]
        else:
            names = [format % libname for format in self.name_formats]

        for dir in self.getdirs(libname):
            for name in names:
                yield os.path.join(dir,name)

    def getdirs(self,libname):
        '''Implements the dylib search as specified in Apple documentation:

        http://developer.apple.com/documentation/DeveloperTools/Conceptual/
            DynamicLibraries/Articles/DynamicLibraryUsageGuidelines.html

        Before commencing the standard search, the method first checks
        the bundle's ``Frameworks`` directory if the application is running
        within a bundle (OS X .app).
        '''

        dyld_fallback_library_path = _environ_path("DYLD_FALLBACK_LIBRARY_PATH")
        if not dyld_fallback_library_path:
            dyld_fallback_library_path = [os.path.expanduser('~/lib'),
                                          '/usr/local/lib', '/usr/lib']

        dirs = []

        if '/' in libname:
            dirs.extend(_environ_path("DYLD_LIBRARY_PATH"))
        else:
            dirs.extend(_environ_path("LD_LIBRARY_PATH"))
            dirs.extend(_environ_path("DYLD_LIBRARY_PATH"))

        dirs.extend(self.other_dirs)
        dirs.append(".")
        dirs.append(os.path.dirname(__file__))

        if hasattr(sys, 'frozen') and sys.frozen == 'macosx_app':
            dirs.append(os.path.join(
                os.environ['RESOURCEPATH'],
                '..',
                'Frameworks'))

        dirs.extend(dyld_fallback_library_path)

        return dirs

# Posix

class PosixLibraryLoader(LibraryLoader):
    _ld_so_cache = None

    def _create_ld_so_cache(self):
        # Recreate search path followed by ld.so.  This is going to be
        # slow to build, and incorrect (ld.so uses ld.so.cache, which may
        # not be up-to-date).  Used only as fallback for distros without
        # /sbin/ldconfig.
        #
        # We assume the DT_RPATH and DT_RUNPATH binary sections are omitted.

        directories = []
        for name in ("LD_LIBRARY_PATH",
                     "SHLIB_PATH", # HPUX
                     "LIBPATH", # OS/2, AIX
                     "LIBRARY_PATH", # BE/OS
                    ):
            if name in os.environ:
                directories.extend(os.environ[name].split(os.pathsep))
        directories.extend(self.other_dirs)
        directories.append(".")
        directories.append(os.path.dirname(__file__))

        try: directories.extend([dir.strip() for dir in open('/etc/ld.so.conf')])
        except IOError: pass

        unix_lib_dirs_list = ['/lib', '/usr/lib', '/lib64', '/usr/lib64']
        if sys.platform.startswith('linux'):
            # Try and support multiarch work in Ubuntu
            # https://wiki.ubuntu.com/MultiarchSpec
            bitage = platform.architecture()[0]
            if bitage.startswith('32'):
                # Assume Intel/AMD x86 compat
                unix_lib_dirs_list += ['/lib/i386-linux-gnu', '/usr/lib/i386-linux-gnu']
            elif bitage.startswith('64'):
                # Assume Intel/AMD x86 compat
                unix_lib_dirs_list += ['/lib/x86_64-linux-gnu', '/usr/lib/x86_64-linux-gnu']
            else:
                # guess...
                unix_lib_dirs_list += glob.glob('/lib/*linux-gnu')
        directories.extend(unix_lib_dirs_list)

        cache = {}
        lib_re = re.compile(r'lib(.*)\.s[ol]')
        ext_re = re.compile(r'\.s[ol]$')
        for dir in directories:
            try:
                for path in glob.glob("%s/*.s[ol]*" % dir):
                    file = os.path.basename(path)

                    # Index by filename
                    if file not in cache:
                        cache[file] = path

                    # Index by library name
                    match = lib_re.match(file)
                    if match:
                        library = match.group(1)
                        if library not in cache:
                            cache[library] = path
            except OSError:
                pass

        self._ld_so_cache = cache

    def getplatformpaths(self, libname):
        if self._ld_so_cache is None:
            self._create_ld_so_cache()

        result = self._ld_so_cache.get(libname)
        if result: yield result

        path = ctypes.util.find_library(libname)
        if path: yield os.path.join("/lib",path)

# Windows

class _WindowsLibrary(object):
    def __init__(self, path):
        self.cdll = ctypes.cdll.LoadLibrary(path)
        self.windll = ctypes.windll.LoadLibrary(path)

    def __getattr__(self, name):
        try: return getattr(self.cdll,name)
        except AttributeError:
            try: return getattr(self.windll,name)
            except AttributeError:
                raise

class WindowsLibraryLoader(LibraryLoader):
    name_formats = ["%s.dll", "lib%s.dll", "%slib.dll"]

    def load_library(self, libname):
        try:
            result = LibraryLoader.load_library(self, libname)
        except ImportError:
            result = None
            if os.path.sep not in libname:
                for name in self.name_formats:
                    try:
                        result = getattr(ctypes.cdll, name % libname)
                        if result:
                            break
                    except WindowsError:
                        result = None
            if result is None:
                try:
                    result = getattr(ctypes.cdll, libname)
                except WindowsError:
                    result = None
            if result is None:
                raise ImportError("%s not found." % libname)
        return result

    def load(self, path):
        return _WindowsLibrary(path)

    def getplatformpaths(self, libname):
        if os.path.sep not in libname:
            for name in self.name_formats:
                dll_in_current_dir = os.path.abspath(name % libname)
                if os.path.exists(dll_in_current_dir):
                    yield dll_in_current_dir
                path = ctypes.util.find_library(name % libname)
                if path:
                    yield path

# Platform switching

# If your value of sys.platform does not appear in this dict, please contact
# the Ctypesgen maintainers.

loaderclass = {
    "darwin":   DarwinLibraryLoader,
    "cygwin":   WindowsLibraryLoader,
    "win32":    WindowsLibraryLoader
}

loader = loaderclass.get(sys.platform, PosixLibraryLoader)()

def add_library_search_dirs(other_dirs):
    loader.other_dirs = other_dirs

load_library = loader.load_library

del loaderclass


text:
from m5.SimObject import SimObject
from m5.params import *

class PVFAnalyzer():
    cxx_header = "base/pvf/pvf_main.hh"


text:
New Product Category

            Home

            ProductCategory

            New Product Category

        <form data-ng-submit="create()" novalidate>
          <fieldset>

                Category Name

                  <input type="Text" data-ng-model="categoryName" id="categoryName" class="form-control" placeholder="" required />

                Image Of The Category

                  <input type="Text" data-ng-model="imageOfTheCategory" id="imageOfTheCategory" class="form-control" placeholder="" required />

                <input type="submit" class="btn btn-primary" />

          </fieldset>
        </form>


text:
package io.protostuff.jetbrains.plugin.formatter;

import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.ASSIGN;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.COMMA;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.GT;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.LCURLY;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.LINE_COMMENT;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.LPAREN;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.LSQUARE;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.LT;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.RCURLY;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.RPAREN;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.RSQUARE;
import static io.protostuff.jetbrains.plugin.formatter.StatementBlock.SEMICOLON;

import com.intellij.formatting.Alignment;
import com.intellij.formatting.FormattingModel;
import com.intellij.formatting.FormattingModelProvider;
import com.intellij.formatting.SpacingBuilder;
import com.intellij.formatting.Wrap;
import com.intellij.formatting.WrapType;
import com.intellij.lang.ASTNode;
import com.intellij.openapi.util.TextRange;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.codeStyle.CodeStyleSettings;
import com.intellij.psi.codeStyle.CommonCodeStyleSettings;
import io.protostuff.jetbrains.plugin.ProtoLanguage;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

/**
 * Formatting model builder.
 *
 * @author Kostiantyn Shchepanovskyi
 */
public class FormattingModelBuilder implements com.intellij.formatting.FormattingModelBuilder {

    /**
     * Create spacing builder using given settings.
     */
    public static SpacingBuilder createSpacingBuilder(CodeStyleSettings settings) {
        CommonCodeStyleSettings protoSettings = settings.getCommonSettings(ProtoLanguage.INSTANCE);
        SpacingBuilder builder = new SpacingBuilder(settings, ProtoLanguage.INSTANCE);
        builder.around(ASSIGN).spacing(1, 1, 0, false, 0);
        builder.before(SEMICOLON).spacing(0, 0, 0, false, 0);
        builder.after(LINE_COMMENT).spacing(0, 0, 1, true, 2);
        builder.after(LCURLY).spacing(0, 0, 1, true, 2);
        builder.before(RCURLY).spacing(0, 0, 1, true, 2);
        builder.after(LPAREN).spacing(0, 0, 0, false, 0);
        builder.before(RPAREN).spacing(0, 0, 0, false, 0);
        builder.after(LSQUARE).spacing(0, 0, 0, false, 0);
        builder.before(RSQUARE).spacing(0, 0, 0, false, 0);
        builder.before(LT).spacing(0, 0, 0, false, 0);
        builder.after(LT).spacing(0, 0, 0, false, 0);
        builder.before(GT).spacing(0, 0, 0, false, 0);
        builder.before(COMMA).spacing(0, 0, 0, false, 0);
        builder.before(SEMICOLON).spacing(0, 0, 0, false, 0);
        builder.after(COMMA).spacing(1, 1, 0, false, 0);
        return builder;
    }

    @NotNull
    @Override
    public FormattingModel createModel(PsiElement element, CodeStyleSettings settings) {
        PsiFile containingFile = element.getContainingFile().getViewProvider().getPsi(ProtoLanguage.INSTANCE);
        ASTNode fileNode = containingFile.getNode();
        Wrap wrap = Wrap.createWrap(WrapType.NONE, false);
        Alignment alignment = Alignment.createAlignment();
        ProtoFileBlock block = new ProtoFileBlock(fileNode, wrap, alignment, settings);
        return FormattingModelProvider.createFormattingModelForPsiFile(containingFile, block, settings);
    }

    @Nullable
    @Override
    public TextRange getRangeAffectingIndent(PsiFile file, int offset, ASTNode elementAtOffset) {
        return null;
    }
}


text:
const math = require('./math');
console.log(math.square(2));
math.demoModuleWrapper();


text:
'use strict';

describe('Test of rangeFilter', function() {
  beforeEach(module('lappsFilters'));

  var $filter;

  beforeEach(inject(function(_$filter_) {
    $filter = _$filter_;
  }));

  it('should have a given length', function() {
    expect($filter('rangeFilter')(10)).toEqual(10);
  });
});


text:
===========================
 Response Comparison Table
===========================

b=WebBob
z=Werkzeug
x=both
 =neither

WEBOB NAME                         write  read  WERKZEUG NAME                      NOTES
=================================  =====  ====  =================================  ===========================================
default_content_type                 x      x   default_mimetype                   wb default: "text/html", wz: "text/plain"
default_charset                      b      b  	                                   wz uses class var default for charset
charset                              x      x   charset
unicode_errors                       b      b
default_conditional_response         b      b
from_file() (classmethod)            b      b
copy                                 b      b
status (string)                      x      x   status
status_int                           x      x   status_code
                                            z   default_status
headers                              b      b
body                                 b      b
unicode_body                         x      x   data
body_file                                   b                                      File-like obj returned is writeable
app_iter                             b      x   get_app_iter()
                                            z   iter_encoded()
allow                                b      x   allow
vary                                 b      x   vary
content_type                         x      x   content_type
content_type_params                  x      x   mime_type_params
                                     z      z   mime_type                          content_type str wo parameters
content_length                       x      x   content_length
content_encoding                     x      x   content_encoding
content_language                     b      x   content_language
content_location                     x      x   content_location
content_md5                          x      x   content_md5
content_disposition                  b      b
accept_ranges                        b      b
content_range                        b      b
date                                 x      x   date
expires                              x      x   expires
last_modified                        x      x   last_modified
cache_control                        b      z   cache_control
cache_expires (dwim)                 b      b
conditional_response (bool)          b      x   make_conditional()
etag                                 b      x   add_etag()
etag                                 b      x   get_etag()
etag                                 b      x   set_etag()
                                            z   freeze()
location                             x      x   location
pragma                               b      b
age                                  x      x   age
retry_after                          x      x   retry_after
server                               b      b
www_authenticate                     b      z   www_authenticate
                                     x      x   date
retry_after                          x      x   retry_after
set_cookie()                                    set_cookie()
delete_cookie()                                 delete_cookie()
unset_cookie()
                                            z   is_streamed
                                            z   is_sequence
body_file                                   x   stream
                                                close()
                                                get_wsgi_headers()
                                                get_wsgi_response()
__call__()                                      __call__()


text:
package bova.rtmapi;
//Copyright 2012 Giovanni Pini
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//http://www.apache.org/licenses/LICENSE-2.0
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.

import java.io.Serializable;

/**
 * A class for the definition of a estimate duration for a Task. The fields are
 *

 *
 quantity - A float value
 *
 unit - The unit of duration associated to float value(minutes, hours or days)
 * @author Giovanni Pini
 *
 */
public class Estimate implements Serializable{
	private float quantity;
	private DurationUnit unit;
	public Estimate(float quantity, DurationUnit unit) {
		this.unit = unit;
		this.quantity = quantity;
	}

	public void setQuantity(float quantity) {
		this.quantity = quantity;
	}

	public float getQuantity() {
		return this.quantity;
	}

	public void setUnit(DurationUnit unit) {
		this.unit = unit;;
	}

	public DurationUnit getUnit() {
		return this.unit;
	}

	public enum DurationUnit {
		MINUTES,HOURS,DAYS,NONE;
	}

	public String toString() {
		return "" + quantity + " " + unit;
	}
}


text:
Many of the smaller repos under the scala organization
(for example, [scala-xml](https://github.com/scala/scala-xml))
use [Travis-CI](http://travis-ci.org) for continuous integration.
Travis requires the least setup or administration, so it's the easiest
way for maintainers from the open-source community to participate.

But for the [main Scala repository](https://github.com/scala/scala)
itself, we have found that we want the full power of Jenkins: for
capacity, for performance, and for full control over every aspect of
the builds.  Also, we want to build every commit, not just every push.

(We do, however, try to design our Jenkins configurations to be
consistent, within reason, with a possible eventual move to Travis.)

# How it works
The idea is to use chef to configure EC2 instances for both the master and the slaves. The jenkins config will be captured in chef recipes. Everything is versioned, with server and workers not allowed to maintain state.

This is inspired by https://erichelgeson.github.io/blog/2014/05/10/automating-your-automation-federated-jenkins-with-chef/

# Get some tools
```
brew cask install cord
brew install awscli
```

# One-time EC2/IAM setup
## Adding users
Create two users: one for admin ([aws console](https://typesafe-scala.signin.aws.amazon.com/console)) access (generate a password), one for CLI access (using the access key). The `awscli` package provides the `aws` cli, which is used by knife for ec2 provisioning. Add the script user to the `jenkins-knife` group, the console user to the `admin` group.

Once you have your usernames, run `aws configure`. Enter the access key for your `user-scripts` username, set the default region to `us-west-1`. Test by running `aws ec2 describe-instances`.

## Create a script user for use with knife
Never run scripts as root. Best to have a completely separate user.

This user needs the following policy (WIP!):
```
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "iam:List*",
        "iam:Get*",
        "iam:PassRole",
        "iam:PutRolePolicy"
      ],
      "Resource": "*"
    },
    {
      "Action": "ec2:*",
      "Effect": "Allow",
      "Resource": "*"
    }
  ]
}
```

## Create security group (ec2 firewall)

CONFIGURATOR_IP is the ip of the machine running knife to initiate the bootstrap -- it can be removed once chef-client is running.

```
aws ec2 create-security-group --group-name "Master" --description "Remote access to the Jenkins master"
aws ec2 authorize-security-group-ingress --group-name "Master" --protocol tcp --port 22 --cidr $CONFIGURATOR_IP/32 # ssh bootstrap
aws ec2 authorize-security-group-ingress --group-name "Master" --protocol tcp --port 8080 --cidr 0.0.0.0/0
```

| Type             | Protocol | Port Range | Source                |
|------------------|----------|------------|-----------------------|
| HTTP             |  TCP     |  80        | 0.0.0.0/0             |
| HTTPS            |  TCP     |  443       | 0.0.0.0/0             |
| Custom TCP Rule  |  TCP     |  node['scabot']['port']      | 0.0.0.0/0             |
| All traffic      |  All     |  All       | sg-ecb06389 (Workers) |
| All traffic      |  All     |  All       | sg-1dec3d78 (Windows) |
| SSH              |  TCP     |  22        | $CONFIGURATOR_IP/32   |

```
aws ec2 create-security-group --group-name "Windows" --description "Remote access to Windows instances"
aws ec2 authorize-security-group-ingress --group-name "Windows" --protocol tcp --port 5985 --cidr $CONFIGURATOR_IP/32 # allow WinRM from the machine that will execute `knife ec2 server create` below
aws ec2 authorize-security-group-ingress --group-name "Windows" --protocol tcp --port 0-65535 --source-group Master
```

| Type             | Protocol | Port Range | Source                |
|------------------|----------|------------|-----------------------|
| All TCP            | TCP | 0 - 65535 |  sg-7afd2d1f (Master) |
| Custom TCP Rule    | TCP | 445       |  $CONFIGURATOR_IP/32  |
| RDP                | TCP | 3389      |  $CONFIGURATOR_IP/32  |
| SSH                | TCP | 22        |  $CONFIGURATOR_IP/32  |
| Custom TCP Rule    | TCP | 5985      |  $CONFIGURATOR_IP/32  |

```
aws ec2 create-security-group --group-name "Workers" --description "Jenkins workers nodes"
aws ec2 authorize-security-group-ingress --group-name "Workers" --protocol tcp --port 22 --cidr $CONFIGURATOR_IP/32 # ssh bootstrap
aws ec2 authorize-security-group-ingress --group-name "Workers" --protocol tcp --port 0-65535 --source-group Master
```

| Type             | Protocol | Port Range | Source                |
|------------------|----------|------------|-----------------------|
| All TCP     |   TCP    | 0 - 65535  | sg-7afd2d1f (Master) |
| SSH         |   TCP    | 22         | $CONFIGURATOR_IP/32  |

## Instance profiles
This avoids passing credentials for instances to use aws services.

### Create instance profiles
An instance profile must be passed when instance is created. It must contain one role. Can attach more policies to that role at any time.

Based on http://domaintest001.com/aws-iam/

```
aws iam create-instance-profile --instance-profile-name JenkinsMaster
aws iam create-instance-profile --instance-profile-name JenkinsWorkerPublish
aws iam create-instance-profile --instance-profile-name JenkinsWorker

aws iam create-role --role-name jenkins-master         --assume-role-policy-document file://$PWD/chef/ec2-role-trust-policy.json
aws iam create-role --role-name jenkins-worker         --assume-role-policy-document file://$PWD/chef/ec2-role-trust-policy.json
aws iam create-role --role-name jenkins-worker-publish --assume-role-policy-document file://$PWD/chef/ec2-role-trust-policy.json

aws iam add-role-to-instance-profile --instance-profile-name JenkinsMaster        --role-name jenkins-master
aws iam add-role-to-instance-profile --instance-profile-name JenkinsWorker        --role-name jenkins-worker
aws iam add-role-to-instance-profile --instance-profile-name JenkinsWorkerPublish --role-name jenkins-worker-publish
```

### Attach policies to roles:
NOTE: if you get syntax errors, check the policy doc URL

```
aws iam put-role-policy --role-name jenkins-master --policy-name jenkins-ec2-start-stop        --policy-document file://$PWD/chef/jenkins-ec2-start-stop.json
aws iam put-role-policy --role-name jenkins-master --policy-name jenkins-dynamodb              --policy-document file://$PWD/chef/dynamodb.json
aws iam put-role-policy --role-name jenkins-master --policy-name jenkins-ebs-create-vol        --policy-document file://$PWD/chef/ebs-create-vol.json
```

```
aws iam put-role-policy --role-name jenkins-worker --policy-name jenkins-ebs-create-vol        --policy-document file://$PWD/chef/ebs-create-vol.json
```

TODO: once https://github.com/sbt/sbt-s3/issues/14 is fixed, remove s3credentials from nodes (use IAM profile below instead)
```
aws iam put-role-policy --role-name jenkins-worker-publish --policy-name jenkins-s3-upload      --policy-document file://$PWD/chef/jenkins-s3-upload.json
aws iam put-role-policy --role-name jenkins-worker-publish --policy-name jenkins-ebs-create-vol --policy-document file://$PWD/chef/ebs-create-vol.json
```

## Create an Elastic IP for each node
TODO: attach to elastic IPs

# Install chef/knife

```
brew cask install chefdk
eval "$(chef shell-init zsh)" # set up gem environment
gem install knife-ec2 knife-windows knife-github-cookbooks chef-vault
```

## Get credentials for the typesafe-scala chef.io organization
Join chef.io (https://manage.chef.io/signup), and ask to be invited to the typesafe-scala org on slack.

For the CLI to work, you need:
```
export CHEF_ORG="typesafe-scala"
```

If your username on chef.io does not match the local username on your machine, you also need
```
export CHEF_USER="[username]"
```

You can then generate and download your private key on https://www.chef.io/account/password. Put it to `$PWD/.chef/config/$CHEF_USER.pem`, then you can use knife without further config. See `$PWD/.chef/knife.rb` for key locations.

Test if knife works correctly by running `knife cookbook list`.

Obtain the organization validation key from Adriaan and put it to `$PWD/.chef/config/$CHEF_ORG-validator.pem`. (Q: When is this key used exactly? https://docs.chef.io/chef_private_keys.html says it's when a new node runs `chef-client` for the first time.)

## Clone scala-jenkins-infra cookbook and its dependencies

I think you can safely ignore `ERROR: IOError: Cannot open or read **/metadata.rb!` in the below

```
cd ~/git/cookbooks
git init .
g commit --allow-empty -m"Initial"

hub clone scala/scala-jenkins-infra
cd scala-jenkins-infra
ln -sh ~/git/cookbooks $PWD/.chef/

knife cookbook site install cron
knife cookbook site install logrotate
knife cookbook site install chef_handler
knife cookbook site install windows
knife cookbook site install chef-client
knife cookbook site install aws
knife cookbook site install delayed_evaluator
knife cookbook site install ebs
knife cookbook site install apt
knife cookbook site install packagecloud
knife cookbook site install runit
knife cookbook site install yum
knife cookbook site install 7-zip
knife cookbook site install ark
knife cookbook site install artifactory
knife cookbook site install build-essential
knife cookbook site install dmg
knife cookbook site install yum-epel
knife cookbook site install git
knife cookbook site install user
knife cookbook site install partial_search
knife cookbook site install ssh_known_hosts
knife cookbook site install git_user

knife cookbook site install chef-vault
```

### Current cookbooks
 - 7-zip               ==  1.0.2
 - apt                 ==  2.7.0
 - ark                 ==  0.9.0
 - artifactory         ==  0.1.1
 - aws                 ==  2.7.0
 - build-essential     ==  2.2.3
 - chef-client         ==  4.3.0
 - chef_handler        ==  1.1.6
 - cron                ==  1.6.1
 - delayed_evaluator   ==  0.2.0
 - dmg                 ==  2.2.2
 - ebs                 ==  0.3.6
 - git                 ==  4.2.2
 - git_user            ==  0.3.1
 - logrotate           ==  1.9.1
 - packagecloud        ==  0.0.17
 - partial_search      ==  1.0.8
 - runit               ==  1.6.0
 - sbt                 ==  0.1.0
 - sbt-extras          ==  0.4.0
 - ssh_known_hosts     ==  2.0.0
 - user                ==  0.4.2
 - windows             ==  1.36.6
 - yum                 ==  3.6.0
 - yum-epel            ==  0.6.0

### Switch to unreleased versions from github
```
knife cookbook github install adriaanm/jenkins/fix305  # custom fixes + https://github.com/opscode-cookbooks/jenkins/pull/313 (b-dean/jenkins/http_ca_fixes)
knife cookbook github install adriaanm/java/windows-jdk1.6  # jdk 1.6 installer barfs on re-install -- wipe its INSTALLDIR
knife cookbook github install adriaanm/chef-sbt
knife cookbook github install gildegoma/chef-sbt-extras
knife cookbook github install adriaanm/artifactory
```

### Upload cookbooks to chef server
```
knife cookbook upload --all
```

## Cache installers locally
- they are tricky to access, might disappear,...
- checksum is computed with `shasum -a 256`
- TODO: host them on an s3 bucket (credentials are available automatically)

# Configuring the jenkins cluster

## Secure data (one-time setup, can be done before bootstrap)

from http://jtimberman.housepub.org/blog/2013/09/10/managing-secrets-with-chef-vault/

NOTE: the JSON must not have a field "id"!!!

### Chef user with keypair for jenkins cli access
```
eval "$(chef shell-init zsh)" # use chef's ruby, which has the net/ssh gem
ruby chef/keypair.rb > $PWD/.chef/keypair.json
ruby chef/keypair.rb > $PWD/.chef/scabot-keypair.json

# extract private key to $PWD/.chef/scabot.pem

knife vault create master scala-jenkins-keypair \
  --json $PWD/.chef/keypair.json \
  --search 'name:jenkins*' \
  --admins adriaan

knife vault create master scabot-keypair \
  --json $PWD/.chef/scabot-keypair.json \
  --search 'name:jenkins-master' \
  --admins adriaan

knife vault create master scabot \
  {
    "jenkins": {
      "token": "..."
    },
    "github": {
      "scala":    {"token": "..."}
      "lampepfl": {"token": "..."}
    }
  }
  --search 'name:jenkins-master' \
  --admins adriaan

```

### For github oauth

https://github.com/settings/applications/new --> https://github.com/settings/applications/154904
 - Authorization callback URL = https://scala-ci.typesafe.com/securityRealm/finishLogin

```
knife vault create master github-api \
  '{"client-id":"<Client ID>","client-secret":"<Client secret>"}' \
  --search 'name:jenkins-master' \
  --admins adriaan
```

### For nginx ssl

```
knife vault create master scala-ci-key \
  --json scalaci-key.json \
  --search 'name:jenkins-master' \
  --admins adriaan
```

### Workers that need to publish
```
knife vault create worker-publish sonatype \
  '{"user":"XXX","pass":"XXX"}' \
  --search 'name:jenkins-worker-ubuntu-publish' \
  --admins adriaan

knife vault create worker-publish private-repo \
  '{"user":"XXX","pass":"XXX"}' \
  --search 'name:jenkins-worker-ubuntu-publish' \
  --admins adriaan

knife vault create worker-publish s3-downloads \
  '{"user":"XXX","pass":"XXX"}' \
  --search 'name:jenkins-worker-*-publish' \
  --admins adriaan

knife vault create worker-publish chara-keypair \
  --json $PWD/.chef/config/chara-keypair.json \
  --search 'name:jenkins-worker-ubuntu-publish' \
  --admins adriaan

knife vault create worker-publish gnupg \
  --json   $PWD/.chef/config/gnupg.json \
  --search 'name:jenkins-worker-ubuntu-publish' \
  --admins adriaan

knife vault create worker private-repo-public-jobs \
  '{"user":"XXX","pass":"XXX"}' \
  --search 'name:jenkins-worker-behemoth-*' \
  --admins adriaan

```

#  Dev machine convenience
This is how I set up my desktop to make it easier to connect to the EC2 nodes.
The README assumes you're using this as well.

## /etc/hosts
Note that the IPs are stable by allocating elastic IPs and associating them to nodes.

```
54.67.111.226 jenkins-master
54.67.33.167  jenkins-worker-ubuntu-publish
54.183.156.89 jenkins-worker-windows-publish
54.153.2.9    jenkins-worker-behemoth-1
54.153.1.99   jenkins-worker-behemoth-2
```

## ~/.ssh/config
```
Host jenkins-worker-ubuntu-publish
  IdentityFile ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
  User ubuntu

Host jenkins-worker-behemoth-1
  IdentityFile ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
  User ec2-user

Host jenkins-worker-behemoth-2
  IdentityFile ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
  User ec2-user

Host jenkins-master
  IdentityFile ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
  User ec2-user

Host scabot
  HostName jenkins-master
  IdentityFile $PWD/.chef/scabot.pem
  User scabot

Host jenkins-worker-windows-publish
  IdentityFile ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
  User jenkins
```

# Launch instance on EC2
## Create (ssh) key pair

TODO: I don't think the name matters as long as it's used consistently, ultimately your access key and secret credentials are used by aws-cli to generate the keys etc

If your username on AWS does not match the local username on your machine, define
```
export AWS_USER="[username]"
```

Create a keypair and store locally to authenticate with instances over ssh/winrm:
```
echo $(aws ec2 create-key-pair --key-name $AWS_USER | jq .KeyMaterial) | perl -pe 's/"//g' > ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
chmod 0600 ~/.ssh/typesafe-scala-aws-$AWS_USER.pem
```

## Selected AMIs

  - jenkins-master: ami-3b14f27f (Amazon Linux AMI 2015.03 on HVM Instance Store 64-bit for US West N. California)
  - windows:        ami-cfa5b68a (Windows_Server-2012-R2_RTM-English-64Bit-Base-2014.12.10)
  - ubuntu:         ami-2915e16d (search for "vivid hvm:ebs-ssd us-west-1" on https://cloud-images.ubuntu.com/locator/ec2/)

## Bootstrap
NOTE:

  - name is important (used to allow access to vault etc); it can't be changed later, and duplicates aren't allowed (can bite when repeating knife ec2 create)
  - can't access the vault on bootstrap (see After bootstrap below)

```
   --subnet subnet-4bb3b80d --associate-eip 54.67.111.226 \
   --server-connect-attribute public_ip_address           \

knife ec2 server create -N jenkins-master                  \
   --flavor m3.large                                       \
   --region us-west-1                                      \
   -I ami-3b14f27f                                         \
   -G Master --ssh-user ec2-user                           \
   --iam-profile JenkinsMaster                             \
   --security-group-ids sg-7afd2d1f                        \
   --identity-file ~/.ssh/typesafe-scala-aws-$AWS_USER.pem \
   --run-list "scala-jenkins-infra::master-init"

knife ec2 server create -N jenkins-worker-windows-publish \
   --flavor c4.xlarge                                     \
   --region us-west-1                                     \
   -I ami-45332200 --user-data chef/userdata/win2012.txt  \
   --iam-profile JenkinsWorkerPublish                     \
   --ebs-optimized --ebs-volume-type gp2                  \
   --security-group-ids sg-1dec3d78                       \
   --subnet subnet-4bb3b80d --associate-eip 54.183.156.89 \
   --server-connect-attribute public_ip_address           \
   --identity-file ~/.ssh/typesafe-scala-aws-$AWS_USER.pem             \
   --run-list "scala-jenkins-infra::worker-init"

// NOTE: c3.large is much slower than c3.xlarge (scala-release-2.11.x-build takes 2h53min vs 1h40min )

# NOTE: Make sure to first remove the ips in $workerIp from your ~/.ssh/known_hosts.
# Also remove the corresponding worker from the chef server (can be only one with the same name).

workerName=(jenkins-worker-behemoth-1 jenkins-worker-behemoth-2 jenkins-worker-ubuntu-publish)
workerIp=(54.153.2.9 54.153.1.99 54.67.33.167)
workerFlavor=(c4.2xlarge c4.2xlarge c4.xlarge)

for worker in 1 2 3
do knife ec2 server create -N ${workerName[$worker]}             \
   --flavor ${workerFlavor[$worker]}                             \
   --region us-west-1                                            \
   -I ami-81afbcc4 --ssh-user ubuntu                             \
   --hint ec2                                                    \
   --iam-profile JenkinsWorker                                   \
   --ebs-optimized --ebs-volume-type gp2                         \
   --security-group-ids sg-ecb06389                              \
   --subnet subnet-4bb3b80d --associate-eip ${workerIp[$worker]} \
   --server-connect-attribute public_ip_address                  \
   --identity-file ~/.ssh/typesafe-scala-aws-$AWS_USER.pem       \
   --run-list "scala-jenkins-infra::worker-init"
done
```

### NOTES
- `--hint ec2` should enable ec2 detection (so that `node[:ec2]` gets populated by ohai);
  It does the equivalent of `ssh ${workerIp[$worker]} sudo mkdir -p /etc/chef/ohai/hints/ && sudo touch /etc/chef/ohai/hints/ec2.json`

- userdata.txt must be one line, no line endings (mac/windows issues?)
  `<script>winrm quickconfig -q & winrm set winrm/config/service @{AllowUnencrypted="true"} & winrm set winrm/config/service/auth @{Basic="true"} & netsh advfirewall firewall set rule group="remote administration" new enable=yes & netsh advfirewall firewall add rule name="WinRM Port" dir=in action=allow protocol=TCP  localport=5985</script>`

## After bootstrap (or when nodes are added)

### Update access to vault
```
knife vault update master github-api            --search 'name:jenkins-master'

knife vault update master scala-jenkins-keypair --search 'name:jenkins*'

knife vault update worker private-repo-public-jobs --search 'name:jenkins-worker-behemoth-*'

knife vault update worker-publish s3-downloads  --search 'name:jenkins-worker-*-publish'

knife vault update worker-publish sonatype      --search 'name:jenkins-worker-ubuntu-publish'
knife vault update worker-publish private-repo  --search 'name:jenkins-worker-ubuntu-publish'
knife vault update worker-publish chara-keypair --search 'name:jenkins-worker-ubuntu-publish'
knife vault update worker-publish gnupg         --search 'name:jenkins-worker-ubuntu-publish'
```

### Add run-list items that need the vault
```
knife node run_list set jenkins-master  "recipe[chef-vault],scala-jenkins-infra::master-init,scala-jenkins-infra::master-config,scala-jenkins-infra::master-jenkins"

for w in jenkins-worker-windows-publish jenkins-worker-ubuntu-publish jenkins-worker-behemoth-1 jenkins-worker-behemoth-2
  do knife node run_list set $w  "recipe[chef-vault],scala-jenkins-infra::worker-init,scala-jenkins-infra::worker-config"
done
```

### Re-run chef manually

- windows:
```
PASS=$(aws ec2 get-password-data --instance-id i-f67c0a35 --priv-launch-key ~/.ssh/typesafe-scala-aws-$AWS_USER.pem | jq .PasswordData | xargs echo)
knife winrm jenkins-worker-windows-publish chef-client -m -P $PASS
```

- linux
```
ssh jenkins-worker-ubuntu-publish
sudo su --login # --login needed on ubuntu to set SSL_CERT_FILE (it's done in /etc/profile.d)
chef-client
```

### Attach eips

```
aws ec2 associate-address --allocation-id eipalloc-df0b13bd --instance-id i-94adaa5e  # jenkins-master
```

# Example of bringing up a new version of our beloved behemoths
- delete slave in jenkins: https://scala-ci.typesafe.com/computer/jenkins-worker-behemoth-1/delete
- rename EC2 instance in EC2 console (suffix with -old)
- disassociate its EIP (54.153.2.9)
- delete node from manage.chef.io (we only have 5 max)
- remove jenkins-worker-behemoth-1 entry from ~/.ssh/known_hosts
- bootstrap:
```
$ knife ec2 server create -N jenkins-worker-behemoth-1        \
>    --flavor c4.2xlarge                                      \
>    --region us-west-1                                       \
>    -I ami-81afbcc4 --ssh-user ubuntu                        \
>    --iam-profile JenkinsWorker                              \
>    --ebs-optimized --ebs-volume-type gp2                    \
>    --security-group-ids sg-ecb06389                         \
>    --subnet subnet-4bb3b80d --associate-eip 54.153.2.9     \
>    --server-connect-attribute public_ip_address             \
>    --identity-file ~/.ssh/typesafe-scala-aws-$AWS_USER.pem  \
>    --run-list "scala-jenkins-infra::worker-init"
```

- knife vault update master scala-jenkins-keypair --search 'name:jenkins-worker-behemoth-1'
- knife vault update worker private-repo-public-jobs --search 'name:jenkins-worker-behemoth-1'
- knife node run_list set jenkins-worker-behemoth-1  "recipe[chef-vault],scala-jenkins-infra::worker-init,scala-jenkins-infra::worker-config"

- mate ~/.ssh/config
```
Host jenkins-worker-behemoth-1
  IdentityFile /Users/adriaan/.ssh/typesafe-scala-aws-adriaan-scripts.pem
  User ubuntu
```

- ssh jenkins-worker-behemoth-1 and sudo su
```
$ mkdir -p /etc/chef/ohai/hints/
$ touch /etc/chef/ohai/hints/ec2.json
$ chef-client
```

- ssh-jenkins master, sudo and run chef-client to add back the deleted worker

# MANUAL STEPS
## Scabot access to jenkins
The jenkins token for scabot has to be configured manually:
 - get the API token from https://scala-ci.typesafe.com/user/scala-jenkins/configure
 - use it create `scabot-jenkins.json` as follows
 ```
 {
   "id": "scabot",
   "jenkins": {
     "token": "<TOKEN>"
   }
 }
 ```
 - do `knife vault update master scabot -J scabot-jenkins.json`

# Artifactory
 - Set admin password.
 - create repos (TODO: automate)
 - Create scala-ci user that can push to scala-release-temp and scala-pr-validation-snapshots,
 - coordinate scala-ci credentials with jenkins via
```
knife vault update worker-publish private-repo -J private-repo.json
```

where `private-repo.json`:
```
{
  "id": "private-repo",
  "user": "scala-ci",
  "pass": "???"
}
```

# Misc

## Worker offline?

If you see "pending -- (worker) is offline", try waiting ~5 minutes;
it takes time for ec2-start-stop to spin up workers.

## "ERROR: null" in slave agent launch log
There are probably multiple instances with the same name on EC2: https://github.com/adriaanm/ec2-start-stop/issues/4
Workaround: make sure EC2 instance names are unique.

## Testing locally using vagrant

http://blog.gravitystorm.co.uk/2013/09/13/using-vagrant-to-test-chef-cookbooks/:

See `$PWD/.chef/Vagrantfile` -- make sure you first populated `$PWD/.chef/cookbooks/` using knife,
as [documented above](#get-cookbooks)

## If connections hang
Make sure security group allows access, winrm was enabled using --user-data...

## SSL cert
```
$ openssl genrsa -out scala-ci.key 2048
```
and

```
$ openssl req -new -out scala-ci.csr -key scala-ci.key -config ssl-certs/scalaci.openssl.cnf
```

Send CSR to SSL provider, receive scalaci.csr. Store scala-ci.key securely in vault master scala-ci-key (see above).

Incorporate the cert into an ssl chain for nginx:
```
(cd ssl-certs && cat 00\ -\ scala-ci.crt 01\ -\ COMODORSAOrganizationValidationSecureServerCA.crt 02\ -\ COMODORSAAddTrustCA.crt 03\ -\ AddTrustExternalCARoot.crt > ../files/default/scala-ci.crt)
```

For [forward secrecy](http://axiacore.com/blog/enable-perfect-forward-secrecy-nginx/):
```
openssl dhparam -out files/default/dhparam.pem 1024
```

Using 1024 bits (instead of 2048) for DH to be Java 6 compatible... Bye-bye A+ on https://www.ssllabs.com/ssltest/analyze.html?d=scala-ci.typesafe.com

Confirm values in the csr using:

```
$ openssl req -text -noout -in scala-ci.csr
```

## Retry bootstrap
```
knife bootstrap -c $PWD/.chef/knife.rb jenkins-worker-ubuntu-publish --ssh-user ubuntu --sudo -c $PWD/.chef/knife.rb -N jenkins-worker-ubuntu-publish -r "scala-jenkins-infra::worker-init"
```

## WinRM troubles?
If it appears stuck at "Waiting for remote response before bootstrap.", the userdata didn't make it across
(check C:\Program Files\Amazon\Ec2ConfigService\Logs) we need to enable unencrypted authentication:

```
aws ec2 get-password-data --instance-id $INST --priv-launch-key ~/.ssh/typesafe-scala-aws-$AWS_USER.pem

cord $IP, log in using password above and open a command line:

  winrm set winrm/config/service @{AllowUnencrypted="true"}
  winrm set winrm/config/service/auth @{Basic="true"}

knife bootstrap -V windows winrm $IP
```

## Alternative windows AMIs
too stripped down (bootstraps in 8 min, though): ami-23a5b666 Windows_Server-2012-R2_RTM-English-64Bit-Core-2014.12.10
userdata.txt: `<script>winrm quickconfig -q & winrm set winrm/config/service @{AllowUnencrypted="true"} & winrm set winrm/config/service/auth @{Basic="true"} & netsh advfirewall firewall set rule group="remote administration" new enable=yes & netsh advfirewall firewall add rule name="WinRM Port" dir=in action=allow protocol=TCP  localport=5985</script>`

older: ami-e9a4b7ac amazon/Windows_Server-2008-SP2-English-64Bit-Base-2014.12.10
userdata.txt: '<script>winrm quickconfig -q & winrm set winrm/config/service @{AllowUnencrypted="true"} & winrm set winrm/config/service/auth @{Basic="true"}</script>'

older: ami-6b34252e Windows_Server-2008-R2_SP1-English-64Bit-Base-2014.11.19
doesn't work: ami-59a8bb1c Windows_Server-2003-R2_SP2-English-64Bit-Base-2014.12.10


text:
Run your shell inside docker and take docker with you.

## Synopsis

    $ sudo ./dsh
    Docker image not found. Rebuilding.
    Sending build context to Docker daemon
    Step 0 : FROM phusion/baseimage
    [...]
    Successfully built [...]
    # docker ps
    CONTAINER ID        IMAGE                       COMMAND                CREATED             STATUS
    cdbab3de4ac7        dsh:latest              "/bin/sh -c 'cd /roo   3 seconds ago       Up 2 seconds
    # exit
    $

## What it does

Docker shell (dsh) is a bash shell inside a container, and will bring
any docker configuration along, making it possible to control docker from
inside dsh.

The shell comes with a data container of its own that is persisted across
invocations of dsh. Anything under `/root` (a symlink to `/workspace`) is
persisted in the data container named `dsh-ws`.

When run, the shell will automatically launch a smb file share, exposing
the volumes avaliable in dsh (mainly `/workspace`), making it easy to share
files with the shell.

## Parameters

 -s|--samba: start samba server for file sharing
 -b|--build: rebuild dsh image

Use `--` to indicate start of command line to run inside dsh:

    $ dsh -- ls /
    bin   dev  home  lib64	mnt  proc  run	 srv  tmp  var
    boot  etc  lib	 media	opt  root  sbin  sys  usr  workspace
    $

## Author

Written by Michael Zedeler <michael@zedeler.dk>

## Copyright and license

Copyright Nota (http://nota.dk), licensed under the MIT license, see LICENSE.


text:
#ifndef RENDERSCRIPT_MIGRATION_SAMPLE_COMPUTE_PIPELINE_H
#define RENDERSCRIPT_MIGRATION_SAMPLE_COMPUTE_PIPELINE_H

#include <android/asset_manager_jni.h>
#include <vulkan/vulkan_core.h>

#include <chrono>
#include <memory>

#include "VulkanContext.h"
#include "VulkanResources.h"

namespace sample {

// ComputePipeline manages the Vulkan objects for a single compute task with a compute shader.
// In this sample app, the compute shaders always take 2D images as the input and output, with
// runtime parameters passed by an uniform buffer. The image and buffer resources are managed
// outside of this class.
class ComputePipeline {
   public:
    // Create a compute pipeline with the input shader.
    // Return the created ComputePipeline on success, or nullptr if failed.
    static std::unique_ptr<ComputePipeline> create(const VulkanContext* context, const char* shader,
                                                   AAssetManager* assetManager,
                                                   uint32_t pushConstantSize,
                                                   bool useUniformBuffer);

    // Prefer ComputePipeline::create
    ComputePipeline(const VulkanContext* context, uint32_t pushConstantSize)
        : mContext(context),
          mDescriptorSetLayout(context->device()),
          mPipelineLayout(context->device()),
          mPipeline(context->device()),
          mPushConstantSize(pushConstantSize) {}

    // Record the compute pipeline to the command buffer with the given uniform buffer and
    // input/output image.
    void recordComputeCommands(VkCommandBuffer cmd, const void* pushConstantData,
                               const Image& inputImage, const Image& outputImage,
                               const Buffer* uniformBuffer = nullptr);

   protected:
    // Initialization
    bool createDescriptorSet(bool useUniformBuffer);
    bool createComputePipeline(const char* shader, AAssetManager* assetManager);

    // Update descriptor sets with the given input and output image.
    bool updateDescriptorSets(const Image& inputImage, const Image& outputImage,
                              const Buffer* uniformBuffer);

    // Context
    const VulkanContext* mContext;

    // Compute pipeline
    VulkanDescriptorSetLayout mDescriptorSetLayout;
    VulkanPipelineLayout mPipelineLayout;
    VkDescriptorSet mDescriptorSet = VK_NULL_HANDLE;
    VulkanPipeline mPipeline;
    uint32_t mPushConstantSize;
};

}  // namespace sample

#endif  // RENDERSCRIPT_MIGRATION_SAMPLE_COMPUTE_PIPELINE_H


text:
/**
 * @module ol/format/JSONFeature
 */
import FeatureFormat from '../format/Feature.js';
import FormatType from '../format/FormatType.js';

/**
 * @classdesc
 * Abstract base class; normally only used for creating subclasses and not
 * instantiated in apps.
 * Base class for JSON feature formats.
 *
 * @abstract
 */
var JSONFeature = (function (FeatureFormat) {
  function JSONFeature() {
    FeatureFormat.call(this);
  }

  if ( FeatureFormat ) JSONFeature.__proto__ = FeatureFormat;
  JSONFeature.prototype = Object.create( FeatureFormat && FeatureFormat.prototype );
  JSONFeature.prototype.constructor = JSONFeature;

  /**
   * @inheritDoc
   */
  JSONFeature.prototype.getType = function getType () {
    return FormatType.JSON;
  };

  /**
   * Read a feature.  Only works for a single feature. Use `readFeatures` to
   * read a feature collection.
   *
   * @param {ArrayBuffer|Document|Node|Object|string} source Source.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @return {module:ol/Feature} Feature.
   * @api
   */
  JSONFeature.prototype.readFeature = function readFeature (source, opt_options) {
    return this.readFeatureFromObject(
      getObject(source), this.getReadOptions(source, opt_options));
  };

  /**
   * Read all features.  Works with both a single feature and a feature
   * collection.
   *
   * @param {ArrayBuffer|Document|Node|Object|string} source Source.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @return {Array.<module:ol/Feature>} Features.
   * @api
   */
  JSONFeature.prototype.readFeatures = function readFeatures (source, opt_options) {
    return this.readFeaturesFromObject(
      getObject(source), this.getReadOptions(source, opt_options));
  };

  /**
   * @abstract
   * @param {Object} object Object.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @protected
   * @return {module:ol/Feature} Feature.
   */
  JSONFeature.prototype.readFeatureFromObject = function readFeatureFromObject (object, opt_options) {};

  /**
   * @abstract
   * @param {Object} object Object.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @protected
   * @return {Array.<module:ol/Feature>} Features.
   */
  JSONFeature.prototype.readFeaturesFromObject = function readFeaturesFromObject (object, opt_options) {};

  /**
   * Read a geometry.
   *
   * @param {ArrayBuffer|Document|Node|Object|string} source Source.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @return {module:ol/geom/Geometry} Geometry.
   * @api
   */
  JSONFeature.prototype.readGeometry = function readGeometry (source, opt_options) {
    return this.readGeometryFromObject(
      getObject(source), this.getReadOptions(source, opt_options));
  };

  /**
   * @abstract
   * @param {Object} object Object.
   * @param {module:ol/format/Feature~ReadOptions=} opt_options Read options.
   * @protected
   * @return {module:ol/geom/Geometry} Geometry.
   */
  JSONFeature.prototype.readGeometryFromObject = function readGeometryFromObject (object, opt_options) {};

  /**
   * Read the projection.
   *
   * @param {ArrayBuffer|Document|Node|Object|string} source Source.
   * @return {module:ol/proj/Projection} Projection.
   * @api
   */
  JSONFeature.prototype.readProjection = function readProjection (source) {
    return this.readProjectionFromObject(getObject(source));
  };

  /**
   * @abstract
   * @param {Object} object Object.
   * @protected
   * @return {module:ol/proj/Projection} Projection.
   */
  JSONFeature.prototype.readProjectionFromObject = function readProjectionFromObject (object) {};

  /**
   * Encode a feature as string.
   *
   * @param {module:ol/Feature} feature Feature.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {string} Encoded feature.
   * @api
   */
  JSONFeature.prototype.writeFeature = function writeFeature (feature, opt_options) {
    return JSON.stringify(this.writeFeatureObject(feature, opt_options));
  };

  /**
   * @abstract
   * @param {module:ol/Feature} feature Feature.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {Object} Object.
   */
  JSONFeature.prototype.writeFeatureObject = function writeFeatureObject (feature, opt_options) {};

  /**
   * Encode an array of features as string.
   *
   * @param {Array.<module:ol/Feature>} features Features.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {string} Encoded features.
   * @api
   */
  JSONFeature.prototype.writeFeatures = function writeFeatures (features, opt_options) {
    return JSON.stringify(this.writeFeaturesObject(features, opt_options));
  };

  /**
   * @abstract
   * @param {Array.<module:ol/Feature>} features Features.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {Object} Object.
   */
  JSONFeature.prototype.writeFeaturesObject = function writeFeaturesObject (features, opt_options) {};

  /**
   * Encode a geometry as string.
   *
   * @param {module:ol/geom/Geometry} geometry Geometry.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {string} Encoded geometry.
   * @api
   */
  JSONFeature.prototype.writeGeometry = function writeGeometry (geometry, opt_options) {
    return JSON.stringify(this.writeGeometryObject(geometry, opt_options));
  };

  /**
   * @abstract
   * @param {module:ol/geom/Geometry} geometry Geometry.
   * @param {module:ol/format/Feature~WriteOptions=} opt_options Write options.
   * @return {Object} Object.
   */
  JSONFeature.prototype.writeGeometryObject = function writeGeometryObject (geometry, opt_options) {};

  return JSONFeature;
}(FeatureFormat));

/**
 * @param {Document|Node|Object|string} source Source.
 * @return {Object} Object.
 */
function getObject(source) {
  if (typeof source === 'string') {
    var object = JSON.parse(source);
    return object ? /** @type {Object} */ (object) : null;
  } else if (source !== null) {
    return source;
  } else {
    return null;
  }
}

export default JSONFeature;

//# sourceMappingURL=JSONFeature.js.map


text:
<?php

class BxShopifyUpdater extends BxDolStudioUpdater
{
    function __construct($aConfig)
    {
        parent::__construct($aConfig);
    }

    public function actionExecuteSql($sOperation)
    {
        if($sOperation == 'install') {
            if(!$this->oDb->isFieldExists('bx_shopify_cmts', 'cmt_pinned'))
                $this->oDb->query("ALTER TABLE `bx_shopify_cmts` ADD `cmt_pinned` int(11) NOT NULL default '0' AFTER `cmt_replies`");
        }

        return parent::actionExecuteSql($sOperation);
    }
}


text:
<?php

class HTMLRawFormatter {

	public function buildHTML($bodyContent){

		$htmlStart = "
		<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN' 'http://www.w3.org/TR/html4/loose.dtd'>

				<meta http-equiv='Content-Type' content='text/html; charset=UTF-8'>
				<title>ESWC - Semantic Publishing Challenge Evaluation</title>
				<link rel='stylesheet' type='text/css' href='style.css'/>

			";

		$htmlEnd = "";

		date_default_timezone_set("CET");
		$timestampDIV = "
Generated on: ".date("D, d M Y H:i:s")."
";

		$html = $htmlStart.$bodyContent.$timestampDIV.$htmlEnd;

		return $html;
	}

	public function renderIntroductionAsHTML($evaluationLevel, $taskNumber, $submissionNumber, $zipOption){

		$htmlIntroContent = "

SemPub Challenge @ ESWC  - Evaluation

This page shows the results of the evaluation of submission #$submissionNumber, for the best-performing approach of task #$taskNumber. See SemPub Challenge wiki for more information.

The output of some queries on the submitted LOD is compared with a gold standard, and precision and recall are measured.
		Both the gold standard and the output under evaluation are expected to be CSV files with a common structure.
";

		if ($zipOption)
			$htmlIntroContent = $htmlIntroContent . "
The source files are available here (as ZIP archives): Gold Standard and Output Under Evaluation. Note that CSV files are not included in the ZIP file under evaluation if the corresponding query returned an empty output.
";

		$htmlIntroContent = $htmlIntroContent . "
The evaluation process is described in info.html. This page shows the results of the $evaluationLevel evaluation.
";

		return "
".$htmlIntroContent."
";
	}
}
?>


text:
from django.test import TestCase
from django.contrib.auth.models import User
from django.shortcuts import reverse
from django.utils import timezone

import datetime

from .models import Budget, BudgetCategory, Wallet

class MainViewsTests(TestCase):
    def setUp(self):
        # Create a user and give them some data to test with
        self.user = User.objects.create_user(id=1, username='test_user', email='test_user@gmail.com', password='tester123')
        self.user.save()

        self.category = BudgetCategory.objects.create(id=1, user_id=1, user=self.user, name='groceries', is_income=False)
        self.category.save()

        self.wallet = Wallet.objects.create(id=1, user_id=1, name='checking', balance='100', created_time=timezone.now())
        self.wallet.save()

        self.budget = Budget.objects.create(budget_id=1, user_id=1, category_id=1, goal=100, month=datetime.date.today(), wallet_id=1, balance=0, user=self.user)
        self.budget.save()

    def tearDown(self):
        self.budget.delete()
        self.wallet.delete()
        self.category.delete()
        self.user.delete()

    def test_logout_view(self):
        self.client.login(username='test_user', password='tester123')
        response = self.client.get(reverse('logout'), follow=True)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(str(response.context.get('user', '')), 'AnonymousUser')

    def test_login(self):
        self.client.login(username='test_user', password='tester123')
        response = self.client.get(reverse('index'))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(str(response.context.get('user', '')), 'test_user')

    def test_unauthenticated_index_access(self):
        response = self.client.get(reverse('index'))
        self.assertEqual(response.status_code, 302)


text:
package org.scalaide.debug.internal.expression

import org.junit.runner.RunWith
import org.junit.runners.Suite
import org.scalaide.debug.internal.ui.completion.SimpleContentProposalProviderTest
import org.scalaide.debug.internal.ui.completion.SimpleContentProposalProviderIntegrationTest
import org.scalaide.debug.internal.expression.proxies.primitives.PrimitivesOperationsTestSuite
import org.scalaide.debug.internal.expression.conditional.ConditionalBreakpointsTest
import org.scalaide.debug.internal.expression.features.FeaturesTestSuite
import org.scalaide.debug.internal.expression.proxies.phases.PhasesTestSuite

@RunWith(classOf[Suite])
@Suite.SuiteClasses(Array(
  classOf[ExpressionManagerTest],
  classOf[FeaturesTestSuite],
  classOf[PrimitivesOperationsTestSuite],
  classOf[DifferentStackFramesTest],
  classOf[PhasesTestSuite],
  classOf[ConditionalBreakpointsTest],
  classOf[SimpleContentProposalProviderTest],
  classOf[SimpleContentProposalProviderIntegrationTest]
))
class ExpressionEvaluatorTestSuite