text:
/* eslint no-undefined: 0, prefer-spread: 0, no-control-regex: 0 */

'use strict';

const crypto = require('crypto');
const os = require('os');
const fs = require('fs');
const punycode = require('punycode');
const PassThrough = require('stream').PassThrough;
const shared = require('../shared');

const mimeFuncs = require('../mime-funcs');
const qp = require('../qp');
const base64 = require('../base64');
const addressparser = require('../addressparser');
const fetch = require('../fetch');
const LastNewline = require('./last-newline');

/**
 * Creates a new mime tree node. Assumes 'multipart/*' as the content type
 * if it is a branch, anything else counts as leaf. If rootNode is missing from
 * the options, assumes this is the root.
 *
 * @param {String} contentType Define the content type for the node. Can be left blank for attachments (derived from filename)
 * @param {Object} [options] optional options
 * @param {Object} [options.rootNode] root node for this tree
 * @param {Object} [options.parentNode] immediate parent for this node
 * @param {Object} [options.filename] filename for an attachment node
 * @param {String} [options.baseBoundary] shared part of the unique multipart boundary
 * @param {Boolean} [options.keepBcc] If true, do not exclude Bcc from the generated headers
 * @param {Function} [options.normalizeHeaderKey] method to normalize header keys for custom caseing
 * @param {String} [options.textEncoding] either 'Q' (the default) or 'B'
 */
class MimeNode {
    constructor(contentType, options) {
        this.nodeCounter = 0;

        options = options || {};

        /**
         * shared part of the unique multipart boundary
         */
        this.baseBoundary = options.baseBoundary || crypto.randomBytes(8).toString('hex');
        this.boundaryPrefix = options.boundaryPrefix || '--_NmP';

        this.disableFileAccess = !!options.disableFileAccess;
        this.disableUrlAccess = !!options.disableUrlAccess;

        this.normalizeHeaderKey = options.normalizeHeaderKey;

        /**
         * If date headers is missing and current node is the root, this value is used instead
         */
        this.date = new Date();

        /**
         * Root node for current mime tree
         */
        this.rootNode = options.rootNode || this;

        /**
         * If true include Bcc in generated headers (if available)
         */
        this.keepBcc = !!options.keepBcc;

        /**
         * If filename is specified but contentType is not (probably an attachment)
         * detect the content type from filename extension
         */
        if (options.filename) {
            /**
             * Filename for this node. Useful with attachments
             */
            this.filename = options.filename;
            if (!contentType) {
                contentType = mimeFuncs.detectMimeType(this.filename.split('.').pop());
            }
        }

        /**
         * Indicates which encoding should be used for header strings: "Q" or "B"
         */
        this.textEncoding = (options.textEncoding || '').toString().trim().charAt(0).toUpperCase();

        /**
         * Immediate parent for this node (or undefined if not set)
         */
        this.parentNode = options.parentNode;

        /**
         * Hostname for default message-id values
         */
        this.hostname = options.hostname;

        /**
         * An array for possible child nodes
         */
        this.childNodes = [];

        /**
         * Used for generating unique boundaries (prepended to the shared base)
         */
        this._nodeId = ++this.rootNode.nodeCounter;

        /**
         * A list of header values for this node in the form of [{key:'', value:''}]
         */
        this._headers = [];

        /**
         * True if the content only uses ASCII printable characters
         * @type {Boolean}
         */
        this._isPlainText = false;

        /**
         * True if the content is plain text but has longer lines than allowed
         * @type {Boolean}
         */
        this._hasLongLines = false;

        /**
         * If set, use instead this value for envelopes instead of generating one
         * @type {Boolean}
         */
        this._envelope = false;

        /**
         * If set then use this value as the stream content instead of building it
         * @type {String|Buffer|Stream}
         */
        this._raw = false;

        /**
         * Additional transform streams that the message will be piped before
         * exposing by createReadStream
         * @type {Array}
         */
        this._transforms = [];

        /**
         * Additional process functions that the message will be piped through before
         * exposing by createReadStream. These functions are run after transforms
         * @type {Array}
         */
        this._processFuncs = [];

        /**
         * If content type is set (or derived from the filename) add it to headers
         */
        if (contentType) {
            this.setHeader('Content-Type', contentType);
        }
    }

    /////// PUBLIC METHODS

    /**
     * Creates and appends a child node.Arguments provided are passed to MimeNode constructor
     *
     * @param {String} [contentType] Optional content type
     * @param {Object} [options] Optional options object
     * @return {Object} Created node object
     */
    createChild(contentType, options) {
        if (!options && typeof contentType === 'object') {
            options = contentType;
            contentType = undefined;
        }
        let node = new MimeNode(contentType, options);
        this.appendChild(node);
        return node;
    }

    /**
     * Appends an existing node to the mime tree. Removes the node from an existing
     * tree if needed
     *
     * @param {Object} childNode node to be appended
     * @return {Object} Appended node object
     */
    appendChild(childNode) {
        if (childNode.rootNode !== this.rootNode) {
            childNode.rootNode = this.rootNode;
            childNode._nodeId = ++this.rootNode.nodeCounter;
        }

        childNode.parentNode = this;

        this.childNodes.push(childNode);
        return childNode;
    }

    /**
     * Replaces current node with another node
     *
     * @param {Object} node Replacement node
     * @return {Object} Replacement node
     */
    replace(node) {
        if (node === this) {
            return this;
        }

        this.parentNode.childNodes.forEach((childNode, i) => {
            if (childNode === this) {
                node.rootNode = this.rootNode;
                node.parentNode = this.parentNode;
                node._nodeId = this._nodeId;

                this.rootNode = this;
                this.parentNode = undefined;

                node.parentNode.childNodes[i] = node;
            }
        });

        return node;
    }

    /**
     * Removes current node from the mime tree
     *
     * @return {Object} removed node
     */
    remove() {
        if (!this.parentNode) {
            return this;
        }

        for (let i = this.parentNode.childNodes.length - 1; i >= 0; i--) {
            if (this.parentNode.childNodes[i] === this) {
                this.parentNode.childNodes.splice(i, 1);
                this.parentNode = undefined;
                this.rootNode = this;
                return this;
            }
        }
    }

    /**
     * Sets a header value. If the value for selected key exists, it is overwritten.
     * You can set multiple values as well by using [{key:'', value:''}] or
     * {key: 'value'} as the first argument.
     *
     * @param {String|Array|Object} key Header key or a list of key value pairs
     * @param {String} value Header value
     * @return {Object} current node
     */
    setHeader(key, value) {
        let added = false,
            headerValue;

        // Allow setting multiple headers at once
        if (!value && key && typeof key === 'object') {
            // allow {key:'content-type', value: 'text/plain'}
            if (key.key && 'value' in key) {
                this.setHeader(key.key, key.value);
            } else if (Array.isArray(key)) {
                // allow [{key:'content-type', value: 'text/plain'}]
                key.forEach(i => {
                    this.setHeader(i.key, i.value);
                });
            } else {
                // allow {'content-type': 'text/plain'}
                Object.keys(key).forEach(i => {
                    this.setHeader(i, key[i]);
                });
            }
            return this;
        }

        key = this._normalizeHeaderKey(key);

        headerValue = {
            key,
            value
        };

        // Check if the value exists and overwrite
        for (let i = 0, len = this._headers.length; i < len; i++) {
            if (this._headers[i].key === key) {
                if (!added) {
                    // replace the first match
                    this._headers[i] = headerValue;
                    added = true;
                } else {
                    // remove following matches
                    this._headers.splice(i, 1);
                    i--;
                    len--;
                }
            }
        }

        // match not found, append the value
        if (!added) {
            this._headers.push(headerValue);
        }

        return this;
    }

    /**
     * Adds a header value. If the value for selected key exists, the value is appended
     * as a new field and old one is not touched.
     * You can set multiple values as well by using [{key:'', value:''}] or
     * {key: 'value'} as the first argument.
     *
     * @param {String|Array|Object} key Header key or a list of key value pairs
     * @param {String} value Header value
     * @return {Object} current node
     */
    addHeader(key, value) {
        // Allow setting multiple headers at once
        if (!value && key && typeof key === 'object') {
            // allow {key:'content-type', value: 'text/plain'}
            if (key.key && key.value) {
                this.addHeader(key.key, key.value);
            } else if (Array.isArray(key)) {
                // allow [{key:'content-type', value: 'text/plain'}]
                key.forEach(i => {
                    this.addHeader(i.key, i.value);
                });
            } else {
                // allow {'content-type': 'text/plain'}
                Object.keys(key).forEach(i => {
                    this.addHeader(i, key[i]);
                });
            }
            return this;
        } else if (Array.isArray(value)) {
            value.forEach(val => {
                this.addHeader(key, val);
            });
            return this;
        }

        this._headers.push({
            key: this._normalizeHeaderKey(key),
            value
        });

        return this;
    }

    /**
     * Retrieves the first mathcing value of a selected key
     *
     * @param {String} key Key to search for
     * @retun {String} Value for the key
     */
    getHeader(key) {
        key = this._normalizeHeaderKey(key);
        for (let i = 0, len = this._headers.length; i < len; i++) {
            if (this._headers[i].key === key) {
                return this._headers[i].value;
            }
        }
    }

    /**
     * Sets body content for current node. If the value is a string, charset is added automatically
     * to Content-Type (if it is text/*). If the value is a Buffer, you need to specify
     * the charset yourself
     *
     * @param (String|Buffer) content Body content
     * @return {Object} current node
     */
    setContent(content) {
        this.content = content;
        if (typeof this.content.pipe === 'function') {
            // pre-stream handler. might be triggered if a stream is set as content
            // and 'error' fires before anything is done with this stream
            this._contentErrorHandler = err => {
                this.content.removeListener('error', this._contentErrorHandler);
                this.content = err;
            };
            this.content.once('error', this._contentErrorHandler);
        } else if (typeof this.content === 'string') {
            this._isPlainText = mimeFuncs.isPlainText(this.content);
            if (this._isPlainText && mimeFuncs.hasLongerLines(this.content, 76)) {
                // If there are lines longer than 76 symbols/bytes do not use 7bit
                this._hasLongLines = true;
            }
        }
        return this;
    }

    build(callback) {
        let promise;

        if (!callback) {
            promise = new Promise((resolve, reject) => {
                callback = shared.callbackPromise(resolve, reject);
            });
        }

        let stream = this.createReadStream();
        let buf = [];
        let buflen = 0;
        let returned = false;

        stream.on('readable', () => {
            let chunk;

            while ((chunk = stream.read()) !== null) {
                buf.push(chunk);
                buflen += chunk.length;
            }
        });

        stream.once('error', err => {
            if (returned) {
                return;
            }
            returned = true;

            return callback(err);
        });

        stream.once('end', chunk => {
            if (returned) {
                return;
            }
            returned = true;

            if (chunk && chunk.length) {
                buf.push(chunk);
                buflen += chunk.length;
            }
            return callback(null, Buffer.concat(buf, buflen));
        });

        return promise;
    }

    getTransferEncoding() {
        let transferEncoding = false;
        let contentType = (this.getHeader('Content-Type') || '').toString().toLowerCase().trim();

        if (this.content) {
            transferEncoding = (this.getHeader('Content-Transfer-Encoding') || '').toString().toLowerCase().trim();
            if (!transferEncoding || !['base64', 'quoted-printable'].includes(transferEncoding)) {
                if (/^text\//i.test(contentType)) {
                    // If there are no special symbols, no need to modify the text
                    if (this._isPlainText && !this._hasLongLines) {
                        transferEncoding = '7bit';
                    } else if (typeof this.content === 'string' || this.content instanceof Buffer) {
                        // detect preferred encoding for string value
                        transferEncoding = this._getTextEncoding(this.content) === 'Q' ? 'quoted-printable' : 'base64';
                    } else {
                        // we can not check content for a stream, so either use preferred encoding or fallback to QP
                        transferEncoding = this.transferEncoding === 'B' ? 'base64' : 'quoted-printable';
                    }
                } else if (!/^(multipart|message)\//i.test(contentType)) {
                    transferEncoding = transferEncoding || 'base64';
                }
            }
        }
        return transferEncoding;
    }

    /**
     * Builds the header block for the mime node. Append \r\n\r\n before writing the content
     *
     * @returns {String} Headers
     */
    buildHeaders() {
        let transferEncoding = this.getTransferEncoding();
        let headers = [];

        if (transferEncoding) {
            this.setHeader('Content-Transfer-Encoding', transferEncoding);
        }

        if (this.filename && !this.getHeader('Content-Disposition')) {
            this.setHeader('Content-Disposition', 'attachment');
        }

        // Ensure mandatory header fields
        if (this.rootNode === this) {
            if (!this.getHeader('Date')) {
                this.setHeader('Date', this.date.toUTCString().replace(/GMT/, '+0000'));
            }

            // ensure that Message-Id is present
            this.messageId();

            if (!this.getHeader('MIME-Version')) {
                this.setHeader('MIME-Version', '1.0');
            }
        }

        this._headers.forEach(header => {
            let key = header.key;
            let value = header.value;
            let structured;
            let param;
            let options = {};
            let formattedHeaders = ['From', 'Sender', 'To', 'Cc', 'Bcc', 'Reply-To', 'Date', 'References'];

            if (value && typeof value === 'object' && !formattedHeaders.includes(key)) {
                Object.keys(value).forEach(key => {
                    if (key !== 'value') {
                        options[key] = value[key];
                    }
                });
                value = (value.value || '').toString();
                if (!value.trim()) {
                    return;
                }
            }

            if (options.prepared) {
                // header value is
                if (options.foldLines) {
                    headers.push(mimeFuncs.foldLines(key + ': ' + value));
                } else {
                    headers.push(key + ': ' + value);
                }
                return;
            }

            switch (header.key) {
                case 'Content-Disposition':
                    structured = mimeFuncs.parseHeaderValue(value);
                    if (this.filename) {
                        structured.params.filename = this.filename;
                    }
                    value = mimeFuncs.buildHeaderValue(structured);
                    break;

                case 'Content-Type':
                    structured = mimeFuncs.parseHeaderValue(value);

                    this._handleContentType(structured);

                    if (structured.value.match(/^text\/plain\b/) && typeof this.content === 'string' && /[\u0080-\uFFFF]/.test(this.content)) {
                        structured.params.charset = 'utf-8';
                    }

                    value = mimeFuncs.buildHeaderValue(structured);

                    if (this.filename) {
                        // add support for non-compliant clients like QQ webmail
                        // we can't build the value with buildHeaderValue as the value is non standard and
                        // would be converted to parameter continuation encoding that we do not want
                        param = this._encodeWords(this.filename);

                        if (param !== this.filename || /[\s'"\\;:/=(),<>@[\]?]|^-/.test(param)) {
                            // include value in quotes if needed
                            param = '"' + param + '"';
                        }
                        value += '; name=' + param;
                    }
                    break;

                case 'Bcc':
                    if (!this.keepBcc) {
                        // skip BCC values
                        return;
                    }
                    break;
            }

            value = this._encodeHeaderValue(key, value);

            // skip empty lines
            if (!(value || '').toString().trim()) {
                return;
            }

            if (typeof this.normalizeHeaderKey === 'function') {
                let normalized = this.normalizeHeaderKey(key, value);
                if (normalized && typeof normalized === 'string' && normalized.length) {
                    key = normalized;
                }
            }

            headers.push(mimeFuncs.foldLines(key + ': ' + value, 76));
        });

        return headers.join('\r\n');
    }

    /**
     * Streams the rfc2822 message from the current node. If this is a root node,
     * mandatory header fields are set if missing (Date, Message-Id, MIME-Version)
     *
     * @return {String} Compiled message
     */
    createReadStream(options) {
        options = options || {};

        let stream = new PassThrough(options);
        let outputStream = stream;
        let transform;

        this.stream(stream, options, err => {
            if (err) {
                outputStream.emit('error', err);
                return;
            }
            stream.end();
        });

        for (let i = 0, len = this._transforms.length; i < len; i++) {
            transform = typeof this._transforms[i] === 'function' ? this._transforms[i]() : this._transforms[i];
            outputStream.once('error', err => {
                transform.emit('error', err);
            });
            outputStream = outputStream.pipe(transform);
        }

        // ensure terminating newline after possible user transforms
        transform = new LastNewline();
        outputStream.once('error', err => {
            transform.emit('error', err);
        });
        outputStream = outputStream.pipe(transform);

        // dkim and stuff
        for (let i = 0, len = this._processFuncs.length; i < len; i++) {
            transform = this._processFuncs[i];
            outputStream = transform(outputStream);
        }

        return outputStream;
    }

    /**
     * Appends a transform stream object to the transforms list. Final output
     * is passed through this stream before exposing
     *
     * @param {Object} transform Read-Write stream
     */
    transform(transform) {
        this._transforms.push(transform);
    }

    /**
     * Appends a post process function. The functon is run after transforms and
     * uses the following syntax
     *
     *   processFunc(input) -> outputStream
     *
     * @param {Object} processFunc Read-Write stream
     */
    processFunc(processFunc) {
        this._processFuncs.push(processFunc);
    }

    stream(outputStream, options, done) {
        let transferEncoding = this.getTransferEncoding();
        let contentStream;
        let localStream;

        // protect actual callback against multiple triggering
        let returned = false;
        let callback = err => {
            if (returned) {
                return;
            }
            returned = true;
            done(err);
        };

        // for multipart nodes, push child nodes
        // for content nodes end the stream
        let finalize = () => {
            let childId = 0;
            let processChildNode = () => {
                if (childId >= this.childNodes.length) {
                    outputStream.write('\r\n--' + this.boundary + '--\r\n');
                    return callback();
                }
                let child = this.childNodes[childId++];
                outputStream.write((childId > 1 ? '\r\n' : '') + '--' + this.boundary + '\r\n');
                child.stream(outputStream, options, err => {
                    if (err) {
                        return callback(err);
                    }
                    setImmediate(processChildNode);
                });
            };

            if (this.multipart) {
                setImmediate(processChildNode);
            } else {
                return callback();
            }
        };

        // pushes node content
        let sendContent = () => {
            if (this.content) {
                if (Object.prototype.toString.call(this.content) === '[object Error]') {
                    // content is already errored
                    return callback(this.content);
                }

                if (typeof this.content.pipe === 'function') {
                    this.content.removeListener('error', this._contentErrorHandler);
                    this._contentErrorHandler = err => callback(err);
                    this.content.once('error', this._contentErrorHandler);
                }

                let createStream = () => {
                    if (['quoted-printable', 'base64'].includes(transferEncoding)) {
                        contentStream = new (transferEncoding === 'base64' ? base64 : qp).Encoder(options);

                        contentStream.pipe(outputStream, {
                            end: false
                        });
                        contentStream.once('end', finalize);
                        contentStream.once('error', err => callback(err));

                        localStream = this._getStream(this.content);
                        localStream.pipe(contentStream);
                    } else {
                        // anything that is not QP or Base54 passes as-is
                        localStream = this._getStream(this.content);
                        localStream.pipe(outputStream, {
                            end: false
                        });
                        localStream.once('end', finalize);
                    }

                    localStream.once('error', err => callback(err));
                };

                if (this.content._resolve) {
                    let chunks = [];
                    let chunklen = 0;
                    let returned = false;
                    let sourceStream = this._getStream(this.content);
                    sourceStream.on('error', err => {
                        if (returned) {
                            return;
                        }
                        returned = true;
                        callback(err);
                    });
                    sourceStream.on('readable', () => {
                        let chunk;
                        while ((chunk = sourceStream.read()) !== null) {
                            chunks.push(chunk);
                            chunklen += chunk.length;
                        }
                    });
                    sourceStream.on('end', () => {
                        if (returned) {
                            return;
                        }
                        returned = true;
                        this.content._resolve = false;
                        this.content._resolvedValue = Buffer.concat(chunks, chunklen);
                        setImmediate(createStream);
                    });
                } else {
                    setImmediate(createStream);
                }
                return;
            } else {
                return setImmediate(finalize);
            }
        };

        if (this._raw) {
            setImmediate(() => {
                if (Object.prototype.toString.call(this._raw) === '[object Error]') {
                    // content is already errored
                    return callback(this._raw);
                }

                // remove default error handler (if set)
                if (typeof this._raw.pipe === 'function') {
                    this._raw.removeListener('error', this._contentErrorHandler);
                }

                let raw = this._getStream(this._raw);
                raw.pipe(outputStream, {
                    end: false
                });
                raw.on('error', err => outputStream.emit('error', err));
                raw.on('end', finalize);
            });
        } else {
            outputStream.write(this.buildHeaders() + '\r\n\r\n');
            setImmediate(sendContent);
        }
    }

    /**
     * Sets envelope to be used instead of the generated one
     *
     * @return {Object} SMTP envelope in the form of {from: 'from@example.com', to: ['to@example.com']}
     */
    setEnvelope(envelope) {
        let list;

        this._envelope = {
            from: false,
            to: []
        };

        if (envelope.from) {
            list = [];
            this._convertAddresses(this._parseAddresses(envelope.from), list);
            list = list.filter(address => address && address.address);
            if (list.length && list[0]) {
                this._envelope.from = list[0].address;
            }
        }
        ['to', 'cc', 'bcc'].forEach(key => {
            if (envelope[key]) {
                this._convertAddresses(this._parseAddresses(envelope[key]), this._envelope.to);
            }
        });

        this._envelope.to = this._envelope.to.map(to => to.address).filter(address => address);

        let standardFields = ['to', 'cc', 'bcc', 'from'];
        Object.keys(envelope).forEach(key => {
            if (!standardFields.includes(key)) {
                this._envelope[key] = envelope[key];
            }
        });

        return this;
    }

    /**
     * Generates and returns an object with parsed address fields
     *
     * @return {Object} Address object
     */
    getAddresses() {
        let addresses = {};

        this._headers.forEach(header => {
            let key = header.key.toLowerCase();
            if (['from', 'sender', 'reply-to', 'to', 'cc', 'bcc'].includes(key)) {
                if (!Array.isArray(addresses[key])) {
                    addresses[key] = [];
                }

                this._convertAddresses(this._parseAddresses(header.value), addresses[key]);
            }
        });

        return addresses;
    }

    /**
     * Generates and returns SMTP envelope with the sender address and a list of recipients addresses
     *
     * @return {Object} SMTP envelope in the form of {from: 'from@example.com', to: ['to@example.com']}
     */
    getEnvelope() {
        if (this._envelope) {
            return this._envelope;
        }

        let envelope = {
            from: false,
            to: []
        };
        this._headers.forEach(header => {
            let list = [];
            if (header.key === 'From' || (!envelope.from && ['Reply-To', 'Sender'].includes(header.key))) {
                this._convertAddresses(this._parseAddresses(header.value), list);
                if (list.length && list[0]) {
                    envelope.from = list[0].address;
                }
            } else if (['To', 'Cc', 'Bcc'].includes(header.key)) {
                this._convertAddresses(this._parseAddresses(header.value), envelope.to);
            }
        });

        envelope.to = envelope.to.map(to => to.address);

        return envelope;
    }

    /**
     * Returns Message-Id value. If it does not exist, then creates one
     *
     * @return {String} Message-Id value
     */
    messageId() {
        let messageId = this.getHeader('Message-ID');
        // You really should define your own Message-Id field!
        if (!messageId) {
            messageId = this._generateMessageId();
            this.setHeader('Message-ID', messageId);
        }
        return messageId;
    }

    /**
     * Sets pregenerated content that will be used as the output of this node
     *
     * @param {String|Buffer|Stream} Raw MIME contents
     */
    setRaw(raw) {
        this._raw = raw;

        if (this._raw && typeof this._raw.pipe === 'function') {
            // pre-stream handler. might be triggered if a stream is set as content
            // and 'error' fires before anything is done with this stream
            this._contentErrorHandler = err => {
                this._raw.removeListener('error', this._contentErrorHandler);
                this._raw = err;
            };
            this._raw.once('error', this._contentErrorHandler);
        }

        return this;
    }

    /////// PRIVATE METHODS

    /**
     * Detects and returns handle to a stream related with the content.
     *
     * @param {Mixed} content Node content
     * @returns {Object} Stream object
     */
    _getStream(content) {
        let contentStream;

        if (content._resolvedValue) {
            // pass string or buffer content as a stream
            contentStream = new PassThrough();
            setImmediate(() => contentStream.end(content._resolvedValue));
            return contentStream;
        } else if (typeof content.pipe === 'function') {
            // assume as stream
            return content;
        } else if (content && typeof content.path === 'string' && !content.href) {
            if (this.disableFileAccess) {
                contentStream = new PassThrough();
                setImmediate(() => contentStream.emit('error', new Error('File access rejected for ' + content.path)));
                return contentStream;
            }
            // read file
            return fs.createReadStream(content.path);
        } else if (content && typeof content.href === 'string') {
            if (this.disableUrlAccess) {
                contentStream = new PassThrough();
                setImmediate(() => contentStream.emit('error', new Error('Url access rejected for ' + content.href)));
                return contentStream;
            }
            // fetch URL
            return fetch(content.href, { headers: content.httpHeaders });
        } else {
            // pass string or buffer content as a stream
            contentStream = new PassThrough();
            setImmediate(() => contentStream.end(content || ''));
            return contentStream;
        }
    }

    /**
     * Parses addresses. Takes in a single address or an array or an
     * array of address arrays (eg. To: [[first group], [second group],...])
     *
     * @param {Mixed} addresses Addresses to be parsed
     * @return {Array} An array of address objects
     */
    _parseAddresses(addresses) {
        return [].concat.apply(
            [],
            [].concat(addresses).map(address => {
                // eslint-disable-line prefer-spread
                if (address && address.address) {
                    address.address = this._normalizeAddress(address.address);
                    address.name = address.name || '';
                    return [address];
                }
                return addressparser(address);
            })
        );
    }

    /**
     * Normalizes a header key, uses Camel-Case form, except for uppercase MIME-
     *
     * @param {String} key Key to be normalized
     * @return {String} key in Camel-Case form
     */
    _normalizeHeaderKey(key) {
        key = (key || '')
            .toString()
            // no newlines in keys
            .replace(/\r?\n|\r/g, ' ')
            .trim()
            .toLowerCase()
            // use uppercase words, except MIME
            .replace(/^X-SMTPAPI$|^(MIME|DKIM|ARC|BIMI)\b|^[a-z]|-(SPF|FBL|ID|MD5)$|-[a-z]/gi, c => c.toUpperCase())
            // special case
            .replace(/^Content-Features$/i, 'Content-features');

        return key;
    }

    /**
     * Checks if the content type is multipart and defines boundary if needed.
     * Doesn't return anything, modifies object argument instead.
     *
     * @param {Object} structured Parsed header value for 'Content-Type' key
     */
    _handleContentType(structured) {
        this.contentType = structured.value.trim().toLowerCase();

        this.multipart = /^multipart\//i.test(this.contentType) ? this.contentType.substr(this.contentType.indexOf('/') + 1) : false;

        if (this.multipart) {
            this.boundary = structured.params.boundary = structured.params.boundary || this.boundary || this._generateBoundary();
        } else {
            this.boundary = false;
        }
    }

    /**
     * Generates a multipart boundary value
     *
     * @return {String} boundary value
     */
    _generateBoundary() {
        return this.rootNode.boundaryPrefix + '-' + this.rootNode.baseBoundary + '-Part_' + this._nodeId;
    }

    /**
     * Encodes a header value for use in the generated rfc2822 email.
     *
     * @param {String} key Header key
     * @param {String} value Header value
     */
    _encodeHeaderValue(key, value) {
        key = this._normalizeHeaderKey(key);

        switch (key) {
            // Structured headers
            case 'From':
            case 'Sender':
            case 'To':
            case 'Cc':
            case 'Bcc':
            case 'Reply-To':
                return this._convertAddresses(this._parseAddresses(value));

            // values enclosed in <>
            case 'Message-ID':
            case 'In-Reply-To':
            case 'Content-Id':
                value = (value || '').toString().replace(/\r?\n|\r/g, ' ');

                if (value.charAt(0) !== '<') {
                    value = '<' + value;
                }

                if (value.charAt(value.length - 1) !== '>') {
                    value = value + '>';
                }
                return value;

            // space separated list of values enclosed in <>
            case 'References':
                value = [].concat
                    .apply(
                        [],
                        [].concat(value || '').map(elm => {
                            // eslint-disable-line prefer-spread
                            elm = (elm || '')
                                .toString()
                                .replace(/\r?\n|\r/g, ' ')
                                .trim();
                            return elm.replace(/<[^>]*>/g, str => str.replace(/\s/g, '')).split(/\s+/);
                        })
                    )
                    .map(elm => {
                        if (elm.charAt(0) !== '<') {
                            elm = '<' + elm;
                        }
                        if (elm.charAt(elm.length - 1) !== '>') {
                            elm = elm + '>';
                        }
                        return elm;
                    });

                return value.join(' ').trim();

            case 'Date':
                if (Object.prototype.toString.call(value) === '[object Date]') {
                    return value.toUTCString().replace(/GMT/, '+0000');
                }

                value = (value || '').toString().replace(/\r?\n|\r/g, ' ');
                return this._encodeWords(value);

            case 'Content-Type':
            case 'Content-Disposition':
                // if it includes a filename then it is already encoded
                return (value || '').toString().replace(/\r?\n|\r/g, ' ');

            default:
                value = (value || '').toString().replace(/\r?\n|\r/g, ' ');
                // encodeWords only encodes if needed, otherwise the original string is returned
                return this._encodeWords(value);
        }
    }

    /**
     * Rebuilds address object using punycode and other adjustments
     *
     * @param {Array} addresses An array of address objects
     * @param {Array} [uniqueList] An array to be populated with addresses
     * @return {String} address string
     */
    _convertAddresses(addresses, uniqueList) {
        let values = [];

        uniqueList = uniqueList || [];

        [].concat(addresses || []).forEach(address => {
            if (address.address) {
                address.address = this._normalizeAddress(address.address);

                if (!address.name) {
                    values.push(address.address);
                } else if (address.name) {
                    values.push(this._encodeAddressName(address.name) + ' <' + address.address + '>');
                }

                if (address.address) {
                    if (!uniqueList.filter(a => a.address === address.address).length) {
                        uniqueList.push(address);
                    }
                }
            } else if (address.group) {
                values.push(
                    this._encodeAddressName(address.name) + ':' + (address.group.length ? this._convertAddresses(address.group, uniqueList) : '').trim() + ';'
                );
            }
        });

        return values.join(', ');
    }

    /**
     * Normalizes an email address
     *
     * @param {Array} address An array of address objects
     * @return {String} address string
     */
    _normalizeAddress(address) {
        address = (address || '').toString().trim();

        let lastAt = address.lastIndexOf('@');
        if (lastAt < 0) {
            // Bare username
            return address;
        }
        let user = address.substr(0, lastAt);
        let domain = address.substr(lastAt + 1);

        // Usernames are not touched and are kept as is even if these include unicode
        // Domains are punycoded by default
        // 'jõgeva.ee' will be converted to 'xn--jgeva-dua.ee'
        // non-unicode domains are left as is

        return user + '@' + punycode.toASCII(domain.toLowerCase());
    }

    /**
     * If needed, mime encodes the name part
     *
     * @param {String} name Name part of an address
     * @returns {String} Mime word encoded string if needed
     */
    _encodeAddressName(name) {
        if (!/^[\w ']*$/.test(name)) {
            if (/^[\x20-\x7e]*$/.test(name)) {
                return '"' + name.replace(/([\\"])/g, '\\$1') + '"';
            } else {
                return mimeFuncs.encodeWord(name, this._getTextEncoding(name), 52);
            }
        }
        return name;
    }

    /**
     * If needed, mime encodes the name part
     *
     * @param {String} name Name part of an address
     * @returns {String} Mime word encoded string if needed
     */
    _encodeWords(value) {
        // set encodeAll parameter to true even though it is against the recommendation of RFC2047,
        // by default only words that include non-ascii should be converted into encoded words
        // but some clients (eg. Zimbra) do not handle it properly and remove surrounding whitespace
        return mimeFuncs.encodeWords(value, this._getTextEncoding(value), 52, true);
    }

    /**
     * Detects best mime encoding for a text value
     *
     * @param {String} value Value to check for
     * @return {String} either 'Q' or 'B'
     */
    _getTextEncoding(value) {
        value = (value || '').toString();

        let encoding = this.textEncoding;
        let latinLen;
        let nonLatinLen;

        if (!encoding) {
            // count latin alphabet symbols and 8-bit range symbols + control symbols
            // if there are more latin characters, then use quoted-printable
            // encoding, otherwise use base64
            nonLatinLen = (value.match(/[\x00-\x08\x0B\x0C\x0E-\x1F\u0080-\uFFFF]/g) || []).length; // eslint-disable-line no-control-regex
            latinLen = (value.match(/[a-z]/gi) || []).length;
            // if there are more latin symbols than binary/unicode, then prefer Q, otherwise B
            encoding = nonLatinLen < latinLen ? 'Q' : 'B';
        }
        return encoding;
    }

    /**
     * Generates a message id
     *
     * @return {String} Random Message-ID value
     */
    _generateMessageId() {
        return (
            '<' +
            [2, 2, 2, 6].reduce(
                // crux to generate UUID-like random strings
                (prev, len) => prev + '-' + crypto.randomBytes(len).toString('hex'),
                crypto.randomBytes(4).toString('hex')
            ) +
            '@' +
            // try to use the domain of the FROM address or fallback to server hostname
            (this.getEnvelope().from || this.hostname || os.hostname() || 'localhost').split('@').pop() +
            '>'
        );
    }
}

module.exports = MimeNode;


text:
<?php
/**
 * PHP version 7.1
 *
 * This source file is subject to the license that is bundled with this package in the file LICENSE.
 */
namespace App\Model\Gateway;

use App\Model\Dao\CityDao;
use App\Model\Gateway\City as CityGateway;
use ControllerTestCase;
use Edeco\Fixtures\CitiesFixture;
use Mandragora\PHPUnit\DoctrineTest\DoctrineTestInterface;

class CityTest extends ControllerTestCase implements DoctrineTestInterface
{
    /** @test */
    function it_retrieves_all_cities_of_a_given_state()
    {
        $existingCities = $this->fixture->cities();

        $allCities = $this->cityGateway->findAllByStateId($this->fixture->stateId());

        $this->assertCount(5, $allCities);
        $i = 0;
        foreach ($existingCities as $existingCity) {
            $this->assertEquals($existingCity['name'], $allCities[$i]['name']);
            $i++;
        }
    }

    /** @test */
    function it_retrieves_zero_results_with_a_non_existing_state()
    {
        $this->assertCount(0, $this->cityGateway->findAllByStateId(-1));
    }

    /** @before */
    function createStateGateway(): void
    {
        /** @var \Mandragora\Application\Doctrine\Manager $manager */
        $manager = $this->_frontController->getParam('bootstrap')->getResource('doctrine');
        $this->fixture = CitiesFixture::fromDSN($manager->getConfiguration()['dsn']);
        $this->cityGateway = new CityGateway(new CityDao());
    }

    /** @var CityGateway */
    private $cityGateway;

    /** @var CitiesFixture */
    private $fixture;
}

redpajama_set_name: RedPajamaGithub


text:
SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL';

ALTER TABLE `voota`.`convocatoria`
DROP INDEX `nombre_UNIQUE`
, ADD UNIQUE INDEX `nombre_UNIQUE` (`eleccion_id` ASC, `nombre` ASC) ;

ALTER TABLE `voota`.`eleccion` DROP COLUMN `vanity` , ADD COLUMN `imagen` VARCHAR(50) NULL DEFAULT NULL  AFTER `created_at` , ADD COLUMN `vanity` VARCHAR(45) NOT NULL  AFTER `id`
, DROP INDEX `uniq_eleccion`
, ADD UNIQUE INDEX `uniq_eleccion` (`vanity` ASC) ;

ALTER TABLE `voota`.`enlace` ADD COLUMN `eleccion_id` INT(11) NULL DEFAULT NULL  AFTER `propuesta_id` ,
  ADD CONSTRAINT `fk_enlace_3`
  FOREIGN KEY (`eleccion_id` )
  REFERENCES `voota`.`eleccion` (`id` )
  ON DELETE NO ACTION
  ON UPDATE NO ACTION
, ADD INDEX `fk_enlace_3` (`eleccion_id` ASC) ;

SET SQL_MODE=@OLD_SQL_MODE;
SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;
SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;


text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

package com.azure.communication.jobrouter.implementation.convertors;

import com.azure.communication.jobrouter.models.DistributionPolicy;
import com.azure.communication.jobrouter.models.options.CreateDistributionPolicyOptions;
import com.azure.communication.jobrouter.models.options.UpdateDistributionPolicyOptions;

/**
 * Converts request options for create and update Classification Policy to {@link DistributionPolicy}.
 */
public class DistributionPolicyAdapter {
    /**
     * Converts {@link CreateDistributionPolicyOptions} to {@link DistributionPolicy}.
     * @param createDistributionPolicyOptions Container with options to create a DistributionPolicy.
     * @return distribution policy.
     */
    public static DistributionPolicy convertCreateOptionsToDistributionPolicy(CreateDistributionPolicyOptions createDistributionPolicyOptions) {
        return new DistributionPolicy()
            .setMode(createDistributionPolicyOptions.getMode())
            .setOfferTtlSeconds(Long.valueOf(createDistributionPolicyOptions.getOfferTtl().getSeconds()).doubleValue())
            .setName(createDistributionPolicyOptions.getName());
    }

    /**
     * Converts {@link UpdateDistributionPolicyOptions} to {@link DistributionPolicy}.
     * @param updateDistributionPolicyOptions Container with options to update a DistributionPolicy.
     * @return distribution policy.
     */
    public static DistributionPolicy convertUpdateOptionsToClassificationPolicy(UpdateDistributionPolicyOptions updateDistributionPolicyOptions) {
        return new DistributionPolicy()
            .setMode(updateDistributionPolicyOptions.getMode())
            .setName(updateDistributionPolicyOptions.getName())
            .setOfferTtlSeconds(Long.valueOf(updateDistributionPolicyOptions.getOfferTtl().getSeconds()).doubleValue());
    }
}


text:
{-# LANGUAGE DeriveDataTypeable, ExistentialQuantification #-}
-- | [Process Binder]
--
-- This is a frontend to easily use Cloud Haskell inside other monads.
-- If you have a complex system and want to make it distributed withouth
-- having to deal with monad transformers or complex pipeing to connect
-- Cloud Haskell and the rest of the application, this package can
-- easliy abstract away the hassle by handling the communication
-- with Cloud Haskell inside the IO Monad.
module Control.Distributed.Process.Binder.Types (
  ProcessBinder,
  newProcessQueue,
  runAction,
  newChan
  )where

import Control.Concurrent.STM
import qualified Control.Distributed.Process as P
import Data.Binary (Binary)
import Data.Typeable (Typeable,cast)
import Control.Monad (forever)
import Control.Distributed.Process.Node (LocalNode,forkProcess)

data Holder = forall a. Typeable a => Holder{internalValue :: a}

data ProcessRunOption = RunNonBlocking | RunBlocking

data QueuedProcess = BlockingProcess QueuedProcessBody | NonBlockingProcess QueuedProcessBody

data QueuedProcessBody = QueuedProcessBody{
  queuedAction :: P.Process Holder,
  resultHolder :: TMVar Holder
  }

data ProcessBinder = ProcessBinder{
  binderProcessId :: P.ProcessId,
  processQueue :: TChan QueuedProcess
}

class ProcessRunner runner where
  queueProcess :: runner -> a -> IO ()
  readResult :: runner -> IO a

castHolder :: Typeable a => Holder -> Maybe a
castHolder (Holder val) = cast val

newChan :: (Typeable a, Binary a) => ProcessBinder -> IO (P.SendPort a, P.ReceivePort a)
newChan binder = newChan' Nothing
  where
    newChan' Nothing = runBlockingAction binder P.newChan >>= newChan'
    newChan' (Just chan) = return chan

processWrapper :: Typeable a => P.Process a -> P.Process Holder
processWrapper proc = proc >>= return.Holder

processRunner :: TChan QueuedProcess -> P.Process ()
processRunner chan = forever $ do
  qAction <- P.liftIO $ atomically $ readTChan chan
  case qAction of
    BlockingProcess qAction' -> runBlockingProcess qAction'
    NonBlockingProcess qAction' -> runNonBlockingProcess qAction'

  where
    runNonBlockingProcess qAction = do
      P.spawnLocal $ runBlockingProcess qAction
      return ()
    runBlockingProcess qAction = do
      let
        action = queuedAction qAction
        resultVar = resultHolder qAction
      result <- action
      P.liftIO $ atomically $ putTMVar resultVar result

runAction :: Typeable a => ProcessBinder -> P.Process a -> IO (Maybe a)
runAction = runAction' RunNonBlocking

runBlockingAction :: Typeable a => ProcessBinder -> P.Process a -> IO (Maybe a)
runBlockingAction = runAction' RunBlocking

runAction' :: Typeable a => ProcessRunOption -> ProcessBinder -> P.Process a -> IO (Maybe a)
runAction' runOption binder action = do
  resultVar <- newEmptyTMVarIO
  let
    actionWrapper = QueuedProcessBody{queuedAction=processWrapper action,
                                      resultHolder=resultVar}
    actionRunner = case runOption of
      RunNonBlocking -> NonBlockingProcess actionWrapper
      RunBlocking -> BlockingProcess actionWrapper
  atomically $ writeTChan (processQueue binder) actionRunner
  result <- atomically $ takeTMVar (resultHolder actionWrapper)
  return $ castHolder result

newProcessQueue :: LocalNode -> IO ProcessBinder
newProcessQueue localNode = do
  queue <- newTChanIO
  blockingChan <- newTChanIO
  pid <- forkProcess localNode $ processRunner queue
  return $ ProcessBinder{binderProcessId=pid,processQueue=queue}


text:
if [ "$UID" -ne "0" ]
then
  echo ""
  echo "You must be sudoer or root. To run this script enter:"
  echo ""
  echo "sudo chmod +x $0; sudo ./$0"
  echo ""
  exit 1
fi

sudo apt-get update
sudo apt-get install -y unzip vim git-core curl wget build-essential python-software-properties

#############################################################################
### Generate all the locales to support i18n in PHP5
#############################################################################
sudo cp /usr/share/i18n/SUPPORTED /var/lib/locales/supported.d/local
sudo locale-gen

#############################################################################
# FACEBOOK'S HHVM
#############################################################################
wget -O - http://dl.hhvm.com/conf/hhvm.gpg.key | sudo apt-key add -
echo deb http://dl.hhvm.com/ubuntu precise main | sudo tee /etc/apt/sources.list.d/hhvm.list

sudo apt-get -y update
sudo apt-get install -y --force-yes nginx hhvm hhvm-fastcgi

VAR=$(cat <<'END_HEREDOC'

PidFile = /var/run/hhvm/pid

Server {
  Port = 80
  SourceRoot = /var/www/example.com
  DefaultDocument = index.php
}

Log {
  Level = Warning
  AlwaysLogUnhandledExceptions = true
  RuntimeErrorReportingLevel = 8191
  UseLogFile = true
  UseSyslog = false
  File = /var/log/hhvm/error.log
  InjectedStackTrace = true
  NativeStackTrace = true
  Access {
    * {
      File = /var/log/hhvm/access.log
      Format = %h %l %u % t \"%r\" %>s %b
    }
  }
}

Debug {
 FullBacktrace = true
 ServerStackTrace = true
 ServerErrorMessage = true
 TranslateSource = true
}

Repo {
  Central {
    Path = /var/log/hhvm/.hhvm.hhbc
  }
}

MySQL {
  TypedResults = false
}

Eval {
   Jit = true
}

END_HEREDOC
)

echo "$VAR" > /etc/hhvm/server.hdf

## Stop hhvm and start hhvm-fastcgi
sudo service hhvm stop
sudo service hhvm-fastcgi start

#############################################################################
## Cronjob to make sure hhvm-fastcgi never stops running.
#############################################################################
echo '* * * * * if [ $(ps aux | grep hhvm | wc -l) -lt 2 ]; then service hhvm-fastcgi start; fi' >> /var/spool/cron/crontabs/root

#############################################################################
## Replace Nginx with my optimized config.
#############################################################################
wget -O - https://github.com/nilopc/bashInstallers/blob/master/nginx.sh | bash

VAR=$(cat <<'END_HEREDOC'

###############################################################################
# REDIRECT www.example.com to example.com
###############################################################################
server {
  server_name www.example.com;
  rewrite ^ http://example.com$request_uri? permanent;
}

###############################################################################
# example.com
###############################################################################
server {
  listen   80;
  server_name example.com;

  ## Logging
  access_log	/var/log/nginx/example.com.access.log main;
  error_log	/var/log/nginx/example.com.error.log;

  ## Set Document Root
  root /var/www/example.com/;

  ## Set Directory Index
  index index.php index.html index.htm;

  ## PHP: Redirect all request to index.php
  location /
  {
    try_files $uri $uri/ /index.php$is_args$args;
    include /etc/nginx/helpers/extra.conf;
  }

  ## PHP: Pass all PHP request to HHVM
  location ~ \.php$ {
    fastcgi_keep_conn on;
    fastcgi_pass   127.0.0.1:9000;
    fastcgi_index  index.php;
    fastcgi_param  SCRIPT_FILENAME /var/www/example.com$fastcgi_script_name;
    include        fastcgi_params;
  }

  ## Remove the X-Powered-By HHVVM header.
  fastcgi_hide_header X-Powered-By;
}

END_HEREDOC
)

echo "$VAR" > /etc/nginx/sites-available/example.com
service nginx restart

#############################################################################
# Alias HHVM as PHP
#############################################################################
sudo ln -s `which hhvm` /usr/bin/php
sudo ln -s `which hhvm` /usr/bin/php-cli

#############################################################################
# PHPUNIT
#----------------------------------------------------------------------------
# Unit testing suite for PHP.
#############################################################################
sudo curl -o /usr/bin/phpunit https://phar.phpunit.de/phpunit.phar
sudo chmod a+x /usr/bin/phpunit

#############################################################################
# composer
#----------------------------------------------------------------------------
# Download dependencies in your PHP projects.
#############################################################################
sudo apt-get install -y git-core curl
cd
sudo curl -s http://getcomposer.org/installer | php
sudo mv composer.phar /usr/bin/composer
sudo chmod a+x /usr/bin/composer

#############################################################################
# php-cs-fixer
#############################################################################
sudo curl http://cs.sensiolabs.org/get/php-cs-fixer.phar -o /usr/bin/php-cs-fixer
sudo chmod a+x /usr/bin/php-cs-fixer

#############################################################################
# phpmd
#############################################################################
pear channel-discover pear.phpmd.org
pear channel-discover pear.pdepend.org
pear install --alldeps phpmd/PHP_PMD

#############################################################################
# phpcs
#############################################################################
pear install --alldeps PHP_CodeSniffer


text:
using System;
using System.Diagnostics.Contracts;
using System.Linq;
using NMoneys.Extensions;
using NMoneys.Support;

namespace NMoneys.Change
{
	public static partial class ChangeOperations
	{
		/// <summary>
		/// Returns a way to make an optimal change (in terms of number of denominations) for a particular amount of money given a regardless of the set of denominations in
		/// an efficient manner.
		/// </summary>
		/// <remarks>The computation of the change of made using dynamic programming strategies, that is, computing all possible solutions
		/// and selecting an optimal one (in terms of number of denominations used).
		/// <para>This way of computing yields an optimal solution (in terms of number of denominations) even for
		/// arbitrary, non-canonical, denomination sets.</para>
		/// <para>This strategy allows computing the result in O(m*n) time, being n the size of <paramref name="money"/> and m the length of <paramref name="denominations"/>.</para>
		/// <para>An optimal change is only considered for denomination sets that provide a complete (non-partial)
		/// way of making change for the given amount. Partial change solutions are not considered optimal solutions and thus, not calculated.</para>
		/// <para>A practical infinite (or sufficiently large) number of denominations is assumed. That is, we are not going to run out of denominations when making the change.</para>
		/// </remarks>
		/// <param name="money">The monetary quantity to make optimal change of.</param>
		/// <param name="denominations">The monetary denominations for which the optimal change is made.</param>
		/// <returns>A solution with the denominations used for making optimal change.</returns>
		/// <exception cref="ArgumentOutOfRangeException">If <paramref name="money"/> is not positive.</exception>
		/// <exception cref="ArgumentNullException">If <paramref name="denominations"/> is null.</exception>
		[Pure]
		public static OptimalChangeSolution MakeOptimalChange(this Money money, params Denomination[] denominations)
		{
			Positive.Amounts.AssertArgument(nameof(money), money.Amount);
			Guard.AgainstNullArgument(nameof(denominations), denominations);

			long n = money.MinorIntegralAmount;
			int m = denominations.Length;
			Currency operationCurrency = money.GetCurrency();
			IntegralDenomination[] integralDenominations = denominations
				// denominations are ordered because the algorithm is sensitive to order
				.OrderBy(d => d.Value)
				.Select(d => d.ToIntegral(operationCurrency))
				.ToArray();

			var table = new ushort[n + 1];
			var usedDenominations = new IntegralDenomination?[n + 1];

			// Base case (If given value n is 0)
			table[0] = 0;

			// Compute minimum denominations required for all values from 1 to n
			for (long i = 1; i <= n; i++)
			{
				// Initialize all table values as Infinite
				table[i] = ushort.MaxValue;

				// Go through all denominations smaller than i
				for (int j = 0; j < m && integralDenominations[j].IntegralAmount <= i; j++)
				{
					uint subResult = table[i - integralDenominations[j].IntegralAmount];
					if (subResult != ushort.MaxValue && subResult + 1u < table[i])
					{
						table[i] = (ushort)(subResult + 1);
						usedDenominations[i] = integralDenominations[j];
					}
				}
			}

			var solution = new OptimalChangeSolution(n, operationCurrency, table, usedDenominations);
			return solution;
		}

		/// <summary>
		/// Returns a way to make an optimal change (in terms of number of denominations) for a particular amount of money given a regardless of the set of denominations in
		/// an efficient manner.
		/// </summary>
		/// <remarks>This overload is a facility method for easier syntax. <see cref="MakeOptimalChange(NMoneys.Money,NMoneys.Change.Denomination[])"/>
		/// for more information.</remarks>
		/// <param name="money">The monetary quantity to make optimal change of.</param>
		/// <param name="denominationValues">The monetary denomination values for which the optimal change is made.</param>
		/// <returns>A solution with the denominations used for making optimal change.</returns>
		/// <exception cref="ArgumentOutOfRangeException">If <paramref name="money"/> is not positive.</exception>
		/// <exception cref="ArgumentNullException">If <paramref name="denominationValues"/> is null.</exception>
		[Pure]
		public static OptimalChangeSolution MakeOptimalChange(this Money money, params decimal[] denominationValues)
		{
			Guard.AgainstNullArgument(nameof(denominationValues), denominationValues);

			return money.MakeOptimalChange(denominationValues.Select(v => new Denomination(v)).ToArray());
		}
	}
}


text:
package org.moebuff.magi.beatmap;

import org.moebuff.magi.util.PathUtil;
import org.moebuff.magi.util.StringUtil;

import java.io.File;
import java.util.*;

/**
 * beatmap加载器
 *
 * @author MuTo
 */
public class MapLoader {
    public static final String SONGPATH = PathUtil.addSeparator(true, PathUtil.USERDIR, "songs");
    private Set<BeatMap> baetMaps = new HashSet();

    public MapLoader() {
        for (String path : new File(SONGPATH).list())
            baetMaps.add(new BeatMap(SONGPATH + path));
    }

    // Properties
    // -------------------------------------------------------------------------

    public Set<BeatMap> getBaetMaps() {
        return baetMaps;
    }

    public void setBaetMaps(Set<BeatMap> baetMaps) {
        this.baetMaps = baetMaps;
    }
}


text:
package org.apache.isis.viewer.restfulobjects.tck.domainobject.oid;

import javax.ws.rs.core.MediaType;
import org.jboss.resteasy.client.ClientRequest;
import org.jboss.resteasy.client.ClientResponse;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.apache.isis.viewer.restfulobjects.applib.JsonRepresentation;
import org.apache.isis.viewer.restfulobjects.applib.RepresentationType;
import org.apache.isis.viewer.restfulobjects.applib.RestfulHttpMethod;
import org.apache.isis.viewer.restfulobjects.applib.client.RestfulClient;
import org.apache.isis.viewer.restfulobjects.applib.client.RestfulRequest;
import org.apache.isis.viewer.restfulobjects.applib.client.RestfulResponse;
import org.apache.isis.viewer.restfulobjects.applib.version.VersionRepresentation;
import org.apache.isis.viewer.restfulobjects.tck.IsisWebServerRule;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;

public class Get_whenRequestHeaders_Accept_whenInvalid_406 {

    @Rule
    public IsisWebServerRule webServerRule = new IsisWebServerRule();

    private RestfulClient client;

    @Before
    public void setUp() throws Exception {
        client = webServerRule.getClient();
    }

    @Test
    public void applicationJson_profileIncorrect_returns406() throws Exception {

        // given
        RestfulRequest request = client.createRequest(RestfulHttpMethod.GET, "objects/BSRL/64");
        request.withHeader(RestfulRequest.Header.ACCEPT, RepresentationType.USER.getMediaType());

        // when
        final RestfulResponse<VersionRepresentation> restfulResponse = request.executeT();

        assertThat(restfulResponse.getStatus(), is(RestfulResponse.HttpStatusCode.NOT_ACCEPTABLE));
    }

    @Test
    public void incorrectMediaType_returnsNotAcceptable() throws Exception {

        // given
        final ClientRequest clientRequest = client.getClientRequestFactory().createRelativeRequest("objects/BSRL/64");
        clientRequest.accept(MediaType.APPLICATION_ATOM_XML_TYPE);

        // when
        final ClientResponse<?> resp = clientRequest.get();
        final RestfulResponse<JsonRepresentation> restfulResponse = RestfulResponse.of(resp);

        // then
        assertThat(restfulResponse.getStatus(), is(RestfulResponse.HttpStatusCode.NOT_ACCEPTABLE));
    }

}


text:
from sklearn.externals import joblib
from train.build_model import *
from sklearn.metrics import roc_curve, roc_auc_score
import matplotlib.pyplot as plt

xtr, ytr, xte, yte = joblib.load('data/clsdata.pkl')

cls = build_logit(xtr, ytr)

y_hat_prob = model_predict(xte, cls)

fpr, tpr, thr = roc_curve(yte, y_hat_prob)
auc = roc_auc_score(yte, y_hat_prob)
print(auc)

plt.plot(fpr, tpr)
plt.show()


text:
Here is the list of properties that can be configured via `MarkwonTheme.Builder` class.

:::tip
Starting with <Badge text="3.0.0" /> there is no need to manually construct a `MarkwonTheme`.
Instead a `Plugin` should be used:
```java
final Markwon markwon = Markwon.builder(context)
        .usePlugin(new AbstractMarkwonPlugin() {
            @Override
            public void configureTheme(@NonNull MarkwonTheme.Builder builder) {
                builder
                        .codeTextColor(Color.BLACK)
                        .codeBackgroundColor(Color.GREEN);
            }
        })
        .build();
```
:::

## Link color

Controls the color of a [link](#)

<ThemeProperty name="linkColor" type="@ColorInt int" defaults="Default link color of a context where markdown is displayed *" />

* `TextPaint#linkColor` will be used to determine linkColor of a context

## Link underline <Badge text="4.5.0" />

<ThemeProperty name="isLinkUnderlined" type="Boolean" defaults="true" />

## Block margin

Starting margin before text content for the:
* lists
* blockquotes
* task lists

<ThemeProperty name="blockMargin" type="@Px int" defaults="24dp" />

## Block quote

Customizations for the `blockquote` stripe

> Quote

### Stripe width

Width of a blockquote stripe

<ThemeProperty name="blockQuoteWidth" type="@Px int" defaults="1/4 of the block margin" />

### Stripe color

Color of a blockquote stripe

<ThemeProperty name="blockQuoteColor" type="@ColorInt int" defaults="textColor with 25 (0-255) alpha value" />

## List

### List item color

Controls the color of a list item. For ordered list: leading number,
for unordered list: bullet.

* UL
1. OL

<ThemeProperty name="listItemColor" type="@ColorInt int" defaults="Text color" />

### Bullet item stroke width

Border width of a bullet list item (level 2)

* First
* * Second
* * * Third

<ThemeProperty name="bulletListItemStrokeWidth" type="@Px int" defaults="Stroke width of TextPaint" />

### Bullet width

The width of the bullet item

* First
  * Second
    * Third

<ThemeProperty name="bulletWidth" type="@Px int" defaults="min(blockMargin, lineHeight) / 2" />

## Code

### Inline code text color

The color of the `code` content

<ThemeProperty name="codeTextColor" type="@ColorInt int" defaults="Content text color" />

### Inline code background color

The color of `background` of a code content

<ThemeProperty name="codeBackgroundColor" type="@ColorInt int" defaults="inline code text color with 25 (0-255) alpha" />

### Block code text color

```
The color of code block text
```

<ThemeProperty name="codeBlockTextColor" type="@ColorInt int" defaults="inline code text color" />

### Block code background color

```
The color of background of code block text
```

<ThemeProperty name="codeBlockBackgroundColor" type="@ColorInt int" defaults="inline code background color" />

### Block code leading margin

Leading margin for the block code content

<ThemeProperty name="codeMultilineMargin" type="@Px int" defaults="8dip" />

### Code typeface

Typeface of code content

<ThemeProperty name="codeTypeface" type="android.graphics.Typeface" defaults="Typeface.MONOSPACE" />

### Block code typeface <Badge text="3.0.0" />

Typeface of block code content

<ThemeProperty name="codeBlockTypeface" type="android.graphics.Typeface" defaults="codeTypeface if set or Typeface.MONOSPACE" />

### Code text size

Text size of code content

<ThemeProperty name="codeTextSize" type="@Px int" defaults="(Content text size) * 0.87 if no custom Typeface was set, otherwise (content text size)" />

### Block code text size <Badge text="3.0.0" />

Text size of block code content

<ThemeProperty name="codeBlockTextSize" type="@Px int" defaults="codeTextSize if set or (content text size) * 0.87 if no custom Typeface was set, otherwise (content text size)" />

## Heading

### Break height

The height of a brake under H1 & H2

<ThemeProperty name="headingBreakHeight" type="@Px int" defaults="Stroke width of context TextPaint" />

### Break color

The color of a brake under H1 & H2

<ThemeProperty name="headingBreakColor" type="@ColorInt int" defaults="(text color) with 75 (0-255) alpha" />

### Typeface <Badge text="1.1.0" />

The typeface of heading elements

<ThemeProperty name="headingTypeface" type="android.graphics.Typeface" defaults="default text Typeface" />

### Text size <Badge text="1.1.0" />

Array of heading text sizes _ratio_ that is applied to text size

<ThemeProperty name="headingTextSizeMultipliers" type="float[]" defaults="{2.F, 1.5F, 1.17F, 1.F, .83F, .67F} (HTML spec)" />

## Thematic break

### Color

Color of a thematic break

<ThemeProperty name="thematicBreakColor" type="@ColorInt int" defaults="(text color) with 25 (0-255) alpha" />

### Height

Height of a thematic break

<ThemeProperty name="thematicBreakHeight" type="@Px int" defaults="Stroke width of context TextPaint" />


text:
layout: default
---
{% include header.html %}
{% include about.html %}
{% include animate.html %}
{% include footer.html %}


text:
from south.db import db
from django.db import models
from courant.core.news.models import *

class Migration:

    def forwards(self, orm):
        db.create_index('news_article', ['status_id', 'published_at'])
        db.create_index('news_article', ['published_at'])
        db.create_index('news_article', ['content_modified_at'])
        db.create_index('news_articlestatus', ['published'])
        db.create_index('news_articlestatus', ['id'])

    def backwards(self, orm):
        db.delete_index('news_articlestatus', ['id'])
        db.delete_index('news_articlestatus', ['published'])
        db.delete_index('news_article', ['content_modified_at'])
        db.delete_index('news_article', ['published_at'])
        db.delete_index('news_article', ['status_id', 'published_at'])

    models = {
        'media.mediaitem': {
            'Meta': {'ordering': '["-created_at"]', 'get_latest_by': '"-created_at"'},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'dynamic_models.dynamictype': {
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'news.section': {
            'Meta': {'unique_together': '("parent","path")'},
            'dynamic_type': ('models.ForeignKey', ["orm['dynamic_models.DynamicType']"], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'full_path': ('models.CharField', [], {'default': "'/'", 'max_length': '100', 'editable': 'False', 'blank': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', [], {'max_length': '100'}),
            'parent': ('models.ForeignKey', ["orm['news.Section']"], {'related_name': "'subsections'", 'null': 'True', 'blank': 'True'}),
            'path': ('models.CharField', [], {'max_length': '100'})
        },
        'news.articledisplaytype': {
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', [], {'max_length': '40'}),
            'template_name': ('models.CharField', ['"Template name (without file extension)"'], {'max_length': '100'})
        },
        'news.issuearticle': {
            'Meta': {'ordering': "['order']"},
            'article': ('models.ForeignKey', ["orm['news.Article']"], {}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'issue': ('models.ForeignKey', ["orm['news.Issue']"], {}),
            'order': ('models.PositiveIntegerField', [], {})
        },
        'news.articlemedia': {
            'Meta': {'ordering': "['order']"},
            'article': ('models.ForeignKey', ["orm['news.Article']"], {}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'media_item': ('models.ForeignKey', ["orm['media.MediaItem']"], {}),
            'order': ('models.PositiveSmallIntegerField', [], {})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label','model'),)", 'db_table': "'django_content_type'"},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'news.issuedisplaytype': {
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', [], {'max_length': '40'}),
            'template_name': ('models.CharField', ['"Template name (without file extension)"'], {'max_length': '100'})
        },
        'news.articlebyline': {
            'article': ('models.ForeignKey', ["orm['news.Article']"], {}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'order': ('models.PositiveSmallIntegerField', [], {}),
            'position': ('models.CharField', [], {'max_length': '100'}),
            'staffer': ('models.ForeignKey', ["orm['staff.Staffer']"], {})
        },
        'discussions.commentoptions': {
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'news.issue': {
            'Meta': {'ordering': "['-published_at']", 'get_latest_by': "'published_at'"},
            'articles': ('models.ManyToManyField', ["orm['news.Article']"], {'related_name': "'issues'", 'through': "'IssueArticle'"}),
            'created_at': ('CreationDateTimeField', [], {}),
            'display_type': ('models.ForeignKey', ["orm['news.IssueDisplayType']"], {}),
            'dynamic_type': ('models.ForeignKey', ["orm['dynamic_models.DynamicType']"], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'lead_media': ('models.ForeignKey', ["orm['media.MediaItem']"], {'related_name': "'issues'", 'null': 'True', 'blank': 'True'}),
            'modified_at': ('ModificationDateTimeField', [], {}),
            'name': ('models.CharField', [], {'max_length': '100'}),
            'published': ('models.BooleanField', [], {'default': 'False'}),
            'published_at': ('models.DateTimeField', [], {})
        },
        'news.articlestatus': {
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', [], {'max_length': '100'}),
            'published': ('models.BooleanField', [], {'default': 'False'})
        },
        'staff.staffer': {
            'Meta': {'ordering': "['user']", 'unique_together': "['first_name','last_name']"},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'news.article': {
            'Meta': {'ordering': '["-published_at","issuearticle__order"]', 'get_latest_by': '"-published_at"'},
            'authors': ('models.ManyToManyField', ["orm['staff.Staffer']"], {'related_name': "'articles'", 'through': "'ArticleByline'"}),
            'body': ('models.TextField', [], {}),
            'body_html': ('models.TextField', [], {'editable': 'False'}),
            'content_modified_at': ('ModificationDateTimeField', [], {}),
            'content_type': ('models.ForeignKey', ["orm['contenttypes.ContentType']"], {'null': 'True', 'editable': 'False'}),
            'correction': ('models.ForeignKey', ["orm['news.Article']"], {'related_name': "'corrected'", 'null': 'True', 'blank': 'True'}),
            'created_at': ('CreationDateTimeField', [], {}),
            'display_type': ('models.ForeignKey', ["orm['news.ArticleDisplayType']"], {'related_name': '"articles"'}),
            'dynamic_type': ('models.ForeignKey', ["orm['dynamic_models.DynamicType']"], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'heading': ('models.CharField', [], {'max_length': '255'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'media': ('models.ManyToManyField', ["orm['media.MediaItem']"], {'related_name': "'articles'", 'through': "'ArticleMedia'"}),
            'modified_at': ('ModificationDateTimeField', [], {}),
            'published_at': ('models.DateTimeField', [], {}),
            'section': ('models.ForeignKey', ["orm['news.Section']"], {'related_name': '"articles"'}),
            'slug': ('models.SlugField', [], {'unique': 'True'}),
            'status': ('models.ForeignKey', ["orm['news.ArticleStatus']"], {'related_name': '"articles"'}),
            'subheading': ('models.CharField', [], {'max_length': '255', 'blank': 'True'}),
            'summary': ('models.TextField', [], {}),
            'summary_html': ('models.TextField', [], {'editable': 'False'}),
            'tags': ('TagField', [], {})
        }
    }

    complete_apps = ['news']


text:
<?xml version="1.0" encoding="ascii"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
          "DTD/xhtml1-transitional.dtd">

  <title>mHTM</title>
  <link rel="stylesheet" href="epydoc.css" type="text/css" />

         Home

         Trees

         Indices

         Help

nano_htm

        Package mHTM

[hide private]

[frames] | no frames]

Package mHTM

source code

math HTM (mHTM) is a Python build of hierarchical temporal memory
  (HTM). This build specifically utilizes the cortical learning algorithms (CLA). A mathematical
  framework was developed for HTM. This library is built off that
  framework. The framework is currently only for the spatial pooler (SP);
  however, it is by no means limited to expand. This implementation was
  specifically designed to be a completely accurate representation of HTM.
  Additionally, care was taken to ensure that efficient computations are
  occurring.

We have recently submitted a paper to IEEE TNNLS explaining this work.
  A preprint is available on arXiv.

To aid in tying HTM into the machine learning community, this
  implementation was built to be compatible with Scikit-Learn. If
  you are familiar with Scikit-Learn, this API should feel natural.
  Additionally, because the Scikit-Learn interface is used, the SP in this
  implementation may be used in many of the pre-existing Scikit-Learn
  tools, namely those utilizing cross-validation (CV). A custom parameter generator was constructed, explicitly for
  that purpose.

The implementation of the SP is single-threaded; however, multiple
  forms of parallelizations for cross-validation and parameter optimization
  exist. For a local machine, simply using one of Scikit-Learn's CV
  functions supporting parallelization should suffice. For a cluster
  environment, code is provided to create and launch jobs.

This package is platform independent and should run on any system
  containing the prerequisites. If you are using a cluster, the cluster
  must be running SLURM, and as such must be a valid Linux
  distribution.

Prerequisites

        Python 2.7.X

        Numpy

        matplotlib

        Scipy

        Scikit-Learn

        Bottleneck

Installation

        Install all prerequisites

            If you have pip

pip install numpy matplotlib scipy
            scikit-learn bottleneck

            If you are on Windows

You may download the the (unofficial) precompiled versions
            available from UCI. Simply download the appropriate Python
            wheel and install it using pip, i.e. pip install my_file.whl, where my_file is
            the name of your whl file.

        Download this repo and execute python setup.py install.

Usage

The API is fully documented and available in the "docs"
    folder. Simply open "index.html" in your favorite web
    browser.

As a starting point, examples have been prepared for working with
    MNIST. The dataset is additionally, included. Refer to
    "mnist_simple.py" in "src/examples" for a basic
    introduction into the API. In that same folder, refer to
    "mnist_parallel" for examples for using parallelizations
    locally or an cluster.

The "dev" folder contains the latest code regarding new
    experiments. This content is subject to change at any point and is not
    guaranteed to work; however, it can also be used as a basis for
    exploring this library.

Development

For building the docs, Epydoc is used. To generate new docs,

Citing this Work

While this code is completely free of charge, it is highly
    appreciated that all uses of this code are cited (both in publications
    as well as within any modified code). Once the IEEE TNNLS paper is
    approved please cite that work. For now, please cite the preprint:

J. Mnatzaganian, E. Fokoue, and D. Kudithipudi, "A Mathematical
    Formalization of Hierarchical Temporal Memory Cortical Learning
    Algorithm's Spatial Pooler," arXiv preprint arXiv:1601.06116,
    2016.

Bug Reports and Support

No official support is provided; however, support may be provided.
    To ensure all feedback is within the same location, please use the Wiki
    for asking general questions and create issues if any bugs are
    found.

Author

The original author of this code was James Mnatzaganian. For contact
    info, as well as other details, see his corresponding website.

This work was created at RIT's NanoComputing
    Research Laboratory.

Legal

This code is licensed under the MIT
    license, with one caveat. Numenta owns patents on specific items.
    While this code was written without using any of Numenta's code, it is
    possible that those patent laws still apply. Before using this code,
    commercially, it is recommended to seek legal advice.

Connectivity

The following image shows how everything is connected:

  <map id="import_graph" name="import_graph">
<area shape="rect" id="node1_11" href="mHTM-module.html" title="mHTM" alt="" coords="522,379,563,397"/>
<area shape="rect" id="node1" href="mHTM-module.html" title="mHTM" alt="" coords="511,363,573,401"/>
<area shape="rect" id="node2_12" href="mHTM.datasets-module.html" title="mHTM.datasets" alt="" coords="515,311,569,329"/>
<area shape="rect" id="node2" href="mHTM.datasets-module.html" title="datasets" alt="" coords="505,295,579,333"/>
<area shape="rect" id="node3_13" href="mHTM.datasets.loader-module.html" title="mHTM.datasets.loader" alt="" coords="497,90,589,107"/>
<area shape="rect" id="node3" href="mHTM.datasets.loader-module.html" title="datasets.loader" alt="" coords="485,73,599,112"/>
<area shape="rect" id="node5_14" href="mHTM.examples.mnist_parallel-module.html" title="mHTM.examples.mnist_parallel" alt="" coords="16,261,160,278"/>
<area shape="rect" id="node5" href="mHTM.examples.mnist_parallel-module.html" title="examples.mnist_parallel" alt="" coords="5,244,171,283"/>
<area shape="rect" id="node6_15" href="mHTM.examples.mnist_runner-module.html" title="mHTM.examples.mnist_runner" alt="" coords="306,193,445,210"/>
<area shape="rect" id="node6" href="mHTM.examples.mnist_runner-module.html" title="examples.mnist_runner" alt="" coords="295,176,455,215"/>
<area shape="rect" id="node7_16" href="mHTM.examples.mnist_simple-module.html" title="mHTM.examples.mnist_simple" alt="" coords="306,125,446,142"/>
<area shape="rect" id="node7" href="mHTM.examples.mnist_simple-module.html" title="examples.mnist_simple" alt="" coords="295,108,456,147"/>
<area shape="rect" id="node4_17" href="mHTM.examples-module.html" title="mHTM.examples" alt="" coords="513,22,573,39"/>
<area shape="rect" id="node4" href="mHTM.examples-module.html" title="examples" alt="" coords="501,5,583,44"/>
<area shape="rect" id="node8_18" href="mHTM.parallel-module.html" title="mHTM.parallel" alt="" coords="211,261,255,278"/>
<area shape="rect" id="node8" href="mHTM.parallel-module.html" title="parallel" alt="" coords="200,244,265,283"/>
<area shape="rect" id="node9_19" href="mHTM.plot-module.html" title="mHTM.plot" alt="" coords="530,158,554,175"/>
<area shape="rect" id="node9" href="mHTM.plot-module.html" title="plot" alt="" coords="519,141,565,180"/>
<area shape="rect" id="node10_20" href="mHTM.prob-module.html" title="mHTM.prob" alt="" coords="362,261,390,278"/>
<area shape="rect" id="node10" href="mHTM.prob-module.html" title="prob" alt="" coords="351,244,400,283"/>
<area shape="rect" id="node11_21" href="mHTM.region-module.html" title="mHTM.region" alt="" coords="523,243,562,261"/>
<area shape="rect" id="node11" href="mHTM.region-module.html" title="region" alt="" coords="512,227,572,265"/>
</map>

Import Graph

Developer Notes

The following notes are for developers only.

Installation

          Download and install graphviz

          Edit line 95 in dev/epydoc_config.txt to point to the
          directory containing "dot.exe". This is part of the
          graphviz installation.

          Download this repo and execute python setup.py install.

          Download and install Epydoc

Generating the API

From the mHTM folder, execute python epydoc --config=epydoc_config.txt
      mHTM

Author:
        James
        Mnatzaganian

Organization:
        NanoComputing Research Laboratory

Requires:
        Python 2.7.X

Version:
        0.9.0

License:
        The MIT License

Copyright:
        © 2016 James Mnatzaganian

        Submodules
        [hide private]

 mHTM.parallel: Module for parallel processing.

 mHTM.plot: Plotting module.

 mHTM.prob: Module for computing various probabilities

 mHTM.region: Implementation for a region in an HTM.

      Examples

 mHTM.examples: Defines the examples package.

 mHTM.examples.mnist_parallel: Testing SP with MNIST using parameter optimization and doing
        iterations in parallel.

 mHTM.examples.mnist_runner: SLURM runner for MNIST.

 mHTM.examples.mnist_simple: Testing SP with MNIST using a simple demonstration.

      Datasets

 mHTM.datasets: Defines the datasets package.

 mHTM.datasets.loader: Module for loading datasets.

         Home

         Trees

         Indices

         Help

nano_htm

    Generated by Epydoc 3.0.1 on Sat Feb 06 21:46:17 2016

      http://epydoc.sourceforge.net


text:
/* ###
 * IP: GHIDRA
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package ghidra.app.util.bin.format.macho.dyld;

import java.io.IOException;

import ghidra.app.util.bin.BinaryReader;
import ghidra.app.util.bin.StructConverter;
import ghidra.app.util.bin.format.macho.MachConstants;
import ghidra.program.model.data.*;
import ghidra.util.exception.DuplicateNameException;

/**
 * Represents a dyld_cache_range_entry structure.
 *
 * @see dyld_cache_format.h
 */
@SuppressWarnings("unused")
public class DyldCacheRangeEntry implements StructConverter {

	private long startAddress;
	private int size;
	private int imageIndex;

	/**
	 * Create a new {@link DyldCacheRangeEntry}.
	 *
	 * @param reader A {@link BinaryReader} positioned at the start of a DYLD range entry
	 * @throws IOException if there was an IO-related problem creating the DYLD range entry
	 */
	public DyldCacheRangeEntry(BinaryReader reader) throws IOException {
		startAddress = reader.readNextLong();
		size = reader.readNextInt();
		imageIndex = reader.readNextInt();
	}

	@Override
	public DataType toDataType() throws DuplicateNameException, IOException {
		StructureDataType struct = new StructureDataType("dyld_cache_range_entry", 0);
		struct.add(QWORD, "startAddress", "");
		struct.add(DWORD, "size", "");
		struct.add(DWORD, "imageIndex", "");
		struct.setCategoryPath(new CategoryPath(MachConstants.DATA_TYPE_CATEGORY));
		return struct;
	}
}


text:
using MasterDevs.ChromeDevTools;
using Newtonsoft.Json;
using System.Collections.Generic;

namespace MasterDevs.ChromeDevTools.Protocol.Chrome.ServiceWorker
{
	[Event(ProtocolName.ServiceWorker.WorkerVersionUpdated)]
	[SupportedBy("Chrome")]
	public class WorkerVersionUpdatedEvent
	{
		/// <summary>
		/// Gets or sets Versions
		/// </summary>
		public ServiceWorkerVersion[] Versions { get; set; }
	}
}


text:
\section{\textbf{Coarse List}}
% ---------------------------------------------------------------------------- %
\subsection{Particular Case}
\par
In this exercise we are trying to solve the problem of accessing a List data
structure by multiple threads simultaneously. While allowing multiple threads
accessing the data structure, we still want to maintain its consistency.
\par
As mentioned in the book, the invariants that we want to satisfy are:
\par
\begin{enumerate}
\item Sentinels are neither added or removed (i.e. the head and the tail)
\item Nodes are sorted by key
\item Keys are unique
\end{enumerate}
\par
% ---------------------------------------------------------------------------- %
\subsection{Solution}
\par
Our first approach to solving this problem is then through a coarse grained
list. This means that we use a lock in such a way that any access to the list is
sequential because this lock must be hold and there is only one lock.
\par
Let's take a quick look at the add and remove items to grasp the idea behind
this algorithm. Notice how the whole logic of the \textit{add()} and
\textit{remove()} operations are guarded by the calls to \textit{lock()} and
\textit{unlock()}.
\par
\hfil
\begin{lstlisting}[style=numbers]
  /**
   * Add an element.
   * @param item element to add
   * @return true iff element was not there already
   */

  public boolean add(T item) {
    Node pred, curr;
    int key = item.hashCode();
    lock.lock();
    try {
      pred = head;
      curr = pred.next;
      while (curr.key < key) {
        pred = curr;
        curr = curr.next;
      }
      if (key == curr.key) {
        return false;
      } else {
        Node node = new Node(item);
        node.next = curr;
        pred.next = node;
        return true;
      }
    } finally {
      lock.unlock();
    }
  }

  /**
   * Remove an element.
   * @param item element to remove
   * @return true iff element was present
   */
  public boolean remove(T item) {
    Node pred, curr;
    int key = item.hashCode();
    lock.lock();
    try {
      pred = this.head;
      curr = pred.next;
      while (curr.key < key) {
        pred = curr;
        curr = curr.next;
      }
      if (key == curr.key) {  // present
        pred.next = curr.next;
        return true;
      } else {
        return false;         // not present
      }
    } finally {               // always unlock
      lock.unlock();
    }
  }
\end{lstlisting}
\hfill
\par
% ---------------------------------------------------------------------------- %
\subsection{Experiment Description}
\par
We have four test cases:
\par
\begin{enumerate}
\item \textit{testSequential()}. The main thread inserts multiple elements to
the list, then checks if all elements are there and finally removes all elements
one by one.
\item \textit{testParallelAdd()}. The main thread spawns multiple threads that
add elements to the list concurrently. When they are done, the main thread
checks if all elements are in there and finally removes all elements.
\item \textit{testParallelRemove()}. The main thread inserts multiple elements
to the list, then checks if all elements are in there and finally it spawns
multiple threads that remove the elements from the list concurrently.
\item \textit{testParallelBoth()}. The main thread creates multiple threads.
Some of them insert elements to the list and some of them remove elements.
\end{enumerate}
\par
% ---------------------------------------------------------------------------- %
\subsection{Sample Results}
\par
For this test, we saw that it failed consistently with the following error:
\begin{verbatim}
[oraadm@gdlaa008 Lists]$ junit lists.CoarseListTest
.sequential add, contains, and remove
.parallel both
Exception in thread "Thread-3" Exception in thread "Thread-5" Exception in
thread "Thread-9" Exception in thread "Thread-15"
junit.framework.AssertionFailedError: RemoveThread: duplicate remove: 32
        at junit.framework.Assert.fail(Assert.java:57)
        at junit.framework.TestCase.fail(TestCase.java:227)
        at lists.CoarseListTest$RemoveThread.run(CoarseListTest.java:145)
junit.framework.AssertionFailedError: RemoveThread: duplicate remove: 121
        at junit.framework.Assert.fail(Assert.java:57)
        at junit.framework.TestCase.fail(TestCase.java:227)
        at lists.CoarseListTest$RemoveThread.run(CoarseListTest.java:145)
junit.framework.AssertionFailedError: RemoveThread: duplicate remove: 17
        at junit.framework.Assert.fail(Assert.java:57)
        at junit.framework.TestCase.fail(TestCase.java:227)
        at lists.CoarseListTest$RemoveThread.run(CoarseListTest.java:145)
junit.framework.AssertionFailedError: RemoveThread: duplicate remove: 66
        at junit.framework.Assert.fail(Assert.java:57)
        at junit.framework.TestCase.fail(TestCase.java:227)
        at lists.CoarseListTest$RemoveThread.run(CoarseListTest.java:145)
.parallel add
.parallel remove

Time: 0.017

OK (4 tests)
\end{verbatim}
\par
So, the test that fails consistently is \textit{parallel both}. It complains
saying that an specific element was removed multiple times by one or more
threads.
\par
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Proposed solution}
\par
This failure turns out to be a test issue. Each remover thread is in charge of
removing a specific set of elements, say from $25$ to $50$. In the same way, an
inserter thread is in charge of adding a specific set of elements. Since the
threads for a given interval are spawned concurrently, it might be the case that
a remover thread is trying to remove an element that hasn't been inserted yet.
\par
One way of solving this problem is by asking the remover thread to retry till it
is successful. (Obviously this fix should be improved further since it might
lead to a hang in user's application -- Imagine the case where an specific
element is never inserted. However, for this specific case, the fix is good
enough.)
\par
This is the code of the remover thread before the fix:
\par
\hfill
\begin{lstlisting}[style=numbers]
  class removethread extends thread {
    int value;
    removethread(int i) {
      value = i;
    }
    public void run() {
      for (int i = 0; i < per_thread; i++) {
        if (!instance.remove(value + i)) {
          fail("removethread: duplicate remove: " + (value + i));
        }
      }
    }
  }
\end{lstlisting}
\hfill
\par
And here is the code with the fix:
\par
\hfill
\begin{lstlisting}[style=numbers]
  class removethread extends thread {
    int value;
    removethread(int i) {
      value = i;
    }
    public void run() {
      for (int i = 0; i < per_thread; i++) {
        while (!instance.remove(value + i)) {}
      }
    }
  }
\end{lstlisting}
\hfill
\par
And here is the test output after the fix:
\par
\begin{verbatim}
[oraadm@gdlaa008 Lists]$ junit lists.CoarseListTest
.sequential add, contains, and remove
.parallel both
.parallel add
.parallel remove

Time: 0.016

OK (4 tests)
\end{verbatim}
% ---------------------------------------------------------------------------- %
\subsection{Interpretation}
% ---------------------------------------------------------------------------- %
In this experiment we observed how to implement a simple list that allows
concurrent access by making all the access sequential. We saw that even though
the algorithm is correct, a correct usage of the API is also required.


text:
class BulletDebugDrawSRE final : public btIDebugDraw {
public:
    void drawLine(const btVector3 &from, const btVector3 &to, const btVector3 &color) override;

    void drawContactPoint(const btVector3 &PointOnB, const btVector3 &normalOnB, btScalar distance, int lifeTime,
                          const btVector3 &color) override;

    void reportErrorWarning(const char *warningString) override;

    void draw3dText(const btVector3 &location, const char *textString) override;

    void setDebugMode(int debugMode) override;

    int getDebugMode() const override;

    void newFrame();
    void render(sre::RenderPass& renderPass);
private:
    DebugDrawModes mode = DBG_DrawWireframe;
    std::vector<glm::vec3> linePositions;
    std::vector<sre::Color> lineColors;
};


text:
interface Props {
  tag?: React.ReactType;
  className?: string;
}

declare var ModalFooter: React.StatelessComponent<Props>;
export default ModalFooter;


text:
namespace chrome {

bool IsMediaDevice(const FilePath::StringType& mount_point) {
  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::FILE));

  FilePath dcim_path(mount_point);
  FilePath::StringType dcim_dir(kDCIMDirectoryName);
  if (!file_util::DirectoryExists(dcim_path.Append(dcim_dir))) {
    // Check for lowercase 'dcim' as well.
    FilePath dcim_path_lower(dcim_path.Append(StringToLowerASCII(dcim_dir)));
    if (!file_util::DirectoryExists(dcim_path_lower))
      return false;
  }
  return true;
}

string16 GetFullProductName(const std::string& vendor_name,
                            const std::string& model_name) {
  if (vendor_name.empty() && model_name.empty())
    return string16();

  std::string product_name;
  if (vendor_name.empty())
    product_name = model_name;
  else if (model_name.empty())
    product_name = vendor_name;
  else
    product_name = vendor_name + ", " + model_name;
  return IsStringUTF8(product_name) ?
      UTF8ToUTF16("(" + product_name + ")") : string16();
}

string16 GetDisplayNameForDevice(uint64 storage_size_in_bytes,
                                 const string16& name) {
  DCHECK(!name.empty());
  return (storage_size_in_bytes == 0) ?
      name : ui::FormatBytes(storage_size_in_bytes) + ASCIIToUTF16(" ") + name;
}

}  // namespace chrome