text:
package org.opensextant.xtext;

import org.opensextant.ConfigException;
import org.opensextant.util.FileUtility;
import org.opensextant.xtext.converters.EmbeddedContentConverter;
import org.opensextant.xtext.converters.ImageMetadataConverter;
import org.opensextant.xtext.converters.MessageConverter;
import org.opensextant.xtext.converters.TextTranscodingConverter;
import org.opensextant.xtext.converters.TikaHTMLConverter;
import org.opensextant.xtext.converters.DefaultConverter;
import org.opensextant.xtext.converters.WebArchiveConverter;
import org.opensextant.xtext.collectors.ArchiveNavigator;
import org.opensextant.xtext.collectors.mailbox.OutlookPSTCrawler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import gnu.getopt.LongOpt;

import java.io.*;
import java.util.Set;
import java.util.HashSet;
import java.util.Map;
import java.util.HashMap;

import javax.activation.MimeType;
import javax.activation.MimeTypeParseException;

import org.apache.tika.io.IOUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOCase;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.io.filefilter.SuffixFileFilter;

/**
 *
 * Traverse a folder and return text versions of the documents found. Archiving
 * the text only copies at an output location of your choice.
 *
 * <pre>
 *
 * if input is a file, convert. Done.
 *
 * if input is an archive, unpack in temp space, iterate over dir, convert each.
 * Done
 *
 * if input is a folder iterate over dir, convert each. Done
 * </pre>
 *
 * TEXT OUTPUT form includes a JSON document header with metadata properties
 * from the original item. These are valid elements of the conversion process.
 * We try to maintain them apart from the true, readable text of the document.
 *
 *
 * Add a ConversiontListener to XText instance to capture the converted document
 * as it comes out of the main loop for converting archives and folders.
 *
 * extractText() runs over any file type and extracts text, saving it pushing
 * events to one optional listener
 *
 * convertFile(File) will convert a single file, returning a ConvertedDocument
 *
 *
 *
 * @author Marc C. Ubaldino, MITRE, ubaldino at mitre dot org
 */
public final class XText implements ExclusionFilter, Converter {

    private Logger log = LoggerFactory.getLogger(getClass());
    private boolean scrubHTML = false;

    private PathManager paths = new PathManager();

    public PathManager getPathManager() {
        return paths;
    }

    /** flag to manage if children are extracted or not.
     */
    private boolean extractEmbedded = false;

    /**
     * XText default is 1 MB of text
     */
    private int maxBuffer = DefaultConverter.MAX_TEXT_SIZE;
    /**
     * Heuristic - HTML content is likely 5x, maybe a lot more, the size of the
     * plain text it contains.  So with 1 MB the target max text size, 5 MB would be
     * the largest HTML document accepted here, by default.
     */
    private int maxHTMLBuffer = 5 * maxBuffer;
    private long maxFileSize = FILE_SIZE_LIMIT;

    protected Set<String> archiveFileTypes = new HashSet<String>();

    /**
     *
     */
    public static Map<String, Converter> converters = new HashMap<String, Converter>();
    private Converter defaultConversion;
    private Converter embeddedConversion;
    private Set<String> requestedFileTypes = new HashSet<String>();
    private Set<String> ignoreFileTypes = new HashSet<String>();

    /**
     */
    public XText() {
        defaults();
    }

    public void enableOverwrite(boolean b) {
        ConvertedDocument.overwrite = b;
    }

    /** @deprecated  use getPathManager().setConversionRoot( path ) */
    public void setArchiveDir(String root) throws IOException {
        paths.setConversionCache(root);
    }

    public void setMaxBufferSize(int sz) {
        maxBuffer = sz;
    }

    public void setMaxFileSize(int sz) {
        maxFileSize = (long) sz;
    }

    /**
     * Use Tika HTML de-crapifier. Default: No scrubbing.
     */
    public void enableHTMLScrubber(boolean b) {
        scrubHTML = b;
    }

    /**
     * enable/disable the extraction of embedded child documents in found documents.
     * Using embedded extraction may yield many small sub documents, aka children.
     * @param b
     */
    public void enableEmbeddedExtraction(boolean b) {
        extractEmbedded = b;
    }

    /**
     * The overall flag to save converted output or not. DEFAULT: true = save
     * it; provided caller specifies either saveWithInput or provides an
     * archiveRoot
     *
     * @param b
     */
    public void enableSaving(boolean b) {
        paths.enableSaving(b);
    }

    /**
     * Add the file extension for the file type you wish to convert. if Tika
     * supports it by default it should be no problem.
     */
    public void convertFileType(String ext) {
        requestedFileTypes.add(ext.toLowerCase());
    }

    /**
     * Ignore files ending with.... or of type ext. No assumption of case is
     * made. This is case sensitive.
     */
    public void ignoreFileType(String ext) {
        if (ext != null) {
            ignoreFileTypes.add(ext.toLowerCase());
        }
    }

    private ConversionListener postProcessor = null;

    /**
     * A conversion listener is any outside application or routine that will do
     * something more with the converted document. If unset nothing happens. ;)
     */
    public void setConversionListener(ConversionListener processor) {
        postProcessor = processor;
    }

    private boolean useTikaPST = false;

    public void enableTikaPST(boolean flag) {
        useTikaPST = flag;
    }

    /**
     * is the input an archive?
     */
    public boolean isArchive(String fpath) {
        String ext = FilenameUtils.getExtension(fpath);
        if (ext == null) {
            return false;
        }
        return archiveFileTypes.contains(ext.toLowerCase());
    }

    public boolean isPST(String fpath) {
        return isPSTExtension(FilenameUtils.getExtension(fpath));
    }

    public static boolean isPSTExtension(String ext) {
        if (ext == null) {
            return false;
        }
        return ("pst".equalsIgnoreCase(ext));
    }

    protected long total_conv_time = 0;
    protected int average_conv_time = 0;
    protected int total_conversions = 0;

    /**
     * Records overall counts and conversion times for documents converted.
     * This may not account for error'd documents.
     *
     * @param d
     */
    protected void trackStatistics(ConvertedDocument d) {
        if (d != null) {
            total_conv_time += d.conversion_time;
        }
        ++total_conversions;
    }

    public void reportStatistics() {
        average_conv_time = (int) ((float) total_conv_time / total_conversions);
        log.info("TOTAL of N=" + total_conversions + " documents converted"
                + "\n With an average time (ms) of " + average_conv_time);
    }

    protected long start_time = 0;
    protected long stop_time = 0;

    /**
     * Optional API routine.  If XText is used as a main program, this is the entry point for extraction/collection.
     * If XText is used as an API, caller may use convertFile() directly without engaging in the setup and assumptions behind this convenience method.
     * The main entry point to converting compound documents and folders.
     * @throws ConfigException
     */
    public void extractText(String filepath) throws IOException, ConfigException {

        start_time = System.currentTimeMillis();

        log.info("Conversion.  INPUT PATH={}", filepath);
        String path = FilenameUtils.normalize(new File(filepath).getAbsolutePath(), true);
        if (path == null) {
            throw new IOException("Failed to normalize the path: " + filepath);
        }

        File input = new File(path);
        if (!input.exists()) {
            throw new IOException("Non existent input FILE=" + path);
        }

        /* Filter on absolute path */
        if (PathManager.isXTextCache(path)) {
            throw new ConfigException(
                    "XText cannot be directed to extract text from its own cache files. "
                            + "Move the cache files out of ./xtext/ folders if you really need to do this.");
        }

        if (isArchive(input.getName())) {
            // Archive will collect originals to "export"
            // Archive will save conversions to "output"
            // PathManager is STATEFUL for as long as this archive is processing
            // If an archive is uncovered while traversing files, its contents can be dumped to the child export folder.
            convertArchive(input);
        } else if (isPST(input.getName()) && !useTikaPST) {
            this.convertOutlookPST(input);
        } else if (input.isFile()) {
            // If prefix is not set, then conversion will be dumped flatly to output area.
            paths.setInputRoot(input);
            convertFile(input);
        } else if (input.isDirectory()) {
            paths.setInputRoot(input);
            convertFolder(input);
        }

        stop_time = System.currentTimeMillis();

        if (paths.isSaving()) {
            if (paths.isSaveWithInput()) {
                log.info(
                        "Output can be accessed at from the input folder {} in 'xtext' sub-folders",
                        input.getParent());
            } else {
                log.info("Output can be accessed at " + paths.getConversionCache());
            }
        }

        reportStatistics();
    }

    /**
     * Filter out File object if it is an XText conversion of some sort. That is, if
     * file "./a/b/c/xtext/file.doc.txt  is found, it is omitted because it is contained in "./xtext"
     *
     * @param input file obj
     * @return true if file's immediate parent is named 'xtext'
     */
    private boolean filterOutFile(File input) {
        //
        //
        if (PathManager.isXTextCache(input)) {
            return true;
        }

        return filterOutFile(input.getAbsolutePath());
    }

    /**
     * Filter the type of files to ignore.
     */
    @Override
    public boolean filterOutFile(String filepath) {

        // Filter out any of our own xtext caches
        //
        if (PathManager.isXTextCache(filepath)) {
            return true;
        }

        String n = FilenameUtils.getBaseName(filepath);
        if (n.startsWith(".")) {
            return true;
        }
        if (filepath.contains(".svn")) {
            return true;
        }

        // ignore '-utf8.txt' as XText likely generated them.
        //
        if (n.endsWith(ConvertedDocument.CONVERTED_TEXT_EXT)) {
            return true;
        }

        String ext = FilenameUtils.getExtension(filepath).toLowerCase();
        return !requestedFileTypes.contains(ext);
    }

    /**
     * Unpack an archive and convert items found.
     * Given (input)/A.zip
     * The zip is dearchived to
     *    (input)/A_zip/
     * or (archive)/(input)/A_zip
     *
     * Items are then converted in either folder for the conversion archiving; depending on your choice of embedded vs. non-embedded
     * @throws ConfigException
     *
     */
    public void convertArchive(File input) throws IOException, ConfigException {

        if (!paths.verifyArchiveExport(input.getAbsolutePath())) {
            return;
        }

        File saveFolder = paths.getArchiveExportDir(input);
        String savePrefix = paths.getStipPrefixPath();

        paths.setStripPrefixPath(saveFolder.getAbsolutePath());
        paths.setInputRoot(saveFolder);

        ArchiveNavigator deArchiver = new ArchiveNavigator(input, saveFolder.getAbsolutePath(),
                this, this);
        deArchiver.overwrite = ConvertedDocument.overwrite;

        log.info("\tArchive Found ({}). Expanding to {}", input, saveFolder);

        deArchiver.collect();

        // Done:
        paths.setStripPrefixPath(savePrefix);
    }

    /**
     *
     * @param input
     * @throws IOException
     */
    public void convertOutlookPST(File input) throws ConfigException, IOException {
        if (!paths.isSaving()) {
            log.error("Warning -- PST file found, but save = true is required to parse it.  Enable saving and chose a cache folder");
        }

        OutlookPSTCrawler pst = new OutlookPSTCrawler(input);
        pst.setConverter(this);
        pst.overwriteMode = ConvertedDocument.overwrite;
        pst.incrementalMode = true;

        File saveFolder = paths.getArchiveExportDir(input);
        String savePrefix = paths.getStipPrefixPath();

        paths.setStripPrefixPath(saveFolder.getAbsolutePath());
        paths.setInputRoot(saveFolder);
        pst.setOutputPSTDir(saveFolder);
        pst.configure();

        log.info("\tPST Email Archive Found ({}). Expanding to {}", input, saveFolder);

        try {
            pst.collect();
        } catch (Exception err) {
            throw new ConfigException("Unable to fully digest PST file " + input, err);
        }

        // Done:
        paths.setStripPrefixPath(savePrefix);
    }

    /**
     * Arbitrary 32 MB limit on file size. Maybe this should be dependent on the
     * file type.
     */
    public final static long FILE_SIZE_LIMIT = 0x2000000;

    /**
     * This is the proxy interface for traversing archives.
     *
     * Archive Navigator will call this interface to convert and post-process So
     * XText itself is a super-converter, whereas the items in the converter pkg
     * are stateless, simple conversions.
     *
     * this interface implementation calls XText.convertFile() which in turn
     * deals with the details of saving and archiving items
     *
     * Items retrieved from Archive Navigator are deleted from their temp space.
     * @throws ConfigException
     */
    @Override
    public ConvertedDocument convert(File input) throws IOException, ConfigException {
        return convertFile(input);
    }

    /**
     * Unsupported iConvert interface method. To convert text from a String obj
     * rather than a File obj, you would instantiate a converter implementation
     * for the data you think you are converting. E.g., if you know you have a
     * buffer of HTML content and want to save it as text, call
     * TikaHTMLConverter().convert( buffer ) directly.
     *
     */
    @Override
    public ConvertedDocument convert(String data) throws IOException {
        throw new IOException("Unsupported interface:  To convert text or binary data directly "
                + "you must use an instance of a XText converter, e.g., TikaHTMLConverter");
    }

    public ConvertedDocument convertFile(File input) throws IOException, ConfigException {
        return convertFile(input, null);
    }

    /**
     * Convert one file and save it off. We ignore hidden files and files in
     * hidden folders, e.g., .cvs_ignore, mycode/.svn/abc.txt
     *
     * This is the end of the line for the conversion logic; convertFile figures
     * out if it should return the cached version or attempt a conversion; it
     * also tries to save children items As children items may require special
     * attention they are not converted -- caller can pass in ConversionListener
     * and can deal with children file objects on their end.
     *
     * @param input
     * @return converted document object
     * @throws java.io.IOException
     */
    public ConvertedDocument convertFile(File input, ConvertedDocument parent) throws IOException,
            ConfigException {

        if (parent == null && filterOutFile(input)) {
            return null;
        }

        if (paths.isSaving()) {
            if (!paths.isSaveWithInput() && !paths.hasInputRoot()) {

                throw new IOException(
                        "Please set an input root; convertFile() was called in save/cache mode without having PathManager setup");
            }
        }

        String fname = input.getName();

        String ext = FilenameUtils.getExtension(fname).toLowerCase();
        if (ignoreFileTypes.contains(ext)) {
            return null;
        }

        if (!requestedFileTypes.contains(ext)) {
            return null;
        }

        log.debug("Converting FILE=" + input.getAbsolutePath());

        /*
         * Handle archives or PST files. Or other large compound single file.
         */
        if (isArchive(fname)) {
            convertArchive(input);

            // NULL here implies the actual file, A.zip does not have any text representation itself.
            // However its children do.
            return null;
        } else if (isPSTExtension(ext) && !useTikaPST) {
            convertOutlookPST(input);
            return null;
        }

        /*
         * Otherwise this is a normal file...
         */
        if (FileUtils.sizeOf(input) > maxFileSize) {
            log.info("Valid File is too large FILE=" + input.getAbsolutePath());
            return null;
        }

        boolean cachable = true;
        Converter converter = converters.get(ext);
        if (converter == null) {
            if (extractEmbedded && EmbeddedContentConverter.isSupported(ext)) {
                converter = embeddedConversion;
                cachable = false; // Such content is processed every time.  Oh well...
            } else {
                converter = defaultConversion;
            }
        }

        ConvertedDocument textDoc = null;

        // ------------------
        // Retrieve previous conversions
        // ------------------
        if (cachable && !ConvertedDocument.overwrite && paths.isSaving()) {
            textDoc = paths.getCachedConversion(input);
        }

        // ------------------
        // Convert or Read object, IFF no cache exists for that object.
        // ------------------
        if (textDoc == null) {
            // Measure how long conversions take.
            long t1 = System.currentTimeMillis();

            try {
                textDoc = converter.convert(input);
            } catch (Exception convErr) {
                throw new IOException("Conversion error FILE=" + input.getPath(), convErr);
            }
            long t2 = System.currentTimeMillis();
            int duration = (int) (t2 - t1);
            if (textDoc != null) {
                // Buffer can be null. If you got this far, you are interested
                // in the file, as it passed
                // all filters above. Return the document with whatever metadata
                // it found.
                // if (textDoc.buffer == null) {
                // throw new
                // IOException("Engineering error: Doc converted, but converter failed to setText()");
                // }
                if (paths.isSaving() && textDoc.is_converted) {
                    // Get Parent info in there.
                    if (parent != null) {
                        textDoc.setParent(parent);
                    }

                    paths.saveConversion(textDoc);

                    // Children items will be persisted in the same folder
                    // structure where the textdoc.textpath resides.
                    // That is, Email or Embedded objects will be parsed are
                    // saved in ./xtext/ folder or in the separate archive.
                    // But this must be down now, as we have all the dynamic
                    // metadata + raw artifacts; As it is all written out to
                    // disk,
                    // it will be written out together.
                    //
                    if (textDoc.hasRawChildren()) {
                        convertChildren(textDoc);

                        // 1. children saved to disk
                        // 2. children converted.
                        // 3. children attached to parent here.
                        // 'textdoc' should now be well endowed with all the
                        // children metadata.
                    }
                }
            } else {
                textDoc = new ConvertedDocument(input);
            }

            textDoc.conversion_time = duration;
            if (textDoc.filetime == null) {
                textDoc.filetime = textDoc.getFiletime();
            }
        }

        /*
         * Conversion Listeners are called only for parent documents. That is
         * for an email with 4 attachments, this listener is called on the
         * parent email message, but not for the individual 4 attachments. The
         * final parent document here will have all Raw Children (bytes +
         * metadata) and Converted Children (ConvertedDocument obj) Caller will
         * have to detect if returned item via listener is a Parent with
         * Children.
         *
         * Behavior here is TBD.
         */
        if (postProcessor != null && parent == null) {
            postProcessor.handleConversion(textDoc, input.getAbsolutePath());
        }

        trackStatistics(textDoc);
        return textDoc;
    }

    /**
     * Navigate a folder trying to convert each file and return something to the
     * listener. Do not sacrifice the entire job if one file fails, so exception
     * is trapped in loop
     *
     */
    public void convertFolder(File input) throws IOException {
        java.util.Collection<File> files = FileUtils.listFiles(input, new SuffixFileFilter(
                FILE_FILTER, IOCase.INSENSITIVE), FileFilterUtils.trueFileFilter());
        for (File f : files) {
            try {
                convertFile(f);
            } catch (Exception convErr) {
                log.error("Conversion error, FILE=" + f.getPath(), convErr);
            }
        }
    }

    /**
     * Save children objects for a given ConvertedDocument to a location....
     * convert those items immediately, saving the Parent metadata along with
     * them. You should have setParent already
     *
     * @param parentDoc
     * @throws IOException
     */
    public void convertChildren(ConvertedDocument parentDoc) throws IOException {

        if (parentDoc.is_webArchive) {
            // Web Archive is a single document.  Only intent here is to convert to a single text document.
            //
            return;
        }

        parentDoc.evalParentChildContainer();
        FileUtility.makeDirectory(parentDoc.parentContainer);
        String targetPath = parentDoc.parentContainer.getAbsolutePath();

        for (Content child : parentDoc.getRawChildren()) {
            if (child.content == null) {
                log.error("Attempted to write out child object with no content {}", child.id);
                continue;
            }

            OutputStream io = null;
            try {
                // We just assume for now Child ID is filename.
                // Alternatively, child.meta.getProperty(
                // ConvertedDocument.CHILD_ENTRY_KEY )
                // same result, just more verbose.
                //
                File childFile = new File(FilenameUtils.concat(targetPath, child.id));
                io = new FileOutputStream(childFile);
                IOUtils.write(child.content, io);

                ConvertedDocument childConv = convertFile(childFile, parentDoc);
                if (childConv != null) {
                    if (childConv.is_converted) {
                        // Push down all child metadata down to ConvertedDoc
                        for (String k : child.meta.stringPropertyNames()) {
                            String val = child.meta.getProperty(k);
                            childConv.addUserProperty(k, val);
                        }
                        // Save cached version once again.
                        childConv.saveBuffer(new File(childConv.textpath));
                    }

                    if (child.mimeType != null) {
                        try {
                            childConv.setMimeType(new MimeType(child.mimeType));
                        } catch (MimeTypeParseException e) {
                            log.warn("Invalid mime type encountered: {} ignoring.", child.mimeType);
                        }
                    }

                    parentDoc.addChild(childConv);
                }
            } catch (Exception err) {
                log.error("Failed to write out child {}, but will continue with others", child.id,
                        err);
            } finally {
                if (io != null) {
                    io.close();
                }
            }
        }
    }

    /**
     * TODO: this is called by default. duh. To change behavior, adjust
     * settings before setup() is called
     */
    public void defaults() {

        archiveFileTypes.add("zip");
        archiveFileTypes.add("gz");
        archiveFileTypes.add("tar");
        archiveFileTypes.add("tgz");
        archiveFileTypes.add("tar.gz");
        // archive_types.add("7z");

        // Get from a config file.
        requestedFileTypes.add("doc");
        requestedFileTypes.add("docx");
        requestedFileTypes.add("pdf");
        requestedFileTypes.add("htm");
        requestedFileTypes.add("html");
        requestedFileTypes.add("txt"); // only for encoding conversions.
        requestedFileTypes.add("msg");
        requestedFileTypes.add("eml");
        requestedFileTypes.add("emlx");
        requestedFileTypes.add("ppt");
        requestedFileTypes.add("pptx");
        requestedFileTypes.add("xlsx");
        requestedFileTypes.add("xls");
        requestedFileTypes.add("rtf");

        // Testing:
        requestedFileTypes.add("dot");
        requestedFileTypes.add("dotx");
        requestedFileTypes.add("odt");
        requestedFileTypes.add("odf");
        requestedFileTypes.add("docm");

        // Web Archives.
        requestedFileTypes.add("mht");
        //requestedFileTypes.add("wps");  MS Works?  No tika support really.

        // Only Photographic images will be supported by default.
        // BMP, GIF, PNG, ICO, etc. must be added by caller.
        //
        requestedFileTypes.add("jpg");
        requestedFileTypes.add("jpeg");

        // Limited PST support here.  PST will not behave the same as other files.
        // Its closer to a Zip archive than an ordinary file.
        requestedFileTypes.add("pst");

        // requested_types.add("log"); // Uncommon. Caller must expclitly add
        // raw data types and archives.
    }

    /**
     * Start over.
     */
    public void clearSettings() {
        requestedFileTypes.clear();
        converters.clear();
    }

    /**
     * If by this point you have taken items out of the requested types the
     * converters will not be setup. E.g., if you don't want PDF or HTML
     * conversion - those resources will not be initialized.
     */
    public void setup() throws IOException {

        defaultConversion = new DefaultConverter(maxBuffer);
        embeddedConversion = new EmbeddedContentConverter(maxBuffer);

        paths.configure();

        // Invoke converter instances only as requested types suggest.
        // If caller has removed file types from the list, then

        String mimetype = "txt";
        if (requestedFileTypes.contains(mimetype)) {
            converters.put(mimetype, new TextTranscodingConverter());
        }

        mimetype = "html";
        if (requestedFileTypes.contains(mimetype)) {
            Converter webConv = new TikaHTMLConverter(this.scrubHTML, maxHTMLBuffer);
            converters.put(mimetype, webConv);
            converters.put("htm", webConv);
            converters.put("xhtml", webConv);

            requestedFileTypes.add("htm");
            requestedFileTypes.add("xhtml");
        }

        MessageConverter emailParser = new MessageConverter();
        mimetype = "eml";
        if (requestedFileTypes.contains(mimetype)) {
            converters.put(mimetype, emailParser);
        }
        mimetype = "msg";
        if (requestedFileTypes.contains(mimetype)) {
            converters.put(mimetype, emailParser);
        }
        WebArchiveConverter webArchiveParser = new WebArchiveConverter();
        mimetype = "mht"; /* RFC822 */
        if (requestedFileTypes.contains(mimetype)) {
            converters.put(mimetype, webArchiveParser);
        }

        ImageMetadataConverter imgConv = new ImageMetadataConverter();
        String[] imageTypes = { "jpeg", "jpg" };
        for (String img : imageTypes) {
            if (requestedFileTypes.contains(img)) {
                converters.put(img, imgConv);
            }
        }

        // ALWAYS ignore our own text conversions or those of others.
        // So here all known convertable types will need a filter for their
        // conversion, e.g.,
        // pdf => ignore pdf.txt
        // doc => ignore doc.txt
        //
        for (String t : requestedFileTypes) {
            ignoreFileType(t + ".txt");
        }

        FILE_FILTER = requestedFileTypes.toArray(new String[requestedFileTypes.size()]);
    }

    /**
     *
     */
    private String[] FILE_FILTER = null;

    /**
     * Call after setup() has run to add all supported/requested file types
     */
    public Set<String> getFileTypes() {
        return requestedFileTypes;
    }

    public static void usage() {
        System.out.println();
        System.out.println("==========XText Usage=============");
        System.out
                .println("XText --input input  [--output output]|[--embed-conversion]    "
                        + "\n\t[--embed-children]|[--export folder]     [--clean-html]   [--strip-prefix path]");
        System.out.println("  input is file or folder");
        System.out.println("  output is a folder where you want to archive converted docs");
        System.out.println("  -e embeds the saved conversions in the input folder under 'xtext/'");
        System.out.println("  -c embeds the extracted children binaries in the input folder");
        System.out.println("     (NOT the conversions, the binaries from Archives, PST, etc)");
        System.out.println("     Default behavior is to extract originals to output archive.");
        System.out.println("  -x folder\tOpposite of -c. Extract children and save to <folder>");
        System.out.println("  NOTE: -e has same effect as setting output to input");
        System.out.println("  -H enables HTML scrubbing");
        System.out.println("========================");
    }

    /**
     * Purely for logging when using the cmd line variation.
     *
     * @author ubaldino
     *
     */
    static class MainProgramListener implements ConversionListener {

        private Logger log = LoggerFactory.getLogger(getClass());

        @Override
        public void handleConversion(ConvertedDocument doc, String path) {
            boolean converted = false;
            if (doc != null) {
                converted = doc.is_converted;
            }
            log.info("Converted. FILE={} Status={}, Converted={}", path, doc != null, converted);
        }
    }

    public static void main(String[] args) {

        LongOpt[] options = { new LongOpt("input", LongOpt.REQUIRED_ARGUMENT, null, 'i'),
                new LongOpt("output", LongOpt.REQUIRED_ARGUMENT, null, 'o'),
                new LongOpt("export", LongOpt.REQUIRED_ARGUMENT, null, 'x'),
                new LongOpt("strip-prefix", LongOpt.REQUIRED_ARGUMENT, null, 'p'),
                new LongOpt("clean-html", LongOpt.NO_ARGUMENT, null, 'H'),
                new LongOpt("embed-conversion", LongOpt.NO_ARGUMENT, null, 'e'),
                new LongOpt("embed-children", LongOpt.NO_ARGUMENT, null, 'c'),
                new LongOpt("tika-pst", LongOpt.NO_ARGUMENT, null, 'T') };

        // "hcex:i:o:p:"
        gnu.getopt.Getopt opts = new gnu.getopt.Getopt("XText", args, "", options);

        String input = null;
        String output = null;
        boolean embed = false;
        boolean filter_html = false;
        boolean saveChildrenWithInput = false;
        String saveChildrenTo = null;
        String prefix = null;

        XText xt = new XText();

        try {
            int c;
            while ((c = opts.getopt()) != -1) {
                switch (c) {

                case 0:
                    // Long opt processed.

                    break;

                case 'i':
                    input = opts.getOptarg();
                    break;
                case 'o':
                    output = opts.getOptarg();
                    break;
                case 'H':
                    filter_html = true;
                    break;
                case 'c':
                    saveChildrenWithInput = true;
                    break;
                case 'x':
                    saveChildrenTo = opts.getOptarg();
                    break;
                case 'p':
                    prefix = opts.getOptarg();
                    break;
                case 'e':
                    embed = true;
                    System.out
                            .println("Saving conversions to Input folder.  Output folder will be ignored.");
                    break;
                case 'T':
                    xt.enableTikaPST(true);
                    break;
                default:
                    XText.usage();
                    System.exit(1);
                }
            }
        } catch (Exception err) {
            XText.usage();
            System.exit(1);
        }

        if (input == null) {
            System.out.println("An input argument is required, e.g., -Dinput=/Folder/...");
            System.exit(-1);
        }

        // Setting LANG=en_US in your shell.
        //
        // System.setProperty("LANG", "en_US");

        xt.enableOverwrite(true); // Given this is a test application, we will
                                  // overwrite every time XText is called.
        xt.enableSaving(embed || output != null);
        xt.getPathManager().enableSaveWithInput(embed); // creates a ./text/ Folder locally in
        // directory.
        xt.enableHTMLScrubber(filter_html);
        xt.getPathManager().enableSaveChildrenWithInput(saveChildrenWithInput);

        // If user wishes to strip input paths of some prefix
        // Output will be dumped in the resulting relative path.
        xt.getPathManager().setStripPrefixPath(prefix);

        // Manage the extraction of compound files -- archives, PST mailbox file, etc.
        // ... others?
        if (!saveChildrenWithInput && saveChildrenTo != null) {
            xt.getPathManager().setExtractedChildrenCache(saveChildrenTo);
        }

        try {
            if (output == null && !embed) {
                output = "xtext-output";
                FileUtility.makeDirectory(output);
                System.out.println("Default output folder is $PWD/" + output);
            }
            // Notice this main program requires an output path.
            if (output != null) {
                xt.getPathManager().setConversionCache(output);
            }
            // Set itself to listen, as this is the main program.
            xt.setConversionListener(new MainProgramListener());

            xt.setup();
            xt.extractText(input);
        } catch (IOException ioerr) {
            XText.usage();
            ioerr.printStackTrace();
        } catch (ConfigException cfgerr) {
            XText.usage();
            cfgerr.printStackTrace();
        }
    }
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Http;
using System.Web.Mvc;
using System.Web.Optimization;
using System.Web.Routing;
// more...
using AutoMapper;

namespace ColourList
{
    public class WebApiApplication : System.Web.HttpApplication
    {
        protected void Application_Start()
        {
            AreaRegistration.RegisterAllAreas();
            GlobalConfiguration.Configure(WebApiConfig.Register);
            FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
            RouteConfig.RegisterRoutes(RouteTable.Routes);
            BundleConfig.RegisterBundles(BundleTable.Bundles);

            // Load the data store initializer (for simple projects)

            System.Data.Entity.Database.SetInitializer(new Models.StoreInitializer());

            // AutoMapper maps
            // These define mappings between 'design model' and 'resource model' classes
            // Remember, we never work with 'design model' classes in our controllers
            // and end-user 'use cases'

            Mapper.CreateMap<Models.Colour, Controllers.ColourBase>();

            Mapper.CreateMap<Controllers.ColourAdd, Models.Colour>();

        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
This desktop app is made to export large capture from custom map styles made in [Mapbox Studio](https://www.mapbox.com/studio/).

# Install
## On a Mac
You can download the [released version for Mac](https://github.com/jonathanlurie/themap/releases/download/v1.0.0/The_Map_mac10.11.zip). It comes as a *zip* file containing a *dmg*. Then launch it like any other app [but first, make sure you allowed unsigned apps to run on your mac](https://kb.wisc.edu/helpdesk/page.php?id=25443).

## On Linux
### First
This app is for Linux and Mac + you need to have [Imagemagick](https://www.imagemagick.org/script/download.php) installed (and visible, aka. in your PATH)

### Dependencies and launching
1. Download the [zip](https://github.com/jonathanlurie/themap/archive/master.zip) or clone this repository.
2. With a terminal, `cd` into the app folder
3. `$ npm install` to install all the dependencies
4. `$ npm start` to launch the app

## On Windows
Not sure about this one. With a bit of luck you can easily access Imagemagick from you `PATH`environment variable and then it should be pretty similar to the Linux way. Good luck!

# Use The Map
## Configure it
In the option Menu, you can set the `Mapbox token`. Please, use your own Mapbox token, the one given is just for the example. You can fing when you registered on Mapbox.com (free).

Then, to use your own map styles, you will have to write you `Mapbox user id` and the map `style id`. You can find both when using Mapbox Studio, in the URL:

`https://www.mapbox.com/studio/styles/jonathanlurie/cj61518sb28vf2rltdqx9lgg9/edit/`

Here, the user id is: **jonathanlurie** and the map style id is **cj61518sb28vf2rltdqx9lgg9**.

## Define an area
When the app is ready and your credentials are working fine, it's time to draw the area you want to export! To do so:
1. Browse the map in quest of a nice crop to export
2. Adjust your **zoom level** using the `+/-` button on the top left. The zoom level is between 0 and 22 and show on the title bar of the app.
3. Use the **black-square** icon on the top left of the app. You are now in area drawing mode!
4. Click somewhere on the map and drag to enlarge the area as it pleases you
5. If you are not happy with the area you just drew, you can modify or delete if using the **pen** or **trashbin** icon. Don't forget to clisck **save** after!
6. When your selection is ready, open the menu from the **three-bar** icon at the bottom
7. Select your quality (512 will usually give you a good quality for printing, 256 is good for screen and 1024 is unnecessary large, and it will take more time to process. Just go with 512)

## Capture
Now that you have a nice area, you are *almost* ready to export your map, Just click on **launch capture**!

You also need to know the **zoom level** of the final image is the one you where using in the app when you clicked on the **launch capture** button.

While **The Map** is at work, a live status of things is updated in the title bar, on the top of the app. Here is how it goes:
1. Downloading every map tiles (512x512 or 1024x1024 images, depending on the option chosen)
2. Merging some tiles together to get horizontal strips
3. Merging all the horizontal strip together to get the full image
4. Opening the folder where your final image is

Start with a small test area first, so that you are sure it works.

## Good to know
**The Map** need to store temporary data on your local storage while working. This happens in your home directory, in a (hidden) folder called `.themap`.
This folder contains:
- an important config file: `config.json`, that remembers the last position you used, the last zoom level, the last map style and your Mapbox credential. Only things you already know but are very convenient to have automatically pre-filed.
- a folder: `working`, this is where all the temporary tiles and strips are downloaded and exported. This folder is flushed at every launch of the app and again after every successful map building.

If you decide to remove this folder, it will be recreated the next time you launch **The Map** but you will have to rewrite your credential, your map style and your last position will be lost.


text:
using UnityEngine;
using UnityEngine.UI;
using System.Collections;
using UnityEngine.SceneManagement;

public class GameUIController : MonoBehaviour {
  public Image fadePlane;
  public GameObject gameOverUI;
  public RectTransform banner;
  public Text title;
  public Text subtitle;

  void Awake() {
    FindObjectOfType<Player>().OnDeath += OnGameOver;
    FindObjectOfType<Spawner>().OnNewWave += OnNewWave;
  }

  public void StartNewGame() {
    SceneManager.LoadScene("Main");
  }

  void OnNewWave(Wave wave, int waveCount) {
    title.text = "WAVE " + (waveCount + 1);
    subtitle.text = wave.entityCount + " enemies";
    StartCoroutine(AnimateBanner(1.5f, 2.0f));
  }

  void OnGameOver() {
    gameOverUI.SetActive(true);
    StartCoroutine(Fade(Color.clear, Color.black, 1.0f));
  }

  IEnumerator AnimateBanner(float speed, float delay) {
    const int offScreenYPosition = -460;
    const int onScreenYPosition = -200;
    int direction = 1;
    float percentCompleted = 0.0f;
    float restartExecutionTime = Time.time + 1 / speed + delay;

    while (percentCompleted >= 0) {
      percentCompleted += Time.deltaTime * speed * direction;

      if (percentCompleted >= 1) {
        percentCompleted = 1;

        if (Time.time > restartExecutionTime) {
          direction = -1;
        }
      }

      banner.anchoredPosition = Vector2.up * Mathf.Lerp(
        offScreenYPosition, onScreenYPosition, percentCompleted
      );

      yield return null;
    }
  }

  IEnumerator Fade(Color fromColor, Color toColor, float duration) {
    float speed = 1.0f / duration;
    float percentCompleted = 0.0f;

    while (percentCompleted < 1.0f) {
      percentCompleted += Time.deltaTime * speed;
      fadePlane.color = Color.Lerp(fromColor, toColor, percentCompleted);
      yield return null;
    }
  }
}


text:
A simple plugin interface with YouTube for October CMS.

## Setup

1. Clone or download the repo into ```plugins/bluhex/youtube```.
2. Run ``composer update`` to get the latest google client.

## Usage

### Settings

- ```API Key```: YouTube API key is needed to access the API.
- ```Minutes cached```: Time in minutes that the results should be cached for
##### Getting an API Key

1. Head on over to https://console.developers.google.com/project and either select your existing project or create a new one.
2. Open the side bar and select 'API Manager'
3. Select or search for YouTube Data API
4. Click on 'Enable'
5. Select 'Credentials' from the sidebar
6. Select 'Create credentials' -> 'API key' -> 'Server key'
7. Name the key whatever you like and enter your IP address into the 'Accept requests from these server IP addresses' and create
8. Copy the new API Key into the settings and save.

### Latest videos

Use the ```Latest Videos``` component to display a list of the latest videos for a channel. The component has the following properties available for config:

- ```channel_id```: The YouTube channel id can be found over at https://www.youtube.com/account_advanced.

- ```max_items```: Maximum number of videos to display, at this time there is no pagination support.

- ```thumb_resolution```: The resolution of the thumbnail. YouTube isn't great about their thumbnails so there are 4 options:
 - Default: Thumbnail is resized into 120 x 90 with black bars added if needed to fill the aspect ratio.
 - Medium (default option): Thumbnail is resized into 320 x 180 with black bars added if needed to fill the aspect ratio.
 - High: Thumbnail is resized into 480 x 360 with black bars added if needed to fill the aspect ratio.
 - Full Resolution is the absolute highest resolution provided by YouTube. This method isn't part of the API, but is the closest to the original thumbnail uploaded by the user.

### Creating a custom partial

Under the ```CMS > Partials``` menu, create a new partial called ``` latestVideos/default.htm ```.

Inside the videos array each video object contains the following:

- ``link``: URL of the video
- ``title``: Video title
- ``thumbnail``: URL of the maximum resolution version of the thumbnail
- ``description``: Excerpt of the video description
- ``published_at``: Carbon date of the publish date

#### Simple partial

``` twig
{% set videos = __SELF__.videos %}
{% for video in videos %}
  <a href="{{ video.link }}">
    <img src="{{ video.thumbnail }}" alt="{{ video.title }}">
    <span>{{ video.title }}</span>
  </a>
{% endfor %}
```


text:
package org.conscrypt;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.security.Provider;
import java.security.Security;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import javax.net.ssl.SSLContext;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

@RunWith(JUnit4.class)
public class ConscryptTest {

    /**
     * This confirms that the version machinery is working.
     */
    @Test
    public void testVersionIsSensible() {
        Conscrypt.Version version = Conscrypt.version();
        assertNotNull(version);
        // The version object should be a singleton
        assertSame(version, Conscrypt.version());

        assertTrue("Major version: " + version.major(), 1 <= version.major());
        assertTrue("Minor version: " + version.minor(), 0 <= version.minor());
        assertTrue("Patch version: " + version.patch(), 0 <= version.patch());
    }

    @Test
    public void testProviderBuilder() throws Exception {
        Provider p = Conscrypt.newProviderBuilder()
            .setName("test name")
            .provideTrustManager(true)
            .defaultTlsProtocol("TLSv1.2").build();

        assertEquals("test name", p.getName());
        assertTrue(p.containsKey("TrustManagerFactory.PKIX"));

        try {
            Security.insertProviderAt(p, 1);

            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, null, null);
            assertEquals(p, context.getProvider());
            Set<String> expected = new HashSet<>(Arrays.asList("TLSv1.2", "TLSv1.1", "TLSv1"));
            Set<String> found =
                new HashSet<>(Arrays.asList(context.createSSLEngine().getEnabledProtocols()));
            assertEquals(expected, found);

            context = SSLContext.getInstance("Default");
            assertEquals(p, context.getProvider());
            expected = new HashSet<>(Arrays.asList("TLSv1.2", "TLSv1.1", "TLSv1"));
            found = new HashSet<>(Arrays.asList(context.createSSLEngine().getEnabledProtocols()));
            assertEquals(expected, found);
        } finally {
            Security.removeProvider("test name");
        }

        p = Conscrypt.newProviderBuilder()
            .setName("test name 2")
            .provideTrustManager(false)
            .defaultTlsProtocol("TLSv1.3").build();

        assertEquals("test name 2", p.getName());
        assertFalse(p.containsKey("TrustManagerFactory.PKIX"));

        try {
            Security.insertProviderAt(p, 1);

            SSLContext context = SSLContext.getInstance("TLS");
            context.init(null, null, null);
            assertEquals(p, context.getProvider());
            Set<String> expected =
                new HashSet<>(Arrays.asList("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1"));
            Set<String> found =
                new HashSet<>(Arrays.asList(context.createSSLEngine().getEnabledProtocols()));
            assertEquals(expected, found);

            context = SSLContext.getInstance("Default");
            assertEquals(p, context.getProvider());
            expected = new HashSet<>(Arrays.asList("TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1"));
            found = new HashSet<>(Arrays.asList(context.createSSLEngine().getEnabledProtocols()));
            assertEquals(expected, found);
        } finally {
            Security.removeProvider("test name 2");
        }

        try {
            Conscrypt.newProviderBuilder()
                .defaultTlsProtocol("invalid").build();
            fail();
        } catch (IllegalArgumentException expected) {
        }
    }
}


text:
require 'shellwords'

module Ffprober
  module Ffmpeg
    class Exec
      extend T::Sig

      CHAPTER_SUPPORT = T.let(Gem::Version.new('2.0.0'), Gem::Version)

      sig { params(finder: T.any(Ffprober::Ffmpeg::Finder, T.untyped)).void }
      def initialize(finder = Ffprober::Ffmpeg::Finder.new)
        @finder = finder
        @ffprobe_version_output = T.let(nil, T.nilable(String))
      end

      sig { params(filename: String).returns(String) }
      def json_output(filename)
        `#{@finder.path} #{ffprobe_options} #{Shellwords.escape(filename)}`
      end

      sig { returns(String) }
      def ffprobe_version_output
        @ffprobe_version_output ||= if @finder.path.nil?
                                      ''
                                    else
                                      `#{@finder.path} -version`
                                    end
      end

      sig { returns(String) }
      def ffprobe_options
        base_options = '-v quiet -print_format json -show_format ' \
                       '-show_streams -show_error'

        options = "#{base_options} -show_chapters" if ffprobe_version.version >= CHAPTER_SUPPORT

        options || base_options
      end

      sig { returns(Ffprober::Ffmpeg::Version) }
      def ffprobe_version
        Ffprober::Ffmpeg::Version.new(self)
      end
    end
  end
end