text:
package org.fhcrc.cpl.viewer.quant.commandline;

import org.fhcrc.cpl.viewer.commandline.modules.BaseViewerCommandLineModuleImpl;
import org.fhcrc.cpl.viewer.commandline.ViewerCommandModuleUtilities;
import org.fhcrc.cpl.viewer.quant.QuantEventAssessor;
import org.fhcrc.cpl.viewer.quant.QuantEvent;
import org.fhcrc.cpl.viewer.quant.turk.TurkUtilities;
import org.fhcrc.cpl.viewer.quant.gui.QuantitationVisualizer;
import org.fhcrc.cpl.viewer.quant.gui.PanelWithSpectrumChart;
import org.fhcrc.cpl.toolbox.commandline.arguments.*;
import org.apache.log4j.Logger;

import org.fhcrc.cpl.toolbox.ApplicationContext;
import org.fhcrc.cpl.toolbox.statistics.BasicStatistics;
import org.fhcrc.cpl.toolbox.datastructure.Pair;
import org.fhcrc.cpl.toolbox.filehandler.TempFileManager;
import org.fhcrc.cpl.toolbox.gui.chart.PanelWithHistogram;
import org.fhcrc.cpl.toolbox.gui.chart.PanelWithScatterPlot;
import org.fhcrc.cpl.toolbox.gui.chart.PanelWithLineChart;
import org.fhcrc.cpl.toolbox.commandline.CommandLineModuleExecutionException;
import org.fhcrc.cpl.toolbox.commandline.CommandLineModule;
import org.fhcrc.cpl.toolbox.commandline.CommandLineModuleUtilities;
import org.fhcrc.cpl.toolbox.proteomics.*;
import org.fhcrc.cpl.toolbox.proteomics.feature.FeatureSet;
import org.fhcrc.cpl.toolbox.proteomics.feature.Feature;
import org.fhcrc.cpl.toolbox.proteomics.feature.filehandler.PepXMLFeatureFileHandler;
import org.fhcrc.cpl.toolbox.proteomics.feature.extraInfo.MS2ExtraInfoDef;
import org.fhcrc.cpl.toolbox.proteomics.feature.extraInfo.IsotopicLabelExtraInfoDef;

import java.io.*;
import java.util.*;
import java.util.List;

/**
 * This uses a big HACK, adding a dummy search score to peptide features to store the flag reason description
 *
 */
public class FlagQuantEventsCLM extends BaseViewerCommandLineModuleImpl
        implements CommandLineModule
{

    //Controls whether the badScans list is used to calculate sensitivity and specificity.
    //This is for refining flagquant
    protected boolean shouldCalcSensSpecWithHardcodedScans = false;

    protected static final String FEATURE_PROPERTY_QUANTASSESSMENT = "FLAGQUANT_QUANTASSESSMENT";

    public static final String REASON_DUMMY_SEARCH_SCORE_NAME = "SEARCHSCORE_FLAGQUANT_REASON";

//This is scaffolding, for comparing against some known bad events.  Probably no longer effective
List<Integer> badScans = Arrays.asList(new Integer[] {    9603, 6106, 6177, 8008, 1062, 1130, 6923, 7176, 7917, 3139, 3141, 4457, 4673, 2694,});

    List<Float> flaggedReasons = new ArrayList<Float>();
    List<Integer> flaggedScans = new ArrayList<Integer>();
    List<Integer> badFlagged = new ArrayList<Integer>();

    List<Float> reasonsTruePos = new ArrayList<Float>();
    List<Float> reasonsFalsePos = new ArrayList<Float>();

    List<Float> logAlgRatiosWithSinglePeakRatios = new ArrayList<Float>();
    List<Float> weightedGeomMeanRatios = new ArrayList<Float>();

    QuantEventAssessor quantEventAssessor = new QuantEventAssessor();

    protected static Logger _log = Logger.getLogger(FlagQuantEventsCLM.class);

    protected File[] featureFiles;
    protected File outFlaggedFile;
    protected File outNoFlaggedFile;
    protected File outNoFlaggedDir;

    protected File outBadEventTurkFile;
    protected PrintWriter outBadEventTurkPW;
    protected List<QuantEvent> badQuantEvents = new ArrayList<QuantEvent>();
    protected File outBadEventQurateFile;
    protected File outTurkImageDir;

    protected File mzXmlDir;

    //scaffolding
    protected List<Float> lightPrevPeakRatios = new ArrayList<Float>();
    protected List<Float> heavyPrevPeakRatios = new ArrayList<Float>();

    //For calculating statistics on flagged vs non-flagged peptides
    Map<String, List<Pair<Float, Boolean>>> peptidePerFractionRatioFlaggedMap = new HashMap<String, List<Pair<Float,Boolean>>>();

    //all log-ratios and indications of whether flagged or not, for chart
    List<Pair<Float,Boolean>> allLogRatiosFlaggeds = new ArrayList<Pair<Float,Boolean>>();

    Map<String, Integer> peptideNumFractionsFlaggedMap = new HashMap<String, Integer>();
    List<Float> stDevsFromMeanRatioNoFlag = new ArrayList<Float>();
    List<Float> stDevsFromMeanRatioFlag = new ArrayList<Float>();
    List<Float> corrFlagged = new ArrayList<Float>();
    List<Float> corrUnFlagged = new ArrayList<Float>();
    //for sens, spec
    List<Float> corrBad = new ArrayList<Float>();
    List<Float> corrGood = new ArrayList<Float>();

    List<Float> logSinglePeakRatios = new ArrayList<Float>();

    protected boolean shouldWriteBadTurk;
    protected int badTurkId = 0;
    protected QuantitationVisualizer quantVisualizer = null;

    protected Map<String, List<Integer>> fractionScanListMapEventsToProcess = new HashMap<String, List<Integer>>();
    protected boolean onlyProcessSpecifiedEvents = false;

    //scaffolding for calculating ratios using regression based on one datapoint per-scan, like RelEx.
    //I think that method pretty much doesn't work very well.
//    List<Float> singlePeakSlopeRatios = new ArrayList<Float>();
//    List<Float> multiPeakSlopeRatios = new ArrayList<Float>();

    protected Set<String> peptidesWithGoodQuantEvents = new HashSet<String>();
    protected Set<String> proteinsWithGoodQuantEvents = new HashSet<String>();

    public FlagQuantEventsCLM()
    {
        init();
    }

    protected void init()
    {
        mCommandName = "flagquant";
        mShortDescription = "Flag questionable quantitation events";
        mHelpMessage = "Flag questionable quantitation events";
        CommandLineArgumentDefinition[] argDefs =
                {
                        createUnnamedSeriesFileArgumentDefinition(true, "pepXML file"),
                        new DirectoryToReadArgumentDefinition("mzxmldir", true, "mzXML directory"),
                        new FileToWriteArgumentDefinition("outflagged", false,
                                "Output pepXML file containing flagged events only"),
                        new DirectoryToWriteArgumentDefinition("outdir", false, "Output directory for flagged features"),
//                        new EnumeratedValuesArgumentDefinition("label", false, QuantitationUtilities.ALL_LABEL_CODES,
//                               QuantitationUtilities.ALL_LABEL_EXPLANATIONS, QuantitationUtilities.LABEL_LYCINE_CODE),
                        new BooleanArgumentDefinition("showcharts", false, "Show charts?", false),
                        new DecimalArgumentDefinition("minflagratio", false,
                                "Ratios must be higher than this, or lower than maxflagratio, or both, to flag",
                                0f),
                        new DecimalArgumentDefinition("maxflagratio", false,
                                "Ratios must be lower than this, or higher than minflagratio, or both, to flag",
                                999f),
                        new DecimalArgumentDefinition("peakppm", false,
                                "Mass tolerance around each theoretical peak (ppm)",
                                QuantEventAssessor.DEFAULT_PEAK_PPM_TOLERANCE),
                        new FileToWriteArgumentDefinition("outnoflagged", false,
                                "Output pepXML file containing all input features (with and without ratios) " +
                                        "EXCEPT flagged features"),
                        new DirectoryToWriteArgumentDefinition("outbadturkdir", false,
                                "Output directory for Mechanical Turk files for bad events"),
                        new DirectoryToWriteArgumentDefinition("outnoflaggeddir", false,
                                "Output directory for 'outnoflagged' files (for multiple input files)"),
                        //todo: remove this special-purpose arg
                        new FileToReadArgumentDefinition("quratetoprocess", false, "Qurate file.  If this file is present, only process events from this file"),

                };
        addArgumentDefinitions(argDefs);
    }

    public void assignArgumentValues()
            throws ArgumentValidationException
    {
        featureFiles = this.getUnnamedSeriesFileArgumentValues();
        outFlaggedFile = getFileArgumentValue("outflagged");
        File outDir = getFileArgumentValue("outdir");

        if (outFlaggedFile != null)
            assertArgumentAbsent("outdir", "out");

        if (featureFiles.length > 1 &&
                (hasArgumentValue("outdir") || hasArgumentValue("outnoflagged") || hasArgumentValue("outflagged")))
            throw new ArgumentValidationException("arguments ourdir, outflagged and outnoflagged can only be " +
                    "specified with a single input file");

        if (outDir != null)
        {
            outFlaggedFile = new File(outDir, featureFiles[0].getName());
        }

        outNoFlaggedFile = getFileArgumentValue("outnoflagged");
        outNoFlaggedDir = getFileArgumentValue("outnoflaggeddir");
        if (outNoFlaggedDir != null && featureFiles[0].getAbsolutePath().contains(outNoFlaggedDir.getAbsolutePath()))
            ApplicationContext.infoMessage("Warning: is outnoflaggeddir same as directory of input files?  This would cause problems");

        if (outNoFlaggedFile != null && outNoFlaggedDir != null)
             throw new ArgumentValidationException("only outnoflagged or outnoflaggeddir may be specified, not both");

        mzXmlDir = getFileArgumentValue("mzxmldir");

//        quantEventAssessor.setLabelType(((EnumeratedValuesArgumentDefinition)
//                getArgumentDefinition("label")).getIndexForArgumentValue(getStringArgumentValue("label")));

        quantEventAssessor.setPeakPPMTolerance(getFloatArgumentValue("peakppm"));
        quantEventAssessor.setShowCharts(getBooleanArgumentValue("showcharts"));
        quantEventAssessor.setMinFlagRatio(getFloatArgumentValue("minflagratio"));
        quantEventAssessor.setMaxFlagRatio(getFloatArgumentValue("maxflagratio"));
        if (hasArgumentValue("minflagratio") || hasArgumentValue("maxflagratio"))
            ApplicationContext.infoMessage("NOTE: only ratios higher than " + quantEventAssessor.getMinFlagRatio() +
                    " or lower than " + quantEventAssessor.getMaxFlagRatio() + " (or both) will be flagged");

        if (hasArgumentValue("outbadturkdir"))
        {
            shouldWriteBadTurk = true;
            outTurkImageDir = getFileArgumentValue("outbadturkdir");
            outBadEventTurkFile = new File(outTurkImageDir, "bad_event_turk.csv");
            outBadEventQurateFile = new File(outTurkImageDir, "bad_event_qurate.tsv");
            try
            {
                outBadEventTurkPW = new PrintWriter(outBadEventTurkFile);
                outBadEventTurkPW.println(TurkUtilities.createTurkHITFileHeaderLine());
            }
            catch (IOException e)
            {
                throw new ArgumentValidationException("Failed to open file for turk HITs", e);
            }
        }

        //This is pretty special-purpose.  Ignore events unless they appear in a qurate .tsv file
        //todo: remove this when no longer needed
        if (hasArgumentValue("quratetoprocess"))
        {
            onlyProcessSpecifiedEvents = true;
            fractionScanListMapEventsToProcess = new HashMap<String, List<Integer>>();
            try
            {
                List<QuantEvent> quantEvents = QuantEvent.loadQuantEvents(getFileArgumentValue("quratetoprocess"));
                for (QuantEvent quantEvent : quantEvents)
                {
                    List<Integer> scansThisFraction = fractionScanListMapEventsToProcess.get(quantEvent.getFraction());
                    if (scansThisFraction == null)
                    {
                        scansThisFraction = new ArrayList<Integer>();
                        fractionScanListMapEventsToProcess.put(quantEvent.getFraction(), scansThisFraction);
                    }
                    scansThisFraction.add(quantEvent.getScan());
                }
                ApplicationContext.infoMessage("Only processing " + quantEvents.size() + " specified events");

            }
            catch (IOException e)
            {
                throw new ArgumentValidationException("Failed to load Qurate file",e);
            }
        }
    }

    /**
     * do the actual work
     */
    public void execute() throws CommandLineModuleExecutionException
    {
        quantVisualizer = new QuantitationVisualizer();
        for (File featureFile : featureFiles)
            handleFile(featureFile);
    }

    protected void handleFile(File featureFile) throws CommandLineModuleExecutionException
    {
        ApplicationContext.infoMessage("Handling file " + featureFile.getAbsolutePath());

        List<FeatureSet> featureSets = null;
//        Iterator<FeatureSet> featureSetIterator = null;
        try
        {
            featureSets = PepXMLFeatureFileHandler.getSingletonInstance().loadAllFeatureSets(featureFile);

//            featureSetIterator =
//                    new PepXMLFeatureFileHandler.PepXMLFeatureSetIterator(featureFile);
        }
        catch (IOException e)
        {
            throw new CommandLineModuleExecutionException("Failure loading pepXML file " + featureFile.getAbsolutePath(),e);
        }

        int flaggedFeaturesThisFile = 0;
        int totalFeaturesThisFile = 0;

        List<File> tempFlaggedFeatureFiles = new ArrayList<File>();

        //only gets populated if we're saving unflagged features
        List<File> tempUnFlaggedFeatureFiles = new ArrayList<File>();

        int numSetsProcessed = 0;
        for (FeatureSet featureSet : featureSets)
        {
//            FeatureSet featureSet = featureSetIterator.next();
            totalFeaturesThisFile += featureSet.getFeatures().length;
            MSRun run = null;
            try
            {
                File featureSetFile = featureSet.getSourceFile();
                if (MS2ExtraInfoDef.getFeatureSetBaseName(featureSet) != null)
                    featureSetFile = new File(MS2ExtraInfoDef.getFeatureSetBaseName(featureSet) + ".pep.xml");
                File mzXmlFile = ViewerCommandModuleUtilities.findCorrespondingMzXmlFile(
                        featureSetFile, mzXmlDir);
                ApplicationContext.infoMessage("Loading mzXml file " + mzXmlFile.getAbsolutePath());
                run = MSRun.load(mzXmlFile.getAbsolutePath());
                ApplicationContext.infoMessage("Loaded.");
            }
            catch (IOException e)
            {
                throw new CommandLineModuleExecutionException("Can't find or open mzXml file for file " +
                        featureFile.getAbsolutePath(), e);
            }

            ApplicationContext.infoMessage("\tProcessing fraction " + (numSetsProcessed+1) + "...");

            String baseName = MS2ExtraInfoDef.getFeatureSetBaseName(featureSet);
            if (baseName == null)
            {
                baseName = featureFile.getName();
                if (featureSets.size() > 1)
                    baseName = baseName + "_" + numSetsProcessed;
            }
            //if PeptideProphet was run from a directory below the directory containing the
            //mzXML files, we may have ../ in the baseName, which causes trouble in saving
            //the temporary files
            while (baseName.contains(".." + File.separator))
                baseName = baseName.replaceFirst(".." + File.separator, "");

            int numFeaturesThisFraction = featureSet.getFeatures().length;

//            Pair<FeatureSet, FeatureSet> flaggedUnflaggedSets = processFeatureSet(featureSet, run);
            int numFlaggedFeaturesThisFraction = processFeatureSet(featureSet, run);
            run = null;
            System.gc();

//            FeatureSet flaggedFeatureSet = flaggedUnflaggedSets.first;
//            FeatureSet unflaggedFeatureSet = flaggedUnflaggedSets.second;

//            int numFlaggedFeaturesThisFraction = flaggedFeatureSet.getFeatures().length;

            ApplicationContext.infoMessage("Flagged " + numFlaggedFeaturesThisFraction + " out of " +
                    numFeaturesThisFraction + " features this fraction");
            flaggedFeaturesThisFile += numFlaggedFeaturesThisFraction;

//            if (outFlaggedFile != null)
//            {
//                File thisFractionFlaggedFeatureFile = TempFileManager.createTempFile(baseName + ".pep.xml", this);
//                try
//                {
//                    flaggedFeatureSet.savePepXml(thisFractionFlaggedFeatureFile);
//                    _log.debug("Saved fraction flagged file as " + thisFractionFlaggedFeatureFile.getAbsolutePath());
//                    tempFlaggedFeatureFiles.add(thisFractionFlaggedFeatureFile);
//
//                }
//                catch (IOException e)
//                {
//                    throw new CommandLineModuleExecutionException("Failure writing file " +
//                            thisFractionFlaggedFeatureFile.getAbsolutePath(),e);
//                }
//            }
//
//            if (outNoFlaggedFile != null)
//            {
//                File thisFractionUnFlaggedFeatureFile =
//                        TempFileManager.createTempFile(baseName + ".unflagged.pep.xml", this);
//                try
//                {
//                    unflaggedFeatureSet.savePepXml(thisFractionUnFlaggedFeatureFile);
//                    _log.debug("Saved fraction unflagged file as " + thisFractionUnFlaggedFeatureFile.getAbsolutePath());
//                    tempUnFlaggedFeatureFiles.add(thisFractionUnFlaggedFeatureFile);
//                }
//                catch (IOException e)
//                {
//                    throw new CommandLineModuleExecutionException("Failure writing file " +
//                            thisFractionUnFlaggedFeatureFile.getAbsolutePath(),e);
//                }
//            }

            numSetsProcessed++;
        }

        ApplicationContext.infoMessage("Flagged " + flaggedFeaturesThisFile + " out of " + totalFeaturesThisFile +
                " features (" + ((float) flaggedFeaturesThisFile *100f / (float) totalFeaturesThisFile) +
                "%) in this file");

        int numFlaggedWithoutOtherPeptideSupport = 0;
        Set<String> peptidesFlaggedWithoutOtherSupport = new HashSet<String>();
        Set<String> allQuantPeptides = new HashSet<String>();
        int numFlaggedWithoutOtherProteinSupport = 0;
        Set<String> proteinsFlaggedWithoutOtherSupport = new HashSet<String>();
        Set<String> allQuantProteins = new HashSet<String>();
        for (FeatureSet featureSet : featureSets)
        {
            for (Feature feature : featureSet.getFeatures())
            {
                if (IsotopicLabelExtraInfoDef.hasRatio(feature))
                {
                    String peptide = MS2ExtraInfoDef.getFirstPeptide(feature);
                    allQuantPeptides.add(peptide);
                    allQuantProteins.addAll(MS2ExtraInfoDef.getProteinList(feature));
                    QuantEventAssessor.QuantEventAssessment assessment = (QuantEventAssessor.QuantEventAssessment)feature.getProperty(FEATURE_PROPERTY_QUANTASSESSMENT);
                    if (assessment.getStatus() != QuantEventAssessor.FLAG_REASON_OK)
                    {
                        allQuantPeptides.add(peptide);
                        if (!peptidesWithGoodQuantEvents.contains(peptide))
                        {
                            numFlaggedWithoutOtherPeptideSupport++;
                            peptidesFlaggedWithoutOtherSupport.add(peptide);
                        }
                        boolean hasUnsupportedProtein = false;
                        for (String protein : MS2ExtraInfoDef.getProteinList(feature))
                        {
                            if (!proteinsWithGoodQuantEvents.contains(protein))
                            {
                                hasUnsupportedProtein = true;
                                proteinsFlaggedWithoutOtherSupport.add(protein);
                            }
                        }
                        if (hasUnsupportedProtein) numFlaggedWithoutOtherProteinSupport++;
                    }
                }
            }
        }
        ApplicationContext.infoMessage("Overall, " + numFlaggedWithoutOtherPeptideSupport + " out of " + flaggedFeaturesThisFile +
               " flagged features (" + (100f * numFlaggedWithoutOtherPeptideSupport/flaggedFeaturesThisFile) + "%) have no unflagged ratio peptide support");
        ApplicationContext.infoMessage(peptidesFlaggedWithoutOtherSupport.size() + " out of " + allQuantPeptides.size() + " quant peptides would lose ratios if flagged removed");
        ApplicationContext.infoMessage("Overall, " + numFlaggedWithoutOtherProteinSupport + " out of " + flaggedFeaturesThisFile +
               " flagged features (" + (100f * numFlaggedWithoutOtherProteinSupport/flaggedFeaturesThisFile) + "%) have no unflagged ratio protein support");
        ApplicationContext.infoMessage(proteinsFlaggedWithoutOtherSupport.size() + " out of " + allQuantProteins.size() + " quant proteins would lose ratios if flagged removed");

        //combine flagged feature files
        if (outFlaggedFile != null)
        {
            ApplicationContext.infoMessage("Saving flagged features...");
            List<FeatureSet> flaggedFeatureSets = new ArrayList<FeatureSet>();
            for (FeatureSet featureSet : featureSets)
            {
                FeatureSet flaggedFeatureSet = (FeatureSet) featureSet.clone();
                List<Feature> keptFeatures = new ArrayList<Feature>();
                for (Feature feature : flaggedFeatureSet.getFeatures())
                {
                    boolean keep = false;
                    if (IsotopicLabelExtraInfoDef.hasRatio(feature))
                    {
                        QuantEventAssessor.QuantEventAssessment assessment = (QuantEventAssessor.QuantEventAssessment)feature.getProperty(FEATURE_PROPERTY_QUANTASSESSMENT);
                        if (assessment.getStatus() != QuantEventAssessor.FLAG_REASON_OK)
                            keep = true;
                    }
                    if (keep)
                        keptFeatures.add(feature);
                }
                flaggedFeatureSet.setFeatures(keptFeatures.toArray(new Feature[keptFeatures.size()]));
                flaggedFeatureSets.add(flaggedFeatureSet);
            }
            try
            {
                PepXMLFeatureFileHandler.getSingletonInstance().saveFeatureSets(flaggedFeatureSets, outFlaggedFile);
                            ApplicationContext.infoMessage("Saved unflagged features to file " +
                    outFlaggedFile.getAbsolutePath());
            }
            catch (IOException e)
            {
                throw new CommandLineModuleExecutionException("Failed to save output file " + outFlaggedFile.getAbsolutePath(), e);
            }
        }

        File outNoFlaggedFileThisFile = outNoFlaggedFile;
        if (outNoFlaggedFileThisFile == null && outNoFlaggedDir != null)
        {
            outNoFlaggedFileThisFile = new File(outNoFlaggedDir, featureFile.getName());
        }

        //combine unflagged feature files
        if (outNoFlaggedFileThisFile != null)
        {
            ApplicationContext.infoMessage("Saving unflagged features to file " + outNoFlaggedFileThisFile + " ...");
            List<FeatureSet> unflaggedFeatureSets = new ArrayList<FeatureSet>();
            for (FeatureSet featureSet : featureSets)
            {
                FeatureSet unflaggedFeatureSet = (FeatureSet) featureSet.clone();
                List<Feature> keptFeatures = new ArrayList<Feature>();
                for (Feature feature : unflaggedFeatureSet.getFeatures())
                {
                    boolean keep = true;

                    if (IsotopicLabelExtraInfoDef.hasRatio(feature))
                    {
                        QuantEventAssessor.QuantEventAssessment assessment = (QuantEventAssessor.QuantEventAssessment)feature.getProperty(FEATURE_PROPERTY_QUANTASSESSMENT);
                        if (assessment.getStatus() != QuantEventAssessor.FLAG_REASON_OK)
                            keep = false;
                    }
                    if (keep)
                        keptFeatures.add(feature);
                }
                unflaggedFeatureSet.setFeatures(keptFeatures.toArray(new Feature[keptFeatures.size()]));
                unflaggedFeatureSets.add(unflaggedFeatureSet);

            }
            try
            {
                PepXMLFeatureFileHandler.getSingletonInstance().saveFeatureSets(unflaggedFeatureSets, outNoFlaggedFileThisFile);
            ApplicationContext.infoMessage("Saved unflagged features to file " +
                    outNoFlaggedFileThisFile.getAbsolutePath());
            }
            catch (IOException e)
            {
                throw new CommandLineModuleExecutionException("Failed to save output file " + outNoFlaggedFileThisFile.getAbsolutePath(), e);
            }

        //combine flagged feature files
//        if (outFlaggedFile != null)
//        {
//            ApplicationContext.infoMessage("Saving flagged features...");
//            try
//            {
//                if (numSetsProcessed == 1)
//                {
//                    FileReader in = new FileReader(tempFlaggedFeatureFiles.get(0));
//                    FileWriter out = new FileWriter(outFlaggedFile);
//                    int c;
//
//                    while ((c = in.read()) != -1)
//                        out.write(c);
//
//                    in.close();
//                    out.close();
//                }
//                else
//                {
//                    _log.debug("\tCombining individual fraction files... ");
//                    new PepXMLFeatureFileHandler().combinePepXmlFiles(tempFlaggedFeatureFiles, outFlaggedFile);
//                    ApplicationContext.infoMessage("Saved flagged features to file " +
//                            outFlaggedFile.getAbsolutePath());
//                }
//            }
//            catch (IOException e)
//            {
//                throw new CommandLineModuleExecutionException("Failed to save output file " + outFlaggedFile.getAbsolutePath(), e);
//            }
//        }

        //combine unflagged feature files
//        if (outNoFlaggedFile != null)
//        {
//            ApplicationContext.infoMessage("Saving unflagged features...");
//            try
//            {
//                if (numSetsProcessed == 1)
//                {
//                    FileReader in = new FileReader(tempUnFlaggedFeatureFiles.get(0));
//                    FileWriter out = new FileWriter(outNoFlaggedFile);
//                    int c;
//
//                    while ((c = in.read()) != -1)
//                        out.write(c);
//
//                    in.close();
//                    out.close();
//                }
//                else
//                {
//                    _log.debug("\tCombining individual fraction files... ");
//                    new PepXMLFeatureFileHandler().combinePepXmlFiles(tempUnFlaggedFeatureFiles, outNoFlaggedFile);
//                    ApplicationContext.infoMessage("Saved unflagged features to file " +
//                            outNoFlaggedFile.getAbsolutePath());
//                }
//            }
//            catch (IOException e)
//            {
//                throw new CommandLineModuleExecutionException("Failed to save unflagged output file " +
//                        outNoFlaggedFile.getAbsolutePath(), e);
//            }
        }
        ApplicationContext.infoMessage("Done.");

        if (shouldCalcSensSpecWithHardcodedScans && !badScans.isEmpty())
        {
            System.err.println("Bad: " + badScans.size() + ", bad flagged: " + badFlagged.size() +
                    ", sens=" + ((float)badFlagged.size() / (float)badScans.size()) + ", spec=" + (((float) flaggedFeaturesThisFile - (float)badFlagged.size()) / (float) flaggedFeaturesThisFile));
            System.err.println("False negatives: ");
            for (int scan : badScans) if (!badFlagged.contains(scan)) System.err.println(scan);
            System.err.println("False positives: ");
            for (int scan : flaggedScans) if (!badScans.contains(scan)) System.err.println(scan);
        }

        if (quantEventAssessor.isShowCharts())
        {
//                new PanelWithHistogram(reasonsTruePos, "TruePos reasons").displayInTab();
//                new PanelWithHistogram(reasonsFalsePos, "FalsePos reasons").displayInTab();
            new PanelWithHistogram(flaggedReasons, "Flag reasons").displayInTab();

            List<Float> cvs = new ArrayList<Float>();
            List<Float> numFlagged = new ArrayList<Float>();
            List<Float> distFromMeanFlagged = new ArrayList<Float>();
            List<Float> distFromMeanNotFlagged = new ArrayList<Float>();

            for (String peptide : peptidePerFractionRatioFlaggedMap.keySet())
            {
                List<Pair<Float, Boolean>> ratioFlaggeds = peptidePerFractionRatioFlaggedMap.get(peptide);
                List<Float> logRatios = new ArrayList<Float>();

                for (Pair<Float, Boolean> pair : ratioFlaggeds)
                {
                    logRatios.add((float)Math.log(pair.first));

                }
                cvs.add((float)BasicStatistics.coefficientOfVariation(logRatios));
                int numFracsFlagged = 0;
                if (peptideNumFractionsFlaggedMap.containsKey(peptide))
                    numFracsFlagged = peptideNumFractionsFlaggedMap.get(peptide);
                numFlagged.add((float) numFracsFlagged);

                if (logRatios.size() > 2)
                {
                    float meanLog = (float) BasicStatistics.mean(logRatios);
                    float stddevLog = (float) BasicStatistics.standardDeviation(logRatios);
                    if (stddevLog > 0)
                    {
                        for (Pair<Float, Boolean> pair : ratioFlaggeds)
                        {
                            float scaled = ((float) Math.log(pair.first) - meanLog) / stddevLog;

                            if (pair.second)
                            {
                                distFromMeanFlagged.add(Math.abs(scaled));
//     System.err.println("**" +Math.log(pair.first) + ", " + meanLog + ", " +  scaled);

                            }
                            else
                            {
                                distFromMeanNotFlagged.add(Math.abs(scaled));
//       System.err.println(Math.log(pair.first) + ", " + meanLog + ", " + scaled);
                            }
                        }
                    }
                }

            }
//                new PanelWithHistogram(distFromMeanFlagged, "dist flagged").displayInTab();
//                new PanelWithHistogram(distFromMeanNotFlagged, "dist notflagged").displayInTab();
            ApplicationContext.infoMessage("Mean dist from mean, flagged: " + BasicStatistics.mean(distFromMeanFlagged) + ", not: " + BasicStatistics.mean(distFromMeanNotFlagged));

            new PanelWithScatterPlot(cvs, numFlagged, "CV vs num fractions flagged").displayInTab();

            if (shouldCalcSensSpecWithHardcodedScans && !badScans.isEmpty())
            {
                System.err.println("Mean cor bad: " + BasicStatistics.mean(corrBad) + ", good: " + BasicStatistics.mean(corrGood));

                new PanelWithHistogram(corrBad, "cor bad").displayInTab();
                new PanelWithHistogram(corrGood, "cor good").displayInTab();
            }

            if (!lightPrevPeakRatios.isEmpty())
            {
                new PanelWithHistogram(lightPrevPeakRatios, "light belowpeak ratios").displayInTab();
                new PanelWithHistogram(heavyPrevPeakRatios, "heavy belowpeak ratios").displayInTab();
                System.err.println("Light belowpeak median: " + BasicStatistics.median(lightPrevPeakRatios) + ", heavy: " + BasicStatistics.median(heavyPrevPeakRatios));
                new PanelWithScatterPlot(lightPrevPeakRatios,heavyPrevPeakRatios, "light vs heavy belowpeak ratio").displayInTab();
            }

            new PanelWithScatterPlot(logAlgRatiosWithSinglePeakRatios,
                    logSinglePeakRatios, "alg vs singlepeak").displayInTab();

            Collections.sort(allLogRatiosFlaggeds, new Comparator<Pair<Float,Boolean>>()
            {
                public int compare(Pair<Float,Boolean> o1, Pair<Float,Boolean> o2)
                {
                    return o1.first > o2.first ? 1 : (o1.first == o2.first ? 0 : -1);
                }
            });

            int rollingWindowSize = 30;
            List<Boolean> windowAssessments = new ArrayList<Boolean>(rollingWindowSize);
            List<Float> windowLogRatios = new ArrayList<Float>(rollingWindowSize);

            List<Float> allRatiosWithWindows = new ArrayList<Float>();
            List<Float> allProportionsBad = new ArrayList<Float>();
            List<Float> allBadLogRatios = new ArrayList<Float>();
            List<Float> allGoodLogRatios = new ArrayList<Float>();

            for (Pair<Float,Boolean> ratioAndAssessment : allLogRatiosFlaggeds)
            {
                if (ratioAndAssessment.second)
                    allBadLogRatios.add(ratioAndAssessment.first);
                else
                    allGoodLogRatios.add(ratioAndAssessment.first);

                windowAssessments.add(ratioAndAssessment.second);
                windowLogRatios.add(ratioAndAssessment.first);
                if (windowAssessments.size() > rollingWindowSize)
                {
                    windowAssessments.remove(0);
                    windowLogRatios.remove(0);
                }
                if (windowAssessments.size() == rollingWindowSize)
                {
                    int numBad = 0;
                    for (boolean assessment : windowAssessments)
                        if (assessment)
                        {
                            numBad++;
                        }
                    float medianLogRatio = (float) BasicStatistics.median(windowLogRatios);
                    if (!Float.isInfinite(medianLogRatio))
                    {
                        allRatiosWithWindows.add((float) BasicStatistics.median(windowLogRatios));
                        allProportionsBad.add((float) numBad / (float) rollingWindowSize);
                    }
                }
            }
            PanelWithLineChart pwlcProportionBad = new PanelWithLineChart();
            pwlcProportionBad.setName("LogRatio vs. Proportion Bad");
            pwlcProportionBad.addData(allRatiosWithWindows, allProportionsBad, "LogRatio vs. Proportion Bad");
            pwlcProportionBad.displayInTab();

            new PanelWithHistogram(allBadLogRatios, "BadLogRatios").displayInTab();
            new PanelWithHistogram(allGoodLogRatios, "GoodLogRatios").displayInTab();

        }

        if (shouldWriteBadTurk)
        {
            outBadEventTurkPW.close();
            try
            {
                QuantEvent.saveQuantEventsToTSV(badQuantEvents, outBadEventQurateFile, true, false);
            }
            catch (IOException e)
            {
                throw new CommandLineModuleExecutionException("Failed to save Qurate file for bad events", e);
            }
        }
    }

    /**
     * @param ms2FeatureSet
     * @param run
     * @throws CommandLineModuleExecutionException
     * @return a pair of FeatureSets, one containing flagged and one containing unflagged features
     */
    protected int processFeatureSet(FeatureSet ms2FeatureSet, MSRun run)
            throws CommandLineModuleExecutionException
    {
//        List<Feature> flaggedFeatures = new ArrayList<Feature>();
//        List<Feature> unflaggedFeatures = new ArrayList<Feature>();

        Map<String, Map<Integer, List<Float>>> peptideChargeRatiosMap = new HashMap<String, Map<Integer, List<Float>>>();
        Set<String> flaggedPeptides = new HashSet<String>();

        Arrays.sort(ms2FeatureSet.getFeatures(), new Feature.ScanChargeMzAscComparator());

        int numFeatures = ms2FeatureSet.getFeatures().length;
        ApplicationContext.infoMessage("Processing " + numFeatures + " features....");
        int i=0;

        String featureSetBaseName = MS2ExtraInfoDef.getFeatureSetBaseName(ms2FeatureSet);
        if (onlyProcessSpecifiedEvents && !fractionScanListMapEventsToProcess.containsKey(featureSetBaseName))
        {
            ApplicationContext.infoMessage("WARNING!!! Fraction " + featureSetBaseName + " not found in qurate file.  Skipping.");
            FeatureSet dummy = (FeatureSet)ms2FeatureSet.clone();
            dummy.setFeatures(new Feature[0]);
            return 0;
        }
        int numFlaggedFeatures = 0;
        for (Feature feature : ms2FeatureSet.getFeatures())
        {
            //todo: remove this when done with it (for turk)
            if (onlyProcessSpecifiedEvents && !fractionScanListMapEventsToProcess.get(featureSetBaseName).contains(feature.getScan()))
                continue;

            if (numFeatures > 10 && i % (int) (numFeatures/10f) == 0)
                ApplicationContext.infoMessage((10 * (i / (int) (numFeatures/10f))) + "%");
            i++;

            if (!IsotopicLabelExtraInfoDef.hasRatio(feature))
            {
//                unflaggedFeatures.add(feature);
                continue;
            }

            QuantEvent quantEvent = new QuantEvent(feature, MS2ExtraInfoDef.getFeatureSetBaseName(ms2FeatureSet));
            QuantEventAssessor.QuantEventAssessment assessment = quantEventAssessor.assessQuantEvent(quantEvent, run);
            feature.setProperty(FEATURE_PROPERTY_QUANTASSESSMENT, assessment);
            int flagReason = assessment.getStatus();
            String flagReasonDesc = assessment.getExplanation();

            float algRatio = (float) IsotopicLabelExtraInfoDef.getRatio(feature);
            allLogRatiosFlaggeds.add(new Pair<Float, Boolean>((float)Math.log(algRatio),
                    flagReason != QuantEventAssessor.FLAG_REASON_OK));
            if (flagReason == QuantEventAssessor.FLAG_REASON_OK)
            {
                peptidesWithGoodQuantEvents.add(MS2ExtraInfoDef.getFirstPeptide(feature));
                proteinsWithGoodQuantEvents.addAll(MS2ExtraInfoDef.getProteinList(feature));
//                unflaggedFeatures.add(feature);
            }
            else
            {
                badQuantEvents.add(quantEvent);
                flaggedReasons.add((float) flagReason);
                feature.setDescription(flagReasonDesc);
                numFlaggedFeatures++;
//                flaggedFeatures.add(feature);
                flaggedPeptides.add(MS2ExtraInfoDef.getFirstPeptide(feature));

                if (assessment.getSinglePeakRatio() >0 && algRatio > 0)
                {
                    logAlgRatiosWithSinglePeakRatios.add((float)Math.log(algRatio));
                    logSinglePeakRatios.add((float) Math.log(assessment.getSinglePeakRatio()));
                }
                if (shouldWriteBadTurk)
                {
                    PanelWithSpectrumChart spectrumPanel = quantVisualizer.createPanelWithSpectrumChart(run, quantEvent);
                    try
                    {
                        String turkFileLine = quantVisualizer.saveTurkImage(quantEvent, spectrumPanel,
                                outTurkImageDir, badTurkId++);
                        outBadEventTurkPW.println(turkFileLine);
                    }
                    catch (IOException e)
                    {
                        throw new CommandLineModuleExecutionException("Failed to create Turk image", e);
                    }
                }
            }
            float ratio = (float) IsotopicLabelExtraInfoDef.getRatio(feature);

            String peptide = MS2ExtraInfoDef.getFirstPeptide(feature);

            Map<Integer, List<Float>> chargeRatiosMap = peptideChargeRatiosMap.get(peptide);
            if (chargeRatiosMap == null)
            {
                chargeRatiosMap = new HashMap<Integer, List<Float>>();
                peptideChargeRatiosMap.put(peptide, chargeRatiosMap);
            }
            List<Float> ratiosThisCharge = chargeRatiosMap.get(feature.charge);
            if (ratiosThisCharge == null)
            {
                ratiosThisCharge = new ArrayList<Float>();
                chargeRatiosMap.put(feature.charge, ratiosThisCharge);
            }
            ratiosThisCharge.add(ratio);
        }

        for (String peptide : flaggedPeptides)
        {
            int oldNum = 0;
            if (peptideNumFractionsFlaggedMap.containsKey(peptide))
                oldNum = peptideNumFractionsFlaggedMap.get(peptide);
            peptideNumFractionsFlaggedMap.put(peptide, oldNum+1);
        }

        for (String peptide : peptideChargeRatiosMap.keySet())
        {
            Map<Integer, List<Float>> chargeRatiosMap = peptideChargeRatiosMap.get(peptide);
            for (int charge :chargeRatiosMap.keySet() )
            {
            List<Pair<Float, Boolean>> allFracRatiosFlaggedThisPeptide = peptidePerFractionRatioFlaggedMap.get(peptide);
            if (allFracRatiosFlaggedThisPeptide == null)
            {
                allFracRatiosFlaggedThisPeptide = new ArrayList<Pair<Float, Boolean>>();
                peptidePerFractionRatioFlaggedMap.put(peptide, allFracRatiosFlaggedThisPeptide);
            }
            allFracRatiosFlaggedThisPeptide.add(new Pair<Float, Boolean>(
                    (float)BasicStatistics.geometricMean(chargeRatiosMap.get(charge)),
                    flaggedPeptides.contains(peptide)));
            }
        }

//        FeatureSet flaggedFeatureSet = (FeatureSet) ms2FeatureSet.clone();
//        flaggedFeatureSet.setFeatures(flaggedFeatures.toArray(new Feature[flaggedFeatures.size()]));
//
//        FeatureSet unflaggedFeatureSet = (FeatureSet) ms2FeatureSet.clone();
//        unflaggedFeatureSet.setFeatures(unflaggedFeatures.toArray(new Feature[unflaggedFeatures.size()]));

        return numFlaggedFeatures;
    }
}


text:
import json

from django.test import TestCase

from games import fieldmarshal
from games.fieldmarshal import Struct
from games.api import MetricContainer

class Foo(Struct):
    id = unicode

class Bar(Struct):
    id = bool

class Baz(Struct):
    id = int

class Bap(Struct):
    id = None

class Nested(Struct):
    foo = Foo

class SuperNested(Struct):
    nested = Nested

class ArrayFoo(Struct):
    foos = [Foo]

class HashFoo(Struct):
    foos = {}

class FieldMarshalTest(TestCase):

    def test_array(self):
        c = ArrayFoo()
        self.assertEquals(len(c.foos), 0)

    def test_hash(self):
        c = HashFoo()
        self.assertEquals(c.foos.items(), [])

    def test_recursive_dumps(self):
        c = Nested(foo=Foo(id=u"foo"))
        self.assertEquals(fieldmarshal.dumps(c),
                          json.dumps({"foo": {"id": "foo"}}))

    def test_recursive_load(self):
        resp = json.dumps({"foo": {"id": "foo"}})
        c = fieldmarshal.loads(Nested, resp)
        self.assertEquals(c.foo.id, "foo")

    def test_deeper_load(self):
        resp = json.dumps({"nested": {"foo": {"id": "foo"}}})
        c = fieldmarshal.loads(SuperNested, resp)
        self.assertEquals(c.nested.foo.id, "foo")

    def test_dumps(self):
        c = Foo(id=u"foo")
        self.assertEquals(fieldmarshal.dumps(c),
                          json.dumps({"id": "foo"}))

    def test_dumpd(self):
        c = Foo(id=u"foo")
        self.assertEquals(fieldmarshal.dumpd(c),
                          {u"id": u"foo"})

    def test_zero_value_string(self):
        c = Foo()
        self.assertEquals(c.id, "")

    def test_zero_value_none(self):
        c = Bap()
        self.assertEquals(c.id, None)

    def test_zero_value_bool(self):
        c = Bar()
        self.assertEquals(c.id, False)

    def test_zero_value_int(self):
        c = Baz()
        self.assertEquals(c.id, 0)

    def test_zero_value_struct(self):
        c = Nested()
        self.assertEquals(c.foo.id, "")

    def test_loads(self):
        resp = json.dumps({"id": "foo"})
        c = fieldmarshal.loads(Foo, resp)
        self.assertEquals(c.id, "foo")

    def test_loads_wrong_value(self):

        class Number(Struct):
            id = int

        resp = json.dumps({"id": "foo"})

        with self.assertRaises(ValueError):
            fieldmarshal.loads(Number, resp)

    def test_bearraid_metrics(self):
        resp = {
            'metrics': [{
                'event': 'open',
                'properties': {
                    'foo': 'bar',
                },
            }],
        }

        p = fieldmarshal.loads(MetricContainer, json.dumps(resp))

        m = p.metrics[0]

        self.assertEquals('open', m.event)
        self.assertEquals('bar', m.properties['foo'])

    def test_bearraid_metrics_bad_type(self):
        resp = {
            'metrics': [{
                'event': 3,
                'properties': {
                    'foo': 'bar',
                },
            }],
        }

        with self.assertRaises(ValueError):
            fieldmarshal.loads(MetricContainer, json.dumps(resp))

redpajama_set_name: RedPajamaGithub


text:
/* jshint indent:false */
steal("can/view/target", "steal-qunit", function(target){

	module("can/view/target");

	test("basics", function(){

		// "
Hello {{message}}!
"
		var classCallback = function( ){
			equal(this.nodeName.toLowerCase(), "h1", "class on the right element");
			this.className = "selected";
		},
			attributesCallback = function(){
				equal(this.nodeName.toLowerCase(), "h1", "attributes on the right element");
			},
			textNodeCallback = function( ){
				equal(this.nodeType, 3, "got a text node");
				this.nodeValue = "World";
			};

		var data = target([{
			tag: "h1",
			attrs: {
				"id" : "myh1",
				"class" : classCallback
			},
			attributes: [attributesCallback],
			children: [{
				tag: "span",
				children: [
					"Hello ",
					textNodeCallback,
					"!"
				]
			}]
		}]);

		equal( data.clone.childNodes.length, 1, "there is one child");

		var h1 = data.clone.childNodes[0];
		equal( h1.nodeName.toLowerCase(), "h1", "there is one h1");
		equal( h1.id, "myh1", "the h1 has the right id");

		equal( h1.childNodes.length, 1, "the h1 has span");

		equal( h1.childNodes[0].childNodes.length, 3, "the span has 3 children");

		deepEqual( data.paths,
			[{
				path: [0],
				callbacks: [
					{ callback: classCallback },
					{ callback: attributesCallback }
				],
				paths: [{
					path: [0,1],
					callbacks: [
						{callback: target.keepsTextNodes ? textNodeCallback : data.paths[0].paths[0].callbacks[0].callback }
					]
				}]
			}] );

		var result = data.hydrate();

		var newH1 = result.childNodes[0];
		equal(newH1.className, "selected", "got selected class name");
		equal(newH1.innerHTML.toLowerCase(), "hello world!");

	});

	test("replacing items", function(){
		var data = target([
			function(){
				this.parentNode.insertBefore(document.createTextNode("inserted"), this.nextSibling);
			},
			"hi",
			function(){
				equal(this.previousSibling.nodeValue, "hi", "previous is as expected");
			}]);

		data.hydrate();
	});

	test("comments", function(){

		var data = target([
			{ tag: "h1" },
			{comment: "foo bar"}
		]);
		var node = data.clone.childNodes[1];
		equal(node.nodeValue, "foo bar", "node value is right");
		equal(node.nodeType, 8, "node is a comment");

	});

	test("paths should be run in reverse order (#966)", function(){

		var data = target([{
			tag: "h1",
			attributes: [function(){}],
			children:  [
				function(){
					this.parentElement.insertBefore(document.createElement("div"),this.nextSibling);
				},
				{
					tag: "span",
					children: [function(){
						equal(this.nodeType,3, "got an element");
					}]
				}
			]
		}]);
		data.hydrate();
	});

});


text:
#import "supplemental/CollectionsCellSeparatorExample.h"

static const NSInteger kSectionCount = 10;
static const NSInteger kSectionItemCount = 3;
static NSString *const kReusableIdentifierItem = @"itemCellIdentifier";

@implementation CollectionsCellSeparatorExample {
  NSMutableArray <NSMutableArray *>*_content;
}

- (void)viewDidLoad {
  [super viewDidLoad];

  // Register cell class.
  [self.collectionView registerClass:[MDCCollectionViewTextCell class]
          forCellWithReuseIdentifier:kReusableIdentifierItem];

  // Register header.
  [self.collectionView registerClass:[MDCCollectionViewTextCell class]
          forSupplementaryViewOfKind:UICollectionElementKindSectionHeader
                 withReuseIdentifier:UICollectionElementKindSectionHeader];

  // Populate content.
  _content = [NSMutableArray array];
  for (NSInteger i = 0; i < kSectionCount; i++) {
    NSMutableArray *items = [NSMutableArray array];
    for (NSInteger j = 0; j < kSectionItemCount; j++) {
      NSString *itemString = [NSString stringWithFormat:@"Section-%ld Item-%ld", (long)i, (long)j];
      [items addObject:itemString];
    }
    [_content addObject:items];
  }

  // Customize collection view settings.
  self.styler.cellStyle = MDCCollectionViewCellStyleCard;
}

#pragma mark - <UICollectionViewDataSource>

- (NSInteger)numberOfSectionsInCollectionView:(UICollectionView *)collectionView {
  return [_content count];
}

- (NSInteger)collectionView:(UICollectionView *)collectionView
     numberOfItemsInSection:(NSInteger)section {
  return [_content[section] count];
}

- (UICollectionViewCell *)collectionView:(UICollectionView *)collectionView
                  cellForItemAtIndexPath:(NSIndexPath *)indexPath {
  MDCCollectionViewTextCell *cell =
      [collectionView dequeueReusableCellWithReuseIdentifier:kReusableIdentifierItem
                                                forIndexPath:indexPath];
  cell.textLabel.text = _content[indexPath.section][indexPath.item];

  // Customize separators.
  if (indexPath.section == 0) {
    cell.separatorInset = UIEdgeInsetsMake(0, 72, 0, 0);
  } else if (indexPath.section == 1) {
    cell.separatorInset = UIEdgeInsetsMake(0, 0, 0, 72);
  } else if (indexPath.section == 2) {
    cell.shouldHideSeparator = YES;
  }

  return cell;
}

- (UICollectionReusableView *)collectionView:(UICollectionView *)collectionView
           viewForSupplementaryElementOfKind:(NSString *)kind
                                 atIndexPath:(NSIndexPath *)indexPath {
  MDCCollectionViewTextCell *supplementaryView =
      [collectionView dequeueReusableSupplementaryViewOfKind:kind
                                         withReuseIdentifier:kind
                                                forIndexPath:indexPath];

  if ([kind isEqualToString:UICollectionElementKindSectionHeader]) {
    if (indexPath.section == 3) {
      supplementaryView.textLabel.text = @"Header with separator";
    } else if (indexPath.section == 4) {
      supplementaryView.textLabel.text = @"Header without separator";
    }
  }

  return supplementaryView;
}

- (CGSize)collectionView:(UICollectionView *)collectionView
                             layout:(UICollectionViewLayout *)collectionViewLayout
    referenceSizeForHeaderInSection:(NSInteger)section {
  if (section == 3 || section == 4) {
    return CGSizeMake(collectionView.bounds.size.width, MDCCellDefaultOneLineHeight);
  }
  return CGSizeZero;
}

#pragma mark - <MDCCollectionViewStylingDelegate>

- (BOOL)collectionView:(UICollectionView *)collectionView
    shouldHideItemSeparatorAtIndexPath:(NSIndexPath *)indexPath {
  return indexPath.section == 3;
}

- (BOOL)collectionView:(UICollectionView *)collectionView
    shouldHideHeaderSeparatorForSection:(NSInteger)section {
  return section == 4;
}

#pragma mark - CatalogByConvention

+ (NSArray *)catalogBreadcrumbs {
  return @[ @"Collections", @"Cell Separator Example" ];
}

+ (BOOL)catalogIsPrimaryDemo {
  return NO;
}

+ (BOOL)catalogIsPresentable {
  return YES;
}

@end


text:
/**
 * @ngdoc service
 * @name umbraco.services.contentTypeHelper
 * @description A helper service for the content type editor
 **/
function contentTypeHelper(contentTypeResource, dataTypeResource, $filter, $injector, $q) {

    var contentTypeHelperService = {

        createIdArray: function (array) {

            var newArray = [];

            array.forEach(function (arrayItem) {

                if (Utilities.isObject(arrayItem)) {
                    newArray.push(arrayItem.id);
                } else {
                    newArray.push(arrayItem);
                }

            });

            return newArray;

        },

        generateModels: function () {
            var deferred = $q.defer();
            var modelsResource = $injector.has("modelsBuilderManagementResource") ? $injector.get("modelsBuilderManagementResource") : null;
            var modelsBuilderEnabled = Umbraco.Sys.ServerVariables.umbracoPlugins.modelsBuilder.enabled;
            if (modelsBuilderEnabled && modelsResource) {
                modelsResource.buildModels().then(function (result) {
                    deferred.resolve(result);

                    //just calling this to get the servar back to life
                    modelsResource.getModelsOutOfDateStatus();

                }, function (e) {
                    deferred.reject(e);
                });
            }
            else {
                deferred.resolve(false);
            }
            return deferred.promise;
        },

        checkModelsBuilderStatus: function () {
            var deferred = $q.defer();
            var modelsResource = $injector.has("modelsBuilderManagementResource") ? $injector.get("modelsBuilderManagementResource") : null;
            var modelsBuilderEnabled = (Umbraco && Umbraco.Sys && Umbraco.Sys.ServerVariables && Umbraco.Sys.ServerVariables.umbracoPlugins && Umbraco.Sys.ServerVariables.umbracoPlugins.modelsBuilder && Umbraco.Sys.ServerVariables.umbracoPlugins.modelsBuilder.enabled === true);

            if (modelsBuilderEnabled && modelsResource) {
                modelsResource.getModelsOutOfDateStatus().then(function (result) {
                    //Generate models buttons should be enabled if it is 0
                    deferred.resolve(result.status === 0);
                });
            }
            else {
                deferred.resolve(false);
            }
            return deferred.promise;
        },

        makeObjectArrayFromId: function (idArray, objectArray) {
            var newArray = [];

            for (var idIndex = 0; idArray.length > idIndex; idIndex++) {
                var id = idArray[idIndex];

                for (var objectIndex = 0; objectArray.length > objectIndex; objectIndex++) {
                    var object = objectArray[objectIndex];
                    if (id === object.id) {
                        newArray.push(object);
                    }
                }

            }

            return newArray;
        },

        validateAddingComposition: function (contentType, compositeContentType) {

            //Validate that by adding this group that we are not adding duplicate property type aliases

            var propertiesAdding = _.flatten(_.map(compositeContentType.groups, function (g) {
                return _.map(g.properties, function (p) {
                    return p.alias;
                });
            }));
            var propAliasesExisting = _.filter(_.flatten(_.map(contentType.groups, function (g) {
                return _.map(g.properties, function (p) {
                    return p.alias;
                });
            })), function (f) {
                return f !== null && f !== undefined;
            });

            var intersec = _.intersection(propertiesAdding, propAliasesExisting);
            if (intersec.length > 0) {
                //return the overlapping property aliases
                return intersec;
            }

            //no overlapping property aliases
            return [];
        },

        mergeCompositeContentType: function (contentType, compositeContentType) {

            //Validate that there are no overlapping aliases
            var overlappingAliases = this.validateAddingComposition(contentType, compositeContentType);
            if (overlappingAliases.length > 0) {
                throw new Error("Cannot add this composition, these properties already exist on the content type: " + overlappingAliases.join());
            }

            compositeContentType.groups.forEach(function (compositionGroup) {
                // order composition groups based on sort order
                compositionGroup.properties = $filter('orderBy')(compositionGroup.properties, 'sortOrder');

                // get data type details
                compositionGroup.properties.forEach(function (property) {
                    dataTypeResource.getById(property.dataTypeId)
                        .then(function (dataType) {
                            property.dataTypeIcon = dataType.icon;
                            property.dataTypeName = dataType.name;
                        });
                });

                // set inherited state on tab
                compositionGroup.inherited = true;

                // set inherited state on properties
                compositionGroup.properties.forEach(function (compositionProperty) {
                    compositionProperty.inherited = true;
                });

                // set tab state
                compositionGroup.tabState = "inActive";

                // if groups are named the same - merge the groups
                contentType.groups.forEach(function (contentTypeGroup) {

                    if (contentTypeGroup.name === compositionGroup.name) {

                        // set flag to show if properties has been merged into a tab
                        compositionGroup.groupIsMerged = true;

                        // make group inherited
                        contentTypeGroup.inherited = true;

                        // add properties to the top of the array
                        contentTypeGroup.properties = compositionGroup.properties.concat(contentTypeGroup.properties);

                        // update sort order on all properties in merged group
                        contentTypeGroup.properties = contentTypeHelperService.updatePropertiesSortOrder(contentTypeGroup.properties);

                        // make parentTabContentTypeNames to an array so we can push values
                        if (contentTypeGroup.parentTabContentTypeNames === null || contentTypeGroup.parentTabContentTypeNames === undefined) {
                            contentTypeGroup.parentTabContentTypeNames = [];
                        }

                        // push name to array of merged composite content types
                        contentTypeGroup.parentTabContentTypeNames.push(compositeContentType.name);

                        // make parentTabContentTypes to an array so we can push values
                        if (contentTypeGroup.parentTabContentTypes === null || contentTypeGroup.parentTabContentTypes === undefined) {
                            contentTypeGroup.parentTabContentTypes = [];
                        }

                        // push id to array of merged composite content types
                        contentTypeGroup.parentTabContentTypes.push(compositeContentType.id);

                        // get sort order from composition
                        contentTypeGroup.sortOrder = compositionGroup.sortOrder;

                        // splice group to the top of the array
                        var contentTypeGroupCopy = Utilities.copy(contentTypeGroup);
                        var index = contentType.groups.indexOf(contentTypeGroup);
                        contentType.groups.splice(index, 1);
                        contentType.groups.unshift(contentTypeGroupCopy);

                    }

                });

                // if group is not merged - push it to the end of the array - before init tab
                if (compositionGroup.groupIsMerged === false || compositionGroup.groupIsMerged === undefined) {

                    // make parentTabContentTypeNames to an array so we can push values
                    if (compositionGroup.parentTabContentTypeNames === null || compositionGroup.parentTabContentTypeNames === undefined) {
                        compositionGroup.parentTabContentTypeNames = [];
                    }

                    // push name to array of merged composite content types
                    compositionGroup.parentTabContentTypeNames.push(compositeContentType.name);

                    // make parentTabContentTypes to an array so we can push values
                    if (compositionGroup.parentTabContentTypes === null || compositionGroup.parentTabContentTypes === undefined) {
                        compositionGroup.parentTabContentTypes = [];
                    }

                    // push id to array of merged composite content types
                    compositionGroup.parentTabContentTypes.push(compositeContentType.id);

                    // push group before placeholder tab
                    contentType.groups.unshift(compositionGroup);

                }

            });

            // sort all groups by sortOrder property
            contentType.groups = $filter('orderBy')(contentType.groups, 'sortOrder');

            return contentType;

        },

        splitCompositeContentType: function (contentType, compositeContentType) {

            var groups = [];

            contentType.groups.forEach(function (contentTypeGroup) {

                if (contentTypeGroup.tabState !== "init") {

                    var idIndex = contentTypeGroup.parentTabContentTypes.indexOf(compositeContentType.id);
                    var nameIndex = contentTypeGroup.parentTabContentTypeNames.indexOf(compositeContentType.name);
                    var groupIndex = contentType.groups.indexOf(contentTypeGroup);

                    if (idIndex !== -1) {

                        var properties = [];

                        // remove all properties from composite content type
                        contentTypeGroup.properties.forEach(function (property) {
                            if (property.contentTypeId !== compositeContentType.id) {
                                properties.push(property);
                            }
                        });

                        // set new properties array to properties
                        contentTypeGroup.properties = properties;

                        // remove composite content type name and id from inherited arrays
                        contentTypeGroup.parentTabContentTypes.splice(idIndex, 1);
                        contentTypeGroup.parentTabContentTypeNames.splice(nameIndex, 1);

                        // remove inherited state if there are no inherited properties
                        if (contentTypeGroup.parentTabContentTypes.length === 0) {
                            contentTypeGroup.inherited = false;
                        }

                        // remove group if there are no properties left
                        if (contentTypeGroup.properties.length > 1) {
                            //contentType.groups.splice(groupIndex, 1);
                            groups.push(contentTypeGroup);
                        }

                    } else {
                        groups.push(contentTypeGroup);
                    }

                } else {
                    groups.push(contentTypeGroup);
                }

                // update sort order on properties
                contentTypeGroup.properties = contentTypeHelperService.updatePropertiesSortOrder(contentTypeGroup.properties);

            });

            contentType.groups = groups;

        },

        updatePropertiesSortOrder: function (properties) {

            var sortOrder = 0;

            properties.forEach(function (property) {
                if (!property.inherited && property.propertyState !== "init") {
                    property.sortOrder = sortOrder;
                }
                sortOrder++;
            });

            return properties;

        },

        getTemplatePlaceholder: function () {

            var templatePlaceholder = {
                "name": "",
                "icon": "icon-layout",
                "alias": "templatePlaceholder",
                "placeholder": true
            };

            return templatePlaceholder;

        },

        insertDefaultTemplatePlaceholder: function (defaultTemplate) {

            // get template placeholder
            var templatePlaceholder = contentTypeHelperService.getTemplatePlaceholder();

            // add as default template
            defaultTemplate = templatePlaceholder;

            return defaultTemplate;

        },

        insertTemplatePlaceholder: function (array) {

            // get template placeholder
            var templatePlaceholder = contentTypeHelperService.getTemplatePlaceholder();

            // add as selected item
            array.push(templatePlaceholder);

            return array;

        },

        insertChildNodePlaceholder: function (array, name, icon, id) {

            var placeholder = {
                "name": name,
                "icon": icon,
                "id": id
            };

            array.push(placeholder);

        }

    };

    return contentTypeHelperService;
}
angular.module('umbraco.services').factory('contentTypeHelper', contentTypeHelper);


text:
using Android.App;
using Android.OS;
using Android.Views;
using Android.Widget;
using Esri.ArcGISRuntime.Mapping;
using Esri.ArcGISRuntime.UI.Controls;
using System;
using System.Linq;

namespace ArcGISRuntimeXamarin.Samples.ChangeBasemap
{
    [Activity]
    public class ChangeBasemap : Activity
    {
        // Create and hold reference to the used MapView
        private MapView _myMapView = new MapView();

        // String array to store basemap constructor types
        private string[] _basemapTypes = new string[]
        {
            "Topographic",
            "Streets",
            "Imagery",
            "Oceans"
        };

        protected override void OnCreate(Bundle bundle)
        {
            base.OnCreate(bundle);

            Title = "Change basemap";

            // Create the UI, setup the control references and execute initialization
            CreateLayout();
            Initialize();
        }

        private void Initialize()
        {
            // Create new Map with basemap
            Map myMap = new Map(Basemap.CreateTopographic());

            // Provide used Map to the MapView
            _myMapView.Map = myMap;
        }

        private void OnBasemapsClicked(object sender, EventArgs e)
        {
            var mapsButton = sender as Button;

            // Create menu to show map options
            var mapsMenu = new PopupMenu(this, mapsButton);
            mapsMenu.MenuItemClick += OnBasemapsMenuItemClicked;

            // Create menu options
            foreach (var basemapType in _basemapTypes)
                mapsMenu.Menu.Add(basemapType);

            // Show menu in the view
            mapsMenu.Show();
        }

        private void OnBasemapsMenuItemClicked(object sender, PopupMenu.MenuItemClickEventArgs e)
        {
            // Get title from the selected item
            var selectedBasemapType = e.Item.TitleCondensedFormatted.ToString();

            // Get index that is used to get the selected url
            var selectedIndex = _basemapTypes.ToList().IndexOf(selectedBasemapType);

            switch (selectedIndex)
            {
                case 0:

                    // Set the basemap to Topographic
                    _myMapView.Map.Basemap = Basemap.CreateTopographic();
                    break;

                case 1:

                    // Set the basemap to Streets
                    _myMapView.Map.Basemap = Basemap.CreateStreets();
                    break;

                case 2:

                    // Set the basemap to Imagery
                    _myMapView.Map.Basemap = Basemap.CreateImagery();
                    break;

                case 3:

                    // Set the basemap to Oceans
                    _myMapView.Map.Basemap = Basemap.CreateOceans();
                    break;
            }
        }

        private void CreateLayout()
        {
            // Create a new vertical layout for the app
            var layout = new LinearLayout(this) { Orientation = Orientation.Vertical };

            // Create button to show possible map options
            var mapsButton = new Button(this);
            mapsButton.Text = "Basemaps";
            mapsButton.Click += OnBasemapsClicked;

            // Add maps button to the layout
            layout.AddView(mapsButton);

            // Add the map view to the layout
            layout.AddView(_myMapView);

            // Show the layout in the app
            SetContentView(layout);
        }
    }
}


text:
from .cell import Cell

class Row(object):
    def __init__(self, data=[], is_header=False, style=None):
        self._cells = data
        self._is_header = is_header
        self._style = style

    def add_cell(self, cell):
        assert isinstance(cell, Cell)
        self._cells.append(cell)

    def __iter__(self):
        for cell in self._cells:
            yield cell

    @property
    def cells(self):
        return self._cells

    @property
    def is_header(self):
        return self._is_header

    @property
    def height(self):
        return self._style['height']


text:
i18n.translations.de = {
      //Navigation
      "Menu": "Menü",
      "Top": "Top",
      "New": "Neu",
      "Digest": "Zusammenfassung",
      "Categories": "Kategorien",
      "Users": "Benutzer",
      "Settings": "Einstellungen",
      "Admin": "Admin",
      "Post": "Link eintragen",
      "Toolbox": "Werkzeuge",
      "Sign Up/Sign In": "Registrieren/Anmelden",
      "My Account": "Mein Konto",
      "View Profile": "Profil anzeigen",
      "Edit Account": "Konto bearbeiten",

      //Main
      "New Posts": "Neue Links",

      //Commments
      "Your comment has been deleted.": "Dein Kommentar wurde gelöscht.",
      "Comment": "Kommentieren",
      "Delete Comment": "Kommentar löschen",
      "Add Comment": "Kommentar hinzufügen",
      "upvote": "+1",
      "downvote": "-1",
      "link": "link",
      "Edit": "bearbeiten",
      "Reply": "antworten",
      "No comments.": "Keine Kommentare.",

      //Errors
      "You are already logged in": "Du bist bereits eingeloggt",
      "Sorry, this is a private site. Please sign up first.": "Dies ist ein privates Angebot. Du musst dich erst registrieren.",
      "Thanks for signing up!": "Vielen Dank für Deine Registrierung!",
      "The site is currently invite-only, but we will let you know as soon as a spot opens up.": "Derzeit sind Neuregistrierungen nur mit einer Einladung möglich, aber wir werden dich wissen lassen, wenn wir unsere Registrierung wieder öffnen.",
      "Sorry, you don't have the rights to view this page.": "Entschuldigung, Du hast leider keine Rechte diese Seite anzuzeigen.",
      "Not Found!": "Nichts gefunden!",
      "We're sorry; whatever you were looking for isn't here..": "Es tut uns leid, wonach auch immer Du gesucht hast, hier ist es nicht.",

      //Notifications
      "No notifications": "Keine Benachrichtigungen",
      "1 notification": "1 Benachrichtigung",
      "notifications": "Benachrichtigungen",
      "Mark all as read": "Alle als gelesen markieren",

      // Post deleted
      "Your post has been deleted.": "Dein Link wurde gelöscht.",

      // Post digest
      "The top 5 posts of each day.": "Die Top-5-Links eines jeden Tages.",
      "Previous Day": "Einen Tag zurück",
      "Next Day": "Einen Tag vor",
      "Sorry, no posts for today": "Heute gibt es keine Links.",
      "Sorry, no posts for": "Keine Links für",
      "Today": "Heute",
      "Yesterday": "Gestern",

      // Post submit & edit
      "Created": "Erstellt",
      "Title": "Titel",
      "Suggest title": "Titelvorschlag",
      "URL": "URL",
      "Short URL": "Kurz-URL",
      "Body": "Beschreibung",
      "Category": "Kategorie",
      "Inactive?": "Inaktiv?",
      "Sticky?": "Anheften?",
      "Submission Date": "Eintragsdatum",
      "Submission Time": "Eintragszeit",
      "Date": "Datum",
      "Submission": "Eintragung",
      "Note: this post is still pending so it has no submission timestamp yet.": "Hinweis: Dieser Beitrag wartet noch auf Freischaltung, daher gibt es noch kein Datum und keine Uhrzeit.",
      "User": "Benutzer",
      "Status": "Status",
      "Approved": "Genehmigt",
      "Rejected": "Abgelehnt",
      "Delete Post": "Link löschen",
      "Thanks, your post is awaiting approval.": "Vielen Dank, Dein Beitrag wartet auf Freischaltung.",
      "Sorry, couldn't find a title...": "Du hast vergessen einen Titel anzugeben...",
      "Please fill in an URL first!": "Du musst eine URL/einen Link angeben!",

      // Post item
      "Share": "Teilen",
      "Discuss": "Kommentare",
      "Upvote": "Abstimmen",
      "Sticky": "Angeheftet",
      "status": "status",
      "votes": "Stimmen",
      "baseScore": "Punktebasis",
      "score": "Punkte",
      "clicks": "klicks",
      "inactive": "inaktiv",
      "comment": "Kommentar",
      "comments": "Kommentare",
      "point": "Punkt",
      "points": "Punkte",

      //User
      "Please complete your profile below before continuing.": "Bitte füllen Dein Profil vollständig aus bevor du fortfährst.",
      "Account": "Konto",
      "Username": "Benutzername",
      "Display Name": "Angezeigter Name",
      "Email": "Email",
      "Bio": "Bio",
      "Password": "Passwort",
      "Change Password?": "Passwort ändern?",
      "Old Password": "Altes Passwort",
      "New Password": "Neues Passwort",
      "Email Notifications": "Email-Benachrichtigung",
      "New Posts": "Neue Links",
      "Comments on my posts": "Kommentare zu meinen Links",
      "Replies to my comments": "Antworten auf meine Kommentare",
      "Forgot password?": "Passwort vergessen?",
      "Profile updated": "Profil aktualisiert",
      "Please fill in your email below to finish signing up.": "Bitte trage Deine Email-Adresse ein um die Registrierung abzuschließen.",
      "Invite": "Einladen",
      "Uninvite": "Ausladen",
      "Make admin": "Zum Admin ernennen",
      "Unadmin": "Als Admin entfernen",
      "Delete User": "Benutzer löschen",
      "Are you sure you want to delete ": "Bist du Dir sicher, dass du folgendes löschen willst: ",
      "Reset Password": "Passwort zurücksetzen",
      "Password reset link sent!": "Ein Link zum zurücksetzen des Passworts wurde versendet!",
      "Name": "Name",
      "Posts": "Links",
      "Comments": "Kommentare",
      "Karma": "Karma",
      "Is Invited?": "Wurde eingeladen?",
      "Is Admin?": "Ist Admin?",
      "Delete": "Löschen",
      "Member since": "Mitglied seit",
      "Edit profile": "Profil bearbeiten",
      "Sign In": "Einloggen",
      "Sign in!": "Einloggen!",
      "Sign up!": "Registrieren!",
      "Don't have an account?": "Du hast noch kein Konto?",
      "Already have an account?": "Du hast bereits ein Konto?",
      "Sign Up": "Registrieren",
      "Please fill in all fields": "Bitte fülle alle Felder aus",
      "Invite ": "Einladung(en) ",
      "left": " übrig",
      "Invite (none left)": "Einladungen (keine übrig)",
      "All": "Alle",
      "Invited": "Eingeladen",
      "Uninvited": "Nicht eingeladen",
      "Filter by": "Filtern nach",
      "Sort by": "Sortieren nach",

      //helpers
      "Sorry, you do not have access to this page": "Sorry, Du hast keinen Zugang zu dieser Seite",
      "Please Sign In First.": "Bitte melde Dich zuerst an.",
      "Sorry, you  have to be an admin to view this page.": "Sorry, Du musst Admin sein um diese Seite anzeigen zu können.",
      "Sorry, you don't have permissions to add new items.": "Sorry, Du hast keine Berechtigung neue Einträge zu erstellen.",
      "Sorry, you cannot edit this post.": "Sorry, Du kannst diesen Beitrag nicht bearbeiten.",
      "Sorry, you cannot edit this comment.": "Sorry, Du kannst diesen Kommentar nicht bearbeiten.",

      //Collections
      "You need to login and be an admin to add a new category.": "Du musst Dich anmelden und ein Admin sein um eine neue Kategorien hinzuzufügen.",
      "You need to login or be invited to post new comments.": "Du musst dich einloggen oder eingeladen sein um neue Kommentare schreiben zu können.",
      "Please wait ": "Bitte warte ",
      " seconds before commenting again": " Sekunden, bevor du wieder kommentierst.",
      "Your comment is empty.": "Dein Kommentar ist leer.",
      "You don't have permission to delete this comment.": "Du hast keine Berechtigung diesen Kommentar zu löschen.",
      "You need to login or be invited to post new stories.": "Du musst eingeloggt oder eingeladen sein um einen neuen Link zu posten.",
      "Please fill in a headline": "Bitte fülle den Titel aus",
      "This link has already been posted": "Dieser Link wurde bereits gepostet",
      "Sorry, you cannot submit more than ": "Es tut uns leid, Du kannst nicht mehr als ",
      " posts per day": " Links pro Tag eintragen",
      "Someone replied to your comment on": "Jemand hat auf Deinen Kommentar geantwortet bei",
      " has replied to your comment on": " hat auf Deinen Kommentar geantwortet bei",
      "Read more": "weiterlesen",
      "A new comment on your post": "Ein neuer Kommentar zu Deinem Link",
      "You have a new comment by ": "Du hast einen neuen Kommentar von ",
      " on your post": " bei Deinem Link",
      " has created a new post": " hat einen neuen Link erstellt",
      "Your account has been approved.": "Dein Konto wurde freigeschaltet.",
      "Welcome to ": "Willkommen bei ",
      "Start posting.": "Fang an Links einzutragen.",

      //Common
      "Pending": "Wartet",
      "Loading...": "lädt...",
      "Submit": "Abschicken",
      "You must be logged in.": "Du musst angemeldet sein.",
      "Are you sure?": "Bist Du sicher?",
      "Please log in first": "Bitte melde Dich zuerst an",
      "Sign In/Sign Up with Twitter": "Anmelden/Registrieren mit Twitter",
      "Load more": "Mehr Laden"

};


text:
<!DOCTYPE html>

        <meta charset="utf-8" />
        <title>StandardGangliaReporter</title>
        <link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />

Description:

Reporting Task that reports metrics to a Ganglia server. The following metrics are reported:

FlowFiles In (5 mins): The number of FlowFiles received via Site-to-Site in the last 5 minutes

Bytes In (5 mins): The number of bytes received via Site-to-Site in the last 5 minutes

FlowFiles Out (5 mins): The number of FlowFiles pulled from Output Ports via Site-to-Site in the last 5 minutes

Bytes Out (5 mins): The number of bytes pulled from Output Ports via Site-to-Site in the last 5 minutes

Bytes Read (5 mins): The number of bytes read from disk by NiFi in the last 5 minutes

Bytes Written (5 mins): The number of bytes written to disk by NiFi in the last 5 minutes

FlowFiles Queued: The total number of FlowFiles currently queued on the system at the point in time at which the Reporting Task is run

Bytes Queued: The total number of bytes allocated by the FlowFiles that are currently queued on the system at the point in time at which the Reporting Task is run

Active Threads: The number of threads actively running at the point in time at which the Reporting Task is run


text:
package com.facebook.buck.distributed;

import com.facebook.buck.apple.AppleNativeIntegrationTestUtils;
import com.facebook.buck.apple.toolchain.ApplePlatform;
import com.facebook.buck.distributed.thrift.BuildJob;
import com.facebook.buck.distributed.thrift.BuildStatusResponse;
import com.facebook.buck.distributed.thrift.FrontendRequest;
import com.facebook.buck.distributed.thrift.FrontendRequestType;
import com.facebook.buck.distributed.thrift.FrontendResponse;
import com.facebook.buck.distributed.thrift.ReportCoordinatorAliveResponse;
import com.facebook.buck.distributed.thrift.SetFinalBuildStatusResponse;
import com.facebook.buck.rules.Cell;
import com.facebook.buck.testutil.ProcessResult;
import com.facebook.buck.testutil.TemporaryPaths;
import com.facebook.buck.testutil.integration.FakeFrontendHttpServer;
import com.facebook.buck.testutil.integration.ProjectWorkspace;
import com.facebook.buck.testutil.integration.TestDataHelper;
import com.facebook.buck.util.environment.Platform;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;

public class DistBuildIntegrationTest {

  @Rule public TemporaryPaths temporaryFolder = new TemporaryPaths();

  private void runSimpleDistBuildScenario(String scenario, String targetToBuild)
      throws IOException {
    Path sourceFolderPath = temporaryFolder.newFolder("source");
    Path stateFilePath = temporaryFolder.getRoot().resolve("state_dump.bin");
    Path destinationFolderPath = temporaryFolder.newFolder("destination");

    ProjectWorkspace sourceWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, scenario, sourceFolderPath);
    sourceWorkspace.setUp();

    sourceWorkspace
        .runBuckBuild(
            targetToBuild, "--distributed", "--build-state-file", stateFilePath.toString())
        .assertSuccess();

    ProjectWorkspace destinationWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, "empty", destinationFolderPath);
    destinationWorkspace.setUp();

    runDistBuildWithFakeFrontend(
            destinationWorkspace,
            "--build-state-file",
            stateFilePath.toString(),
            "--buildslave-run-id",
            "sl1")
        .assertSuccess();
  }

  @Test
  public void canBuildJavaCode() throws Exception {
    runSimpleDistBuildScenario("simple_java_target", "//:lib1");
  }

  @Test
  public void canBuildAppleBundles() throws Exception {
    Path sourceFolderPath = temporaryFolder.newFolder("source");
    Path destinationFolderPath = temporaryFolder.newFolder("destination");
    Path stateFilePath = temporaryFolder.getRoot().resolve("state_dump.bin");

    Assume.assumeTrue(Platform.detect() == Platform.MACOS);
    Assume.assumeTrue(
        AppleNativeIntegrationTestUtils.isApplePlatformAvailable(ApplePlatform.MACOSX));

    String scenario = "apple_bundle";
    ProjectWorkspace sourceWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, scenario, sourceFolderPath);
    sourceWorkspace.setUp();

    sourceWorkspace
        .runBuckBuild(
            "//:DemoApp#iphonesimulator-x86_64,no-debug",
            "--distributed",
            "--build-state-file",
            stateFilePath.toString())
        .assertSuccess();

    ProjectWorkspace destinationWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, "empty", destinationFolderPath);
    destinationWorkspace.setUp();

    runDistBuildWithFakeFrontend(
            destinationWorkspace,
            "--build-state-file",
            stateFilePath.toString(),
            "--buildslave-run-id",
            "i_am_slave_with_run_id_42")
        .assertSuccess();
  }

  @Test
  public void canBuildJumpingIntoSecondaryCellAndBackToMainCell() throws Exception {
    testCrossCell("multi_cell_out_and_back");
  }

  @Test
  public void canBuildCrossCellWithGenRules() throws Exception {
    testCrossCell("multi_cell_genrule_target");
  }

  private void testCrossCell(String scenario) throws Exception {
    Path sourceFolderPath = temporaryFolder.newFolder("source");
    Path destinationFolderPath = temporaryFolder.newFolder("destination");
    Path stateFilePath = temporaryFolder.getRoot().resolve("state_dump.bin");

    ProjectWorkspace mainCellWorkspace = setupCell(scenario, "main_cell", sourceFolderPath);
    setupCell(scenario, "secondary_cell", sourceFolderPath);

    mainCellWorkspace
        .runBuckBuild(
            "//:cross_cell_gen_rule",
            "--distributed",
            "--build-state-file",
            stateFilePath.toString())
        .assertSuccess();

    ProjectWorkspace destinationWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, "empty", destinationFolderPath);
    destinationWorkspace.setUp();

    runDistBuildWithFakeFrontend(
            destinationWorkspace,
            "--build-state-file",
            stateFilePath.toString(),
            "--buildslave-run-id",
            "i_am_slave_with_run_id_42")
        .assertSuccess();
  }

  @Test
  public void canBuildCrossCellWithSymlinksAndAbsPathTools() throws Exception {
    Path sourceFolderPath = temporaryFolder.newFolder("source");
    Path destinationFolderPath = temporaryFolder.newFolder("destination");
    String scenario = "multi_cell_java_target";
    Path stateFilePath = temporaryFolder.getRoot().resolve("state_dump.bin");

    ProjectWorkspace mainCellWorkspace = setupCell(scenario, "main_cell", sourceFolderPath);
    setupCell(scenario, "secondary_cell", sourceFolderPath);
    ProjectWorkspace absPathWorkspace = setupCell(scenario, "abs_path_dir", sourceFolderPath);
    Cell mainCell = mainCellWorkspace.asCell();

    Path dJavaFileAbsPath = absPathWorkspace.asCell().getFilesystem().resolve("D.java");
    Path dJavaFileSymlinkAbsPath = mainCell.getFilesystem().resolve("D.java");
    mainCell.getFilesystem().createSymLink(dJavaFileSymlinkAbsPath, dJavaFileAbsPath, false);

    mainCellWorkspace
        .runBuckBuild("//:libA", "--distributed", "--build-state-file", stateFilePath.toString())
        .assertSuccess();

    ProjectWorkspace destinationWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(this, "empty", destinationFolderPath);
    destinationWorkspace.setUp();

    runDistBuildWithFakeFrontend(
            destinationWorkspace,
            "--build-state-file",
            stateFilePath.toString(),
            "--buildslave-run-id",
            "sl1")
        .assertSuccess();
  }

  @Test
  public void coercerDoesNotCheckFileExistence() throws Exception {
    // To be able to check this while we 'preload' all recorded dependencies (touch the source
    // files), we create a scenario with a version dependency that gets shaved off in the
    // VersionedTargetGraph. So this dependency doesn't get recorded for pre-loading, and the
    // coercer must skip existence check for this file.

    // Explanation of test scenario: Both lib1 and lib2 exist as dependencies in the
    // UnversionedTargetGraph. But lib1 is shaved off before we record dependencies from the
    // VersionedTargetGraph. When we generate the UnversionedTargetGraph again on the slave, the
    // default PathTypeCoercer checks for the existence of files for lib1, which do not exist,
    // hence failing the test (unless we use the DO_NOT_VERIFY mode).
    runSimpleDistBuildScenario("versioned_target", "//:bin");
  }

  @Test
  public void preloadingMaterializesWhitelist() throws Exception {
    Assume.assumeTrue(Platform.detect() != Platform.WINDOWS);
    runSimpleDistBuildScenario("preloading_whitelist", "//:libA");
  }

  private ProjectWorkspace setupCell(String scenario, String cellSubDir, Path outputDir)
      throws IOException {
    Path cellPath = outputDir.resolve(cellSubDir);
    ProjectWorkspace sourceWorkspace =
        TestDataHelper.createProjectWorkspaceForScenario(
            this, scenario + "/" + cellSubDir, cellPath);
    sourceWorkspace.setUp();
    return sourceWorkspace;
  }

  private static ProcessResult runDistBuildWithFakeFrontend(
      ProjectWorkspace workspace, String... args) throws IOException {
    List<String> argsList = Lists.newArrayList(args);
    try (Server frontendServer = new Server()) {
      argsList.add(frontendServer.getStampedeConfigArg());
      argsList.add(frontendServer.getPingEndpointConfigArg());
      return workspace.runBuckDistBuildRun(argsList.toArray(new String[0]));
    }
  }

  private static class Server extends FakeFrontendHttpServer {

    public Server() throws IOException {}

    @Override
    public FrontendResponse handleRequest(FrontendRequest request) {
      switch (request.getType()) {
        case REPORT_COORDINATOR_ALIVE:
          return new FrontendResponse()
              .setType(FrontendRequestType.REPORT_COORDINATOR_ALIVE)
              .setWasSuccessful(true)
              .setReportCoordinatorAliveResponse(new ReportCoordinatorAliveResponse());

        case SET_FINAL_BUILD_STATUS:
          return new FrontendResponse()
              .setType(FrontendRequestType.SET_FINAL_BUILD_STATUS)
              .setWasSuccessful(true)
              .setSetFinalBuildStatusResponse(new SetFinalBuildStatusResponse());

        case BUILD_STATUS:
          return new FrontendResponse()
              .setType(FrontendRequestType.BUILD_STATUS)
              .setWasSuccessful(true)
              .setBuildStatusResponse(
                  new BuildStatusResponse()
                      .setBuildJob(
                          new BuildJob()
                              .setBuckBuildUuid("11-22")
                              .setStampedeId(request.getBuildStatusRequest().getStampedeId())));

          // $CASES-OMITTED$
        default:
          Assert.fail("Unexpected request type: " + request.getType());
          // Never gets here.
          return new FrontendResponse();
      }
    }
  }
}


text:
package io.appium.java_client.android;

import io.appium.java_client.remote.MobileCapabilityType;
import io.appium.java_client.service.local.AppiumDriverLocalService;

import org.junit.AfterClass;
import org.junit.BeforeClass;

import org.openqa.selenium.remote.DesiredCapabilities;

import java.io.File;

public class BaseAndroidTest {
    private static AppiumDriverLocalService service;
    protected static AndroidDriver<AndroidElement> driver;

    /**
     * initialization.
     */
    @BeforeClass public static void beforeClass() throws Exception {
        service = AppiumDriverLocalService.buildDefaultService();
        service.start();

        if (service == null || !service.isRunning()) {
            throw new RuntimeException("An appium server node is not started!");
        }

        File appDir = new File("src/test/java/io/appium/java_client");
        File app = new File(appDir, "ApiDemos-debug.apk");
        DesiredCapabilities capabilities = new DesiredCapabilities();
        capabilities.setCapability(MobileCapabilityType.DEVICE_NAME, "Android Emulator");
        capabilities.setCapability(MobileCapabilityType.APP, app.getAbsolutePath());
        driver = new AndroidDriver<>(service.getUrl(), capabilities);
    }

    /**
     * finishing.
     */
    @AfterClass public static void afterClass() {
        if (driver != null) {
            driver.quit();
        }
        if (service != null) {
            service.stop();
        }
    }
}


text:
title: "Traefik Data Collection Documentation"
description: "To learn more about how Traefik is being used and improve it, we collect anonymous usage statistics from running instances. Read the technical documentation."
---

# Data Collection

Understanding How Traefik is Being Used
{: .subtitle }

## Configuration Example

Understanding how you use Traefik is very important to us: it helps us improve the solution in many different ways.
For this very reason, the sendAnonymousUsage option is mandatory: we want you to take time to consider whether or not you wish to share anonymous data with us so we can benefit from your experience and use cases.

!!! example "Enabling Data Collection"

    ```yaml tab="File (YAML)"
    global:
      # Send anonymous usage data
      sendAnonymousUsage: true
    ```

    ```toml tab="File (TOML)"
    [global]
      # Send anonymous usage data
      sendAnonymousUsage = true
    ```

    ```bash tab="CLI"
    # Send anonymous usage data
    --global.sendAnonymousUsage
    ```

## Collected Data

This feature comes from the public proposal [here](https://github.com/traefik/traefik/issues/2369).

This feature is activated when using Traefik Pilot to better understand the community's need, and also to get information about plug-ins popularity.

In order to help us learn more about how Traefik is being used and improve it, we collect anonymous usage statistics from running instances.
Those data help us prioritize our developments and focus on what's important for our users (for example, which provider is popular, and which is not).

### What's collected / when ?

Once a day (the first call begins 10 minutes after the start of Traefik), we collect:

- the Traefik version number
- a hash of the configuration
- an **anonymized version** of the static configuration (token, user name, password, URL, IP, domain, email, etc, are removed).

!!! info

    - We do not collect the dynamic configuration information (routers & services).
    - We do not collect this data to run advertising programs.
    - We do not sell this data to third-parties.

### Example of Collected Data

```yaml tab="Original configuration"
entryPoints:
  web:
  address: ":80"

api: {}

providers:
  docker:
    endpoint: "tcp://10.10.10.10:2375"
    exposedByDefault: true
    swarmMode: true

    tls:
      ca: dockerCA
      cert: dockerCert
      key: dockerKey
      insecureSkipVerify: true
```

```yaml tab="Resulting Obfuscated Configuration"
entryPoints:
  web:
  address: ":80"

api: {}

providers:
  docker:
    endpoint: "xxxx"
    exposedByDefault: true
    swarmMode: true

    tls:
      ca: xxxx
      cert: xxxx
      key: xxxx
      insecureSkipVerify: true
```

## The Code for Data Collection

If you want to dig into more details, here is the source code of the collecting system: [collector.go](https://github.com/traefik/traefik/blob/master/pkg/collector/collector.go)

By default we anonymize all configuration fields, except fields tagged with `export=true`.


text:
package org.datanucleus.samples.one_many.bidir_3;

import java.util.ArrayList;
import java.util.List;

/**
 * Abstract container.
 */
public abstract class AbstractSky
{
    long id;

    List<Cloud> clouds;

    public void setId(long id)
    {
        this.id = id;
    }

    public List<Cloud> getClouds()
    {
        if (this.clouds == null)
        {
            this.clouds = new ArrayList<Cloud>();
        }
        return java.util.Collections.unmodifiableList(clouds);
    }

    public void addCloud(Cloud cl)
    {
        if (this.clouds == null)
        {
            this.clouds = new ArrayList<>();
        }
        this.clouds.add(cl);
    }

    public void removeCloud(Cloud cl)
    {
        this.clouds.remove(cl);
    }
}


text:
var gulp = require('gulp');
var runSequence = require('run-sequence');

gulp.task('default', function(callback) {
  runSequence('lint', 'spec', callback);
});


text:
package com.github.javaparser.ast.nodeTypes;

import java.util.Arrays;
import java.util.EnumSet;
import java.util.stream.Collectors;

import com.github.javaparser.ast.Modifier;

/**
 * A Node with Modifiers.
 */
public interface NodeWithModifiers<T> {
    /**
     * Return the modifiers of this variable declaration.
     *
     * @see Modifier
     * @return modifiers
     */
    EnumSet<Modifier> getModifiers();

    T setModifiers(EnumSet<Modifier> modifiers);

    @SuppressWarnings("unchecked")
    default T addModifier(Modifier... modifiers) {
        getModifiers().addAll(Arrays.stream(modifiers)
                .collect(Collectors.toCollection(() -> EnumSet.noneOf(Modifier.class))));
        return (T) this;
    }

    default boolean isStatic() {
        return getModifiers().contains(Modifier.STATIC);
    }

    default boolean isAbstract() {
        return getModifiers().contains(Modifier.ABSTRACT);
    }

    default boolean isFinal() {
        return getModifiers().contains(Modifier.FINAL);
    }

    default boolean isNative() {
        return getModifiers().contains(Modifier.NATIVE);
    }

    default boolean isPrivate() {
        return getModifiers().contains(Modifier.PRIVATE);
    }

    default boolean isProtected() {
        return getModifiers().contains(Modifier.PROTECTED);
    }

    default boolean isPublic() {
        return getModifiers().contains(Modifier.PUBLIC);
    }

    default boolean isStrictfp() {
        return getModifiers().contains(Modifier.STRICTFP);
    }

    default boolean isSynchronized() {
        return getModifiers().contains(Modifier.SYNCHRONIZED);
    }

    default boolean isTransient() {
        return getModifiers().contains(Modifier.TRANSIENT);
    }

    default boolean isVolatile() {
        return getModifiers().contains(Modifier.VOLATILE);
    }
}