text:
package de.uhh.lt.webanno.mm.io;

import de.tudarmstadt.ukp.dkpro.core.api.io.JCasResourceCollectionReader_ImplBase;
import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
import de.uhh.lt.webanno.mm.io.HiatTeiMetadata.Description;
import de.uhh.lt.webanno.mm.io.HiatTeiMetadata.Speaker;
import de.uhh.lt.webanno.mm.type.*;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.CASException;
import org.apache.uima.cas.Feature;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.jdom2.*;
import org.jdom2.filter.ElementFilter;
import org.jdom2.input.SAXBuilder;
import org.jdom2.output.XMLOutputter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.xml.stream.XMLStreamException;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.*;
import java.util.regex.Matcher;
import java.util.stream.Stream;

import static de.uhh.lt.webanno.mm.io.HiatTeiReaderUtils.*;
import static org.apache.commons.io.IOUtils.closeQuietly;

/**
 * Reader for the HIAT TEI format
 *
 */
public class HiatTeiReaderExt extends JCasResourceCollectionReader_ImplBase {

    private static final Logger LOG = LoggerFactory.getLogger(HiatTeiReaderExt.class);

    private static char getUtteranceEndSignature(String utteranceSubtype){
        if("declarative".equals(utteranceSubtype))
            return '.';
        if("exclamative".equals(utteranceSubtype))
            return '!';
        if("interrupted".equals(utteranceSubtype))
            return '…';
        if("interrogative".equals(utteranceSubtype))
            return '?';
        if("modeless".equals(utteranceSubtype))
            return '\u02d9';
        return ' ';
    }

    private static String getXMLID(Element e) {
        for(Attribute a : e.getAttributes())
            if(a.getName().equals("id"))
                return a.getValue();
        return null;
    }

    @Override
    public void getNext(JCas textview)
            throws IOException, CollectionException
    {
        Resource res = nextFile();
        String fname = res.getResolvedUri().toString();
        LOG.info("Reading '{}'", fname);
        initCas(textview, res);

        InputStream is = null;
        try {
            is = res.getInputStream();
            // store the tei sources in a separate view
            JCas teiview = JCasUtil.getView(textview, "tei", true);
            teiview.setDocumentText(IOUtils.toString(is, Charset.defaultCharset()));
            closeQuietly(is);

            is = new BufferedInputStream(res.getInputStream());
            read(is, textview, fname);
        }
        catch (IOException | XMLStreamException | JDOMException e) {
            LOG.error("Reading '{}' failed.", fname);
            logError(LOG, e);
            throw new CollectionException(e);
        }
        finally {
            closeQuietly(is);
        }
    }

    private void read(InputStream is, JCas textview, String source) throws IOException, XMLStreamException, JDOMException, CollectionException  {
        SAXBuilder saxBuilder = new SAXBuilder();
        Document document = saxBuilder.build(is);
        Element root = document.getRootElement();

        HiatTeiMetadata meta = readTeiHeader(textview, root);

        readTeiText(textview, meta, root, saxBuilder);
        meta.addToCas(textview);
    }

    private HiatTeiMetadata readTeiHeader(JCas textview, Element root) throws IOException  {
        Namespace ns = root.getNamespace();
        HiatTeiMetadata meta = HiatTeiMetadata.newInstance();
        meta.created_with = getClass();

        // get teiHeader Element
        Element teiheader = root.getChild("teiHeader", ns);
        if(teiheader == null) {
            logWarning(LOG, new IllegalStateException("teiHeader element not found!"));
            return null;
        }
        meta.tei_header_xml = new XMLOutputter().outputString(teiheader);

        /* read filedesc */
        Element filedescription_element = teiheader.getChild("fileDesc", ns);
        String filedesc_xml = new XMLOutputter().outputString(filedescription_element);
        ElementFilter filter = new ElementFilter("title");
        Iterator<Element> titles = filedescription_element.getDescendants(filter);
        String title = null;
        if(titles.hasNext()){
            Element title_element = titles.next();
            title = StringEscapeUtils.unescapeXml(title_element.getTextNormalize());
        }

        String settingsdesc_xml = "";
        Element profiledescription_element = teiheader.getChild("profileDesc", ns);
        if(profiledescription_element != null){
            Element settingsdescription_element = profiledescription_element.getChild("settingDesc", ns);
            if(settingsdescription_element != null){
                settingsdesc_xml = new XMLOutputter().outputString(settingsdescription_element);
            }
        }
        meta.description = new Description(title, filedesc_xml, settingsdesc_xml);

        /* read media */
        filter = new ElementFilter("media");
        for(Element media_element : filedescription_element.getDescendants(filter)) {
            String mimetype = media_element.getAttributeValue("mimeType");
            String url = media_element.getAttributeValue("url");
            meta.media.add(new HiatTeiMetadata.Media(meta.media.size(), mimetype, url));
        }

        /* read person descriptions */
        filter = new ElementFilter("person");
        for(Element person_element : teiheader.getDescendants(filter)) {
            String xml = new XMLOutputter().outputString(person_element);
            String id = null;
            for(Attribute a : person_element.getAttributes()) {
                if(a.getName().equals("id"))
                    id = a.getValue();
            }

            meta.speakers.add(
                    new HiatTeiMetadata.Speaker(
                            meta.speakers.size(),
                            id,
                            person_element.getAttributeValue("n"), xml));
        }
        meta.speakers.add(0, Speaker.NARRATOR); // add the narrator as a speaker
        return meta;
    }

    private void readTeiText(JCas textview, HiatTeiMetadata meta, Element root, SAXBuilder saxBuilder) throws IOException, CollectionException  {

        /* read timeline */
        ElementFilter filter = new ElementFilter("timeline");
        for(Element timeline : root.getDescendants(filter)) {

            ElementFilter filter2 = new ElementFilter("when");
            for(Element timevalue : timeline.getDescendants(filter2)) {
                String id = getXMLID(timevalue);
                Attribute a = timevalue.getAttribute("interval");
                float interval = 0f;
                try {
                    interval = a != null ? a.getFloatValue() : 0f;
                } catch (DataConversionException e) {
                    logWarning(LOG, e);
                }
                String sinceId = timevalue.getAttributeValue("since");
                // add to metadata
                meta.timeline.add(new HiatTeiMetadata.Timevalue(meta.timeline.size(), id, interval, sinceId));
            }
        }

        /* store everything in temporary textview */
        JCas tempview = JCasUtil.getView(textview, "_temp", true);

        /* read utterances (also within annotationblocks), fills textview with data */
        parseUtterances(tempview, meta, root, saxBuilder);

        /* read span annotations and add them to the textview */
        parseSpanAnnotations(tempview, meta, root);

        /* create speaker views */
        createSpeakerViews(tempview, meta);

        /* read incidents */
        parseUnattachedIncidents(tempview, meta, root);

        /* reorder segments, add to textview and clean up temporary textview */
        fillTextview(tempview, textview);

        /* cleanup */
        meta.textview_speaker_id_anno_index.clear();

        try {
            List<String> viewnames = new ArrayList<String>();
            Iterator<JCas> i = textview.getViewIterator();
            for(JCas j = i.next(); i.hasNext(); j = i.next()){
                viewnames.add(j.getViewName());
            }
            LOG.info(String.format("created %d views: %s", viewnames.size(), viewnames.toString()));
        } catch (CASException e) {
            LOG.warn("Could not get viewnames.", e);
        }
    }

    void fillTextview(JCas tempview, JCas textview){
        textview.setDocumentText(tempview.getDocumentText());
        Stream<Annotation> annotations = JCasUtil.selectAll(tempview).stream().filter(a -> a instanceof Annotation).map(a -> (Annotation)a);
        copyAnnotations(annotations,textview)
            .filter(pa -> {
                if(pa.getRight() == null)
                    LOG.warn("Annotation {} was not copied for an unknown reason. Please check logs.", pa.getLeft());
                return pa != null;
            }).forEach(pa -> pa.getRight().addToIndexes(textview));

    }

    private void parseUtterances(JCas textview, HiatTeiMetadata meta, Element root, SAXBuilder saxBuilder) {
        StringBuilder text = new StringBuilder();
        int count_sentences = 1;
        Queue<Incident> incidents_to_finish = new LinkedList<>();

        ElementFilter filter = new ElementFilter("u");
        for(Element utterance : root.getDescendants(filter)) {
            String id = getXMLID(utterance);

            // create utterance
            Utterance utterance_textview = new Utterance(textview);
            utterance_textview.setBegin(text.length());
            utterance_textview.setID(id);

            // find annotationblock
            Element annotationBlock = utterance.getParentElement();
            if(annotationBlock != null && !annotationBlock.getName().equals("annotationBlock"))
                annotationBlock = null;

            // resolve speaker
            String speakerID = "";
            if(utterance.getAttribute("who") == null && annotationBlock != null && annotationBlock.getAttribute("who") != null)
                speakerID = annotationBlock.getAttributeValue("who");
            else
                speakerID = utterance.getAttributeValue("who");
            speakerID = StringUtils.stripStart(speakerID, "#");
            Speaker speaker = meta.getSpeakerById(speakerID);
            utterance_textview.setSpeakerABBR(speaker.n);
            utterance_textview.setSpeakerID(speakerID);

            // resolve start id
            String startID = "";
            if(utterance.getAttribute("start") == null && annotationBlock != null && annotationBlock.getAttribute("start") != null)
                startID = annotationBlock.getAttributeValue("start");
            else
                startID = utterance.getAttributeValue("start");
            startID = StringUtils.stripStart(startID, "#");
            utterance_textview.setStartID(startID);

            // resolve end id
            String endID = "";
            if(utterance.getAttribute("end") == null && annotationBlock != null && annotationBlock.getAttribute("end") != null)
                endID = annotationBlock.getAttributeValue("end");
            else
                endID = utterance.getAttributeValue("end");
            endID = StringUtils.stripStart(endID, "#");
            utterance_textview.setEndID(endID);

            // each utterance might contain segments (may be empty / non-text), anchors, ...
            List<Element> utterance_kids = utterance.getChildren();

            // create a text anchor at the beginning of each utterance, but only add it to indexes if text was produced
            Anchor ta = meta.addAnchorToIndex(speaker, addAnchor(textview, text, text.length(), speaker.id, id, startID, incidents_to_finish, false, count_sentences));

            for(Element sub_utterance : utterance_kids){
                String tag = sub_utterance.getName();
                if("anchor".equals(tag)) { // found an anchor
                    /* create a new anchor
                     * textview: insert zero-length annotation (the flag!)
                     */
                    ta = meta.addAnchorToIndex(speaker,
                            addAnchor(textview,
                                    text,
                                    text.length(),
                                    speaker.id,
                                    id,
                                    StringUtils.strip(sub_utterance.getAttributeValue("synch"), "#"),
                                    incidents_to_finish,
                                    true,
                                    count_sentences));

                } else if("seg".equals(tag)) {

                    // prepare text segment
                    // <seg xml:id="seg21" type="utterance" subtype="interrupted">
                    String segID = StringUtils.strip(getXMLID(sub_utterance), "#");
                    String segType = sub_utterance.getAttributeValue("type");
                    String segSubtype = sub_utterance.getAttributeValue("subtype");

                    String last_anchor_id = ta.getID();

                    Segment segment_annotation_textview = new Segment(textview);
                    segment_annotation_textview.setBegin(text.length());
                    segment_annotation_textview.setSegmentType(segType);
                    segment_annotation_textview.setSegmentSubtype(segSubtype);
                    segment_annotation_textview.setID(segID);

                    for(Element element : sub_utterance.getChildren()) {
                        if("unclear".equals(element.getName())){
                            text.append('(');
                            new Token(textview, text.length()-1, text.length()).addToIndexes();
                            Iterator<Element> unclear_elements_iter = element.getChildren().iterator();
                            while(unclear_elements_iter.hasNext()){
                                int textoffset = text.length();
                                ta = processSegmentChild(textview, meta, speaker, text, id, unclear_elements_iter.next(), ta, incidents_to_finish, saxBuilder, count_sentences);
                                if(textoffset != text.length() && unclear_elements_iter.hasNext())
                                    text.append(' ');
                            }
                            text.append(')');
                            new Token(textview, text.length()-1, text.length()).addToIndexes();
                            text.append(' ');
                        } else {
                            int textoffset = text.length();
                            ta = processSegmentChild(textview, meta, speaker, text, id, element, ta, incidents_to_finish, saxBuilder, count_sentences);
                            if(textoffset != text.length() && ' ' != text.charAt(text.length()-1))
                                text.append(' ');
                        }
                    }

                    if(segment_annotation_textview.getBegin() < text.length()) { // did we enter the loop? if yes create a sentence annotation for the segment and a playable segment anchor
                        // some text was produced, create a sentence segment and add a new line
                        // append the segment end signature
                        while(text.length() > 0 && ' ' == text.charAt(text.length()-1))
                            text.deleteCharAt(text.length()-1);
                        char mode = getUtteranceEndSignature(segSubtype);
                        if(mode != ' ') {
                            int tb = text.length();
                            text.append(mode);
                            new Token(textview, tb, text.length()).addToIndexes(textview);
                        }
                        int te = findLastNonSpace(text); // -space and newline
                        new Sentence(textview,segment_annotation_textview.getBegin(), te).addToIndexes(textview); // end w/o space
                        segment_annotation_textview.setSentenceNumber(count_sentences++);
                        segment_annotation_textview.setEnd(te);
                        segment_annotation_textview.addToIndexes(textview);

                        PlayableSegmentAnchor taps = new PlayableSegmentAnchor(textview, segment_annotation_textview.getBegin(), segment_annotation_textview.getBegin());
                        taps.setInfo(String.format("%s▶", utterance_textview.getSpeakerABBR()));
                        taps.setAnchorID(last_anchor_id);
                        taps.addToIndexes(textview);

                        text.append('\n');
                    }
                }
            }

            // add a newline if text was produced, also only add the playable begin text anchor of text was produced
            if(utterance_textview.getBegin() < text.length()){
                // also, create a text anchor with the end id of the utterance
                int te = findLastNonSpace(text); // -space and newline
                meta.addAnchorToIndex(speaker, addAnchor(textview, text, te, speaker.id, id, endID, incidents_to_finish, false, count_sentences));
                utterance_textview.setEnd(findLastNonSpace(text)); // -space and newline
                text.append("\n");
            } else {
                // also, create a text anchor with the end id of the utterance
                meta.addAnchorToIndex(speaker, addAnchor(textview, text, text.length(), speaker.id, id, endID, incidents_to_finish, false, count_sentences));
                utterance_textview.setEnd(findLastNonSpace(text));
            }
            utterance_textview.addToIndexes(textview);
        }
        textview.setDocumentText(text.toString());
    }

    private void parseSpanAnnotations(JCas textview, HiatTeiMetadata meta, Element root) throws CollectionException {

        ElementFilter filter = new ElementFilter("annotationBlock");
        for(Element annotation_block : root.getDescendants(filter)) {

            // resolve speaker
            String spk_id = StringUtils.stripStart(annotation_block.getAttributeValue("who"), "#");
            Speaker spk = meta.getSpeakerById(spk_id);

            ElementFilter filter2 = new ElementFilter("span");
            for(Element span : annotation_block.getDescendants(filter2)) {
                // get type from parent spangrp
                String type = "undefined";
                Element span_grp = span.getParentElement();
                if("spanGrp".equals(span_grp.getName()))
                    type = span_grp.getAttributeValue("type");

                String ID_start = StringUtils.stripStart(span.getAttributeValue("from"), "#");;
                String ID_end = StringUtils.stripStart(span.getAttributeValue("to"), "#");
                Annotation x = meta.getElementAnnotation(spk, ID_start);
                if(x == null)
                    throw new CollectionException(new IllegalStateException(String.format("Something is wrong with referencing id %s for speaker. Cannot find text offset.", ID_start, spk)));
                int begin = x.getBegin();
                begin = findFirstNonSpace(textview.getDocumentText(), begin);
                x = meta.getElementAnnotation(spk, ID_end);
                if(x == null)
                    throw new CollectionException(new IllegalStateException(String.format("Something is wrong with referencing id %s for speaker. Cannot find text offset.", ID_start, spk)));
                int end = x.getEnd();
                end = findLastNonSpace(textview.getDocumentText(), end);

                String content = StringEscapeUtils.unescapeXml(span.getTextNormalize().trim());
                if("pos".equals(type)){
                    POS pos = new POS(textview, begin, end);
                    pos.setPosValue(content);
                    pos.addToIndexes(textview);
                }else if("lemma".equals(type)){
                    Lemma lemma = new Lemma(textview, begin, end);
                    lemma.setValue(content);
                    lemma.addToIndexes(textview);
                }else{
                	switch(type) {
                	case "akz":
                        TEIspanAkz span_annotation = new TEIspanAkz(textview, begin, end);
                        setTEIspanInformation(span_annotation, spk_id, ID_start, ID_end, content, type);
                        span_annotation.addToIndexes(textview);
                		break;
                	case "k":
                        TEIspanK span_annotation1 = new TEIspanK(textview, begin, end);
                        setTEIspanInformation(span_annotation1, spk_id, ID_start, ID_end, content, type);
                        span_annotation1.addToIndexes(textview);
                		break;
                	case "sup":
                        TEIspanSup span_annotation2 = new TEIspanSup(textview, begin, end);
                        setTEIspanInformation(span_annotation2, spk_id, ID_start, ID_end, content, type);
                        span_annotation2.addToIndexes(textview);
                		break;
                	case "en":
                        TEIspanEn span_annotation3 = new TEIspanEn(textview, begin, end);
                        setTEIspanInformation(span_annotation3, spk_id, ID_start, ID_end, content, type);
                        span_annotation3.addToIndexes(textview);
                		break;
                	default:
                        String classname = "webanno.custom.TEISpan"+type;
                        Type aType = textview.getTypeSystem().getType(classname);
                        if(aType != null) {
                            CAS aCas = textview.getCas();
                            AnnotationFS annotationFS = aCas.createAnnotation(aType, begin, end);
                            for(Feature f : aType.getFeatures()) {
                                switch(f.getShortName()) {
                                    case "Content":
                                        annotationFS.setStringValue(f, content);
                                        break;
                                    case "SpanType":
                                        annotationFS.setStringValue(f, type);
                                        break;
                                    case "SpeakerID":
                                        annotationFS.setStringValue(f, spk_id);
                                        break;
                                    case "StartID":
                                        annotationFS.setStringValue(f, ID_start);
                                        break;
                                    case "EndID":
                                        annotationFS.setStringValue(f, ID_end);
                                        break;
                                    default:
                                        break;
                                }
                            }
                            aCas.addFsToIndexes(annotationFS);
                        } else {
                            TEIspanGeneric span_annotation4 = new TEIspanGeneric(textview, begin, end);
                            setTEIspanInformation(span_annotation4, spk_id, ID_start, ID_end, content, type);
                            span_annotation4.addToIndexes(textview);
                        }
                        break;
                	}
                    if(!type.startsWith("morph"))
                        // keep track of the various span types
                        meta.spantypes.add(type);
                }
            }
        }
    }

    private void setTEIspanInformation(TEIspan span, String spk_id, String ID_start, String ID_end, String content, String type) {
        span.setSpeakerID(spk_id);
        span.setStartID(ID_start);
        span.setEndID(ID_end);
        span.setContent(content);
        span.setSpanType(type);
    }

    private void createSpeakerViews(JCas textview, HiatTeiMetadata meta) {
        for(Speaker spkr : meta.speakers){
            final String speakerId = spkr.id;
            // create a view for the speaker
            final JCas speakerview = JCasUtil.getView(textview, speakerId + "_", true);
            final StringBuilder speakerText = new StringBuilder();
            // get all utterances for the current speaker
            JCasUtil.select(textview, Utterance.class).stream()
            .filter(u -> speakerId.equals(u.getSpeakerID()))
            .sorted((s1,s2) -> Integer.compare(s1.getBegin(), s2.getBegin())) // ensure ordering
            .forEachOrdered(textutterance -> {
                // create a new utterance
                final Utterance speakerUtterance = new Utterance(speakerview);
                speakerUtterance.setBegin(speakerText.length());
                speakerText.append(textutterance.getCoveredText());
                speakerUtterance.setEnd(speakerText.length());
                speakerUtterance.setID(textutterance.getID());
                speakerUtterance.setSpeakerID(speakerId);
                speakerUtterance.addToIndexes(speakerview);

                /* get all segments for the current utterance
                 * and add them to the speakerview,
                 * also add the covered text
                 */
                for(Segment textsegment : JCasUtil.selectCovered(textview, Segment.class, textutterance)){
                    // create a segment annotation for the speaker view
                    final Segment speakersegment = new Segment(speakerview);
                    // calculate the offset within textutterance and add offset of speakerutterance
                    speakersegment.setBegin(textsegment.getBegin() - textutterance.getBegin() + speakerUtterance.getBegin());
                    speakersegment.setEnd(textsegment.getEnd() - textutterance.getBegin() + speakerUtterance.getBegin());
                    speakersegment.setID(textsegment.getID());
                    speakersegment.setSentenceNumber(textsegment.getSentenceNumber());
                    speakersegment.setUtteranceID(textsegment.getUtteranceID());
                    speakersegment.setSegmentType(textsegment.getSegmentType());
                    speakersegment.setSegmentSubtype(textsegment.getSegmentSubtype());
                    speakersegment.addToIndexes(speakerview);
                }

                /* add covered annotations, at least TextAnchor annotations */
                copyAnnotations(JCasUtil.selectCovered(textview, Annotation.class, textutterance).stream(), speakerview)
                    .filter(pa -> {
                        if(pa.getRight() == null)
                            LOG.warn("Annotation {} was not copied for an unknown reason. Please check logs.", pa.getLeft());
                        return pa != null;
                    })
                    .map(pa -> pa.getRight())
                    .forEach(a -> {
                        a.setBegin(a.getBegin() - textutterance.getBegin() + speakerUtterance.getBegin());
                        a.setEnd(a.getEnd() - textutterance.getBegin() + speakerUtterance.getBegin());
                        a.addToIndexes(speakerview);
                    });

            });
            speakerview.setDocumentText(speakerText.toString());
        }
    }

    private Anchor processSegmentChild(JCas textview, HiatTeiMetadata meta, Speaker speaker, StringBuilder text, String id, Element element, Anchor ta, Queue<Incident> incidents_to_finish, SAXBuilder saxBuilder, int count_sentences) {
        if("anchor".equals(element.getName())){
            // create a new anchor
            Anchor new_anchor =
                    meta.addAnchorToIndex(speaker,
                            addAnchor(textview,
                                    text,
                                    text.length(),
                                    speaker.id,
                                    id,
                                    StringUtils.strip(element.getAttributeValue("synch"), "#"),
                                    incidents_to_finish,
                                    true,
                                    count_sentences));
            return new_anchor;
        } else if("pc".equals(element.getName())){
            String plaintext = StringEscapeUtils.unescapeXml(element.getTextNormalize());
            if(plaintext != null && !StringUtils.isEmpty(plaintext = plaintext.trim())) {
                while(text.length() > 0 && ' ' == text.charAt(text.length()-1))
                    text.deleteCharAt(text.length()-1);
                int tb = text.length();
                text.append(plaintext);
                if(text.length() > tb){
                    Token token = new Token(textview, tb, text.length());
                    token.addToIndexes(textview);
                    text.append(' ');

                    String element_id = getXMLID(element);
                    if(element_id != null)
                        meta.addElementToIndex(element_id, speaker, token);
                }
            }
        } else if("w".equals(element.getName())) {
            String type = element.getAttributeValue("type");
            int tb = text.length();

            // String token = element.text(); woul be too easy, anchors occurr within text
            // <w xml:id="w524">ge<anchor synch="#T142"/>we<anchor synch="#T143"/>sen</w>
            //			String plaintext = element.getText();
            //			text.append(plaintext);

            String htmltext = new XMLOutputter().outputString(element.getContent());
            int child_index = htmltext.indexOf('<');
            while(child_index >= 0){ // subelements found
                text.append(StringEscapeUtils.unescapeXml(Text.normalizeString(htmltext.substring(0, child_index).trim())));
                htmltext = htmltext.substring(child_index);
                int end_html = htmltext.indexOf("/>");
                String htmlelement = htmltext.substring(0, end_html+2);

                try {
                    InputStream stream = new ByteArrayInputStream(htmlelement.getBytes("UTF-8"));
                    Document document = saxBuilder.build(stream);
                    // add anchor
                    ElementFilter filter = new ElementFilter("anchor");
                    for(Element html_element : document.getDescendants(filter)) {
                        if(html_element != null) {
                            ta = meta.addAnchorToIndex(speaker,
                                    addAnchor(textview,
                                            text,
                                            text.length(),
                                            speaker.id,
                                            id,
                                            StringUtils.strip(html_element.getAttributeValue("synch"), "#"),
                                            incidents_to_finish,
                                            true,
                                            count_sentences));
                        }
                        htmltext = htmltext.substring(end_html+2);
                        child_index = htmltext.indexOf('<');
                    }
                } catch (JDOMException | IOException e) {
                    logWarning(LOG, e);
                }
            }
            text.append(StringEscapeUtils.unescapeXml(Text.normalizeString(htmltext.trim())));

            if("repair".equals(type))
                text.append('/');

            int te = findLastNonSpace(text);
            if(te > tb){
                Token token = new Token(textview, tb, te);
                token.addToIndexes(textview);
                String element_id = getXMLID(element);
                if(element_id != null)
                    meta.addElementToIndex(element_id, speaker, token);
            }

        } else if("pause".equals(element.getName())){
            String type = element.getAttributeValue("type");
            String dur = element.getAttributeValue("dur");
            int b = text.length();
            if(!StringUtils.isEmpty(type)){
                // short 1x
                text.append('•');
                if(!"short".equals(type)) // medium or long 2x
                    text.append(" •");
                if("long".equals(type)) // long 3x
                    text.append(" •");
            }
            if(!StringUtils.isEmpty(dur)){
                Matcher m = TIMEFINDER_PATTERN.matcher(dur);
                if(m.find()){
                    String dur_t = m.group();
                    text.append("((");
                    if(!StringUtils.isEmpty(dur_t))
                        text.append(dur_t.toLowerCase());
                    else
                        text.append(dur.toLowerCase());
                    text.append("))");
                }
            }
            if(b != text.length() && !StringUtils.isEmpty(text.substring(b, text.length()))){
                Token pt = new Token(textview, b, text.length());
                pt.addToIndexes();
                Incident incident_anno = new Incident(textview,  b, text.length());
                incident_anno.setSpeakerID(speaker.id);
                incident_anno.setDesc(pt.getCoveredText());
                incident_anno.setIncidentType("pause");
                incident_anno.setIsTextual(true);
                incident_anno.addToIndexes(textview);

            }
        }else if("incident".equals(element.getName())){
            ElementFilter filter = new ElementFilter("desc");
            for(Element description : element.getDescendants(filter)) {
                String desc = StringEscapeUtils.unescapeXml(description.getTextNormalize());
                int b = text.length();
                Token token_anno = new Token(textview);
                token_anno.setBegin(text.length());
                text.append("((");
                token_anno.setEnd(text.length());
                token_anno.addToIndexes();

                int tb = text.length();
                text.append(desc);
                int te = findLastNonSpace(text);
                if(te > tb)
                    new Token(textview, tb, te).addToIndexes();

                token_anno = new Token(textview);
                token_anno.setBegin(text.length());
                text.append("))");
                token_anno.setEnd(text.length());
                token_anno.addToIndexes();

                Incident incident_anno = new Incident(textview, findFirstNonSpace(text, b), findLastNonSpace(text));
                incident_anno.setStartID(ta.getID());
                incident_anno.setSpeakerID(speaker.id);
                incident_anno.setDesc(desc);
                incident_anno.setIncidentType("nv");
                incident_anno.setIsTextual(true);
                incidents_to_finish.add(incident_anno);

                break;
            }
        }
        return ta;
    }

    private void parseUnattachedIncidents(JCas textview, HiatTeiMetadata meta, Element root) {
        Namespace ns = root.getNamespace();
        Element text_element = root.getChild("text", ns);
        if(text_element == null) {
            logWarning(LOG, new IllegalArgumentException("No text tag found!"));
            return;
        }

        ElementFilter filter = new ElementFilter("incident");
        for(Element incident : text_element.getDescendants(filter)) {
            // get attributes
            // incidents within segments don't have a start and end tag because they are defined by the utterance and can be related to an utterance.
            // those kind of incidents are handled before in parseUtterances
            // hence, we skip incidents that don't have a start or end attribute
            // All other incidents merely describe the situation without being referenced to a particular utterance, we collect those as a collection of annotations in
            if(incident.getAttribute("start") == null && incident.getAttribute("end") == null)
                continue;

            String startAnchorID = StringUtils.stripStart(incident.getAttributeValue("start"), "#");
            String endAnchorID = StringUtils.stripStart(incident.getAttributeValue("end"), "#");

            // resolve speaker
            Speaker spk = Speaker.NARRATOR;
            if(incident.getAttribute("who") != null){
                String spk_id = StringUtils.stripStart(incident.getAttributeValue("who"), "#");
                spk = meta.getSpeakerById(spk_id);
            }

            // get description
            ElementFilter filter2 = new ElementFilter("desc");
            String desc = "";
            for(Element description : incident.getDescendants(filter2)) {
                desc = StringEscapeUtils.unescapeXml(description.getTextNormalize());
            }

            // add annotation to speaker or narrator, just collect all annotations at the beginning of the document text
            JCas speakerview = HiatTeiMetadata.getSpeakerView(textview, spk);

            Incident incident_anno = new Incident(speakerview, 0, 0);
            incident_anno.setStartID(startAnchorID);
            incident_anno.setEndID(endAnchorID);
            incident_anno.setSpeakerID(spk.id);
            incident_anno.setDesc(desc);
            incident_anno.setIncidentType("nn");
            incident_anno.addToIndexes(speakerview);
        }
    }

    private Anchor addAnchor(JCas textview, CharSequence text, int positionInText, String speakerId, String utteranceId, String anchorID, Queue<Incident> incidents_to_finish, boolean add_playable_anchor, int current_sentence_no) {
        Anchor ta = new Anchor(textview, positionInText, positionInText);
        ta.setID(anchorID);
        ta.setUtteranceID(utteranceId);
        ta.addToIndexes(textview);
        ta.setSpeakerID(speakerId);
        ta.setSentenceNumber(current_sentence_no);
        if(add_playable_anchor){
            PlayableAnchor tap = new PlayableAnchor(textview, positionInText, positionInText);
            tap.setInfo(String.format("%s▶", anchorID));
            tap.setAnchorID(anchorID);
            tap.addToIndexes(textview);
        }
        while(!incidents_to_finish.isEmpty()){
            Incident incident = incidents_to_finish.poll();
            incident.setEnd(findLastNonSpace(text, ta.getBegin()));
            incident.setEndID(ta.getID());
            incident.addToIndexes();
        }
        return ta;
    }

}


text:
import fs from 'node:fs'
import os from 'node:os'
import randomPath from 'random-path'

import { retryAsync, retrySync } from './retry.js'
import WriteStream from './write-stream.js'

const tmpdir = os.tmpdir()

export function open (template, flags, mode, cb) {
  switch (flags) {
    case 'w': flags = 'wx'; break
    case 'w+': flags = 'wx+'; break
    default: throw new Error('Unknown file open flag: ' + flags)
  }

  if (typeof mode === 'function') {
    cb = mode
    mode = undefined
  }

  let path

  retryAsync(function (cb) {
    path = randomPath(tmpdir, template)
    fs.open(path, flags, mode, cb)
  }, function (err, fd) {
    cb(err, err ? undefined : { fd, path })
  })
}

export function openSync (template, flags, mode) {
  switch (flags) {
    case 'w': flags = 'wx'; break
    case 'w+': flags = 'wx+'; break
    default: throw new Error('Unknown file open flag: ' + flags)
  }

  let path

  const fd = retrySync(function () {
    path = randomPath(tmpdir, template)
    return fs.openSync(path, flags, mode)
  })

  return { fd, path }
}

export function mkdir (template, mode, cb) {
  if (typeof mode === 'function') {
    cb = mode
    mode = undefined
  }

  let path

  retryAsync(function (cb) {
    path = randomPath(tmpdir, template)
    fs.mkdir(path, mode, cb)
  }, function (err) {
    cb(err, err ? undefined : path)
  })
}

export function mkdirSync (template, mode) {
  let path

  retrySync(function () {
    path = randomPath(tmpdir, template)
    fs.mkdirSync(path, mode)
  })

  return path
}

export function writeFile (template, data, options, cb) {
  cb = arguments[arguments.length - 1]

  if (typeof options === 'function' || !options) {
    options = { flag: 'wx' }
  } else if (typeof options === 'string') {
    options = { encoding: options, flag: 'wx' }
  } else if (typeof options === 'object') {
    options.flag = 'wx'
  } else {
    throw new TypeError('Bad arguments')
  }

  let path

  retryAsync(function (cb) {
    path = randomPath(tmpdir, template)
    fs.writeFile(path, data, options, cb)
  }, function (err) {
    cb(err, err ? undefined : path)
  })
}

export function writeFileSync (template, data, options) {
  if (!options) {
    options = { flag: 'wx' }
  } else if (typeof options === 'string') {
    options = { encoding: options, flag: 'wx' }
  } else if (typeof options === 'object') {
    options.flag = 'wx'
  } else {
    throw new TypeError('Bad arguments')
  }

  let path

  retrySync(function () {
    path = randomPath(tmpdir, template)
    fs.writeFileSync(path, data, options)
  })

  return path
}

export function createWriteStream (template, options) {
  return new WriteStream(template, options)
}

redpajama_set_name: RedPajamaGithub


text:
import React, { useState } from 'react';

import { Box, CheckBoxGroup } from 'grommet';

const objectOptions = [];
for (let i = 1; i <= 5; i += 1) {
  objectOptions.push({
    lab: `option ${i}`,
    val: i,
  });
}

export const ObjectOptions = () => {
  const [value, setValue] = useState([]);
  const [value2, setValue2] = useState([]);

  return (
    <Box pad="medium" gap="large">
      <CheckBoxGroup
        labelKey="lab"
        valueKey="val"
        value={value}
        onChange={(event) => {
          setValue(event.value);
          console.log('Group1: ', event.value);
        }}
        options={objectOptions}
      />
      <CheckBoxGroup
        gap="xsmall"
        labelKey="label"
        valueKey="key"
        value={value2}
        onChange={(event) => {
          setValue2(event.value);
          console.log('Group2: ', event.value);
        }}
        options={[
          { label: 'Maui', key: 'M' },
          { label: 'Jerusalem', key: 'J' },
          { label: 'Wuhan', key: 'W' },
        ]}
      />
    </Box>
  );
};

ObjectOptions.storyName = 'Object options';

export default {
  title: 'Input/CheckBoxGroup/Object options',
};


text:
#ifndef BLADERF_BACKEND_CONFIG_H__
#define BLADERF_BACKEND_CONFIG_H__

#define ENABLE_BACKEND_USB
#define ENABLE_BACKEND_LIBUSB
//#cmakedefine ENABLE_BACKEND_CYAPI
//#cmakedefine ENABLE_BACKEND_DUMMY
//#cmakedefine ENABLE_BACKEND_LINUX_DRIVER

#include "backend.h"
#include "./usb/usb.h"

#ifdef ENABLE_BACKEND_DUMMY
    extern const struct backend_fns backend_fns_dummy;
#   define BACKEND_DUMMY &backend_fns_dummy,
#else
#   define BACKEND_DUMMY
#endif

#ifdef ENABLE_BACKEND_USB
    extern const struct backend_fns backend_fns_usb;
#   define BACKEND_USB  &backend_fns_usb,

#   ifdef ENABLE_BACKEND_LIBUSB
        extern const struct usb_driver usb_driver_libusb;
#       define BACKEND_USB_LIBUSB &usb_driver_libusb,
#   else
#       define BACKEND_USB_LIBUSB
#   endif

#   ifdef ENABLE_BACKEND_CYAPI
        extern const struct usb_driver usb_driver_cypress;
#       define BACKEND_USB_CYAPI &usb_driver_cypress,
#   else
#       define BACKEND_USB_CYAPI
#   endif

#   define BLADERF_USB_BACKEND_LIST { \
            BACKEND_USB_LIBUSB \
            BACKEND_USB_CYAPI \
    }

#   if !defined(ENABLE_BACKEND_LIBUSB) && !defined(ENABLE_BACKEND_CYAPI)
#       error "No USB backends are enabled. One or more must be enabled."
#   endif
#else
#   define BACKEND_USB
#endif

/* This list should be ordered by preference (highest first) */
#define BLADERF_BACKEND_LIST { \
    BACKEND_USB \
    BACKEND_DUMMY \
}

#endif


text:
const { Datastore } = require('@google-cloud/datastore');
const ds = new Datastore({ projectId: 'monkeycam-web-app' });
const { PubSub } = require('@google-cloud/pubsub');

const STATE_UNKNOWN = 0;
const STATE_CREATED = 1;
const STATE_QUEUED = 2;
const STATE_FAILED_RETRY = 3;
const STATE_RUNNING = 4;
const STATE_SAVING = 5;
const STATE_COMPLETED = 6;
const STATE_FAILED = 7;

const STATE_NAMES = [
  'Unknown',
  'Created',
  'Queued',
  'Failed, will retry',
  'Running',
  'Saving',
  'Completed',
  'Failed'
];

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

class MonkeyCAMJobInputs {
  constructor(boardConfig, bindingConfig, machineConfig, bindingDist) {
    this.boardConfig = boardConfig;
    this.bindingConfig = bindingConfig;
    this.machineConfig = machineConfig;
    this.bindingDist = bindingDist;
    this.key = null;
  }

  static _jobInputsFromDSEntity(key, entity) {
    let ji = new MonkeyCAMJobInputs();
    ji.key = key;
    Object.assign(ji, entity);
    return ji;
  }

  _updateFromDSEntity(entity) {
    Object.assign(this, entity);
  }

  _toDSEntity() {
    if (this.key == null) this.key = ds.key('JobInputs');
    const excludeFromIndexes = [
      'boardConfig',
      'bindingConfig',
      'machineConfig',
      'bindingDist'
    ];
    const excludedFields = ['key'];
    let e = { key: this.key, data: [] };
    Object.keys(this).forEach(key => {
      if (!excludedFields.includes(key)) {
        let v = { name: key, value: this[key] };
        if (excludeFromIndexes.includes(key)) {
          v['excludeFromIndexes'] = true;
        }
        e.data.push(v);
      }
    });
    return e;
  }

  async save() {
    const entity = this._toDSEntity();
    await ds.save(entity);
    console.log(`Saved ${this.key.path} to data store.`);
    return this.key.id;
  }

  static async get(id) {
    const key = ds.key(['JobInputs', ds.int(id)]);
    const data = await ds.get(key);
    const entity = data[0];
    if (!entity) {
      let err = new Error('Invalid job inputs id');
      err.code = 404;
      throw err;
    }
    return this._jobInputsFromDSEntity(key, entity);
  }
}

class MonkeyCAMJobProgress {
  static async create() {
    const key = ds.key('JobProgress');
    return await MonkeyCAMJobProgress._save(key, 0);
  }

  static async save(id, progress) {
    const key = ds.key(['JobProgress', ds.int(id)]);
    return await MonkeyCAMJobProgress._save(key, progress);
  }

  static async _save(key, progress) {
    const entity = {
      key: key,
      data: [{ name: 'p', value: progress, excludeFromIndexes: true }]
    };
    await ds.save(entity);
    console.log(`Saved ${key.path} to data store: ${progress}`);
    return key.id;
  }

  static async get(id) {
    const key = ds.key(['JobProgress', ds.int(id)]);
    const data = await ds.get(key);
    const entity = data[0];
    if (!entity) {
      let err = new Error('Invalid job progress id');
      err.code = 404;
      throw err;
    }
    return entity.p;
  }
}

class MonkeyCAMJob {
  constructor(boardConfig, bindingConfig, machineConfig, bindingDist) {
    this.inputs = new MonkeyCAMJobInputs(
      boardConfig,
      bindingConfig,
      machineConfig,
      bindingDist
    );
    this.key = null;
    this.inputsId = null;
    this.state = STATE_CREATED;
    if (boardConfig) this.boardName = boardConfig.board.name;
  }

  static _jobFromDSEntity(key, entity) {
    let job = new MonkeyCAMJob();
    job.key = key;
    Object.assign(job, entity);
    return job;
  }

  _updateFromDSEntity(entity) {
    Object.assign(this, entity);
  }

  _toDSEntity() {
    if (this.key == null) this.key = ds.key('Job');
    const excludeFromIndexes = [
      'inputsId',
      'progressId',
      'overviewPublicURL',
      'zipPublicURL',
      'startedAt',
      'finishedAt',
      'monkeyCAMResults',
      'attemptCount',
      'backoffMS'
    ];
    const excludedFields = ['key', 'inputs'];
    let e = { key: this.key, data: [] };
    Object.keys(this).forEach(key => {
      if (!excludedFields.includes(key)) {
        let v = { name: key, value: this[key] };
        if (excludeFromIndexes.includes(key)) {
          v['excludeFromIndexes'] = true;
        }
        e.data.push(v);
      }
    });
    return e;
  }

  static async updateJobState(id, newState, props = {}, opts = {}) {
    const key = ds.key(['Job', ds.int(id)]);
    const maxAttempts = opts.maxAttempts || 3;
    let attemptCount = 0;
    let backoffMS = opts.backoffMS || 100;
    while (attemptCount++ < maxAttempts) {
      if (attemptCount > 1) {
        console.log(`Backoff ${backoffMS}ms in updateJobState to ${newState}`);
        await sleep(backoffMS);
        backoffMS *= 2;
      }
      const transaction = ds.transaction();
      try {
        await transaction.run();
        const data = await transaction.get(key);
        const entity = data[0];
        if (!entity) {
          console.log(
            'Should be impossible: Failed to find job in updateJobState',
            key
          );
          await transaction.rollback();
          continue;
        }
        let job = MonkeyCAMJob._jobFromDSEntity(key, entity);
        console.log('updateJobState', job.state, newState, props);
        if (newState > job.state || newState === STATE_FAILED_RETRY) {
          job.state = newState;
          Object.assign(job, props);
          transaction.save(job._toDSEntity());
          if (opts.sleep) {
            await sleep(opts.sleep); // For forcing races in testing.
          }
          await transaction.commit();
          return job;
        } else {
          console.log('Tried to update to lower job state');
          await transaction.rollback();
          return job;
        }
      } catch (err) {
        console.log(
          'Error running updateJobState transaction, rolling back',
          id,
          newState,
          props,
          err
        );
        try {
          await transaction.rollback();
        } catch (err) {
          // Rollback may fail because commit already rolled back on contention, or due to
          // transient server failure. We will retry in all cases.
          console.log('Rollback failed in updateJobState', err);
        }
        continue;
      }
    }
    console.log('Reached maximum attempts in updateJobState, giving up.');
    throw new Error('Unable to update persistent job state');
  }

  async enqueue(jobQueueTopic) {
    this.createdAt = Date.now();
    this.inputsId = await this.inputs.save();
    this.progressId = await MonkeyCAMJobProgress.create();
    const jobEntity = this._toDSEntity();
    await ds.save(jobEntity);
    console.log(`Saved ${jobEntity.key.path} to data store.`, this);

    try {
      const pubsub = new PubSub({ projectId: 'monkeycam-web-app' });
      const job_queue_topic = pubsub.topic(jobQueueTopic);
      const dataBuffer = Buffer.from(
        JSON.stringify({ id: this.key.id, inputsId: this.inputsId })
      );
      const messageId = await job_queue_topic.publish(dataBuffer);
      console.log(
        'Published message id ' + messageId + ' to topic ' + jobQueueTopic
      );
      try {
        await MonkeyCAMJob.updateJobState(this.key.id, STATE_QUEUED);
      } catch (err) {
        console.log('Warning: Failed updating state after enqueue.', err);
      }
      return this.key.id;
    } catch (err) {
      await MonkeyCAMJob.updateJobState(this.key.id, STATE_FAILED, {
        failureReason: 'Failed to enqueue worker message',
        finishedAt: Date.now()
      });
      throw err;
    }
  }

  static async get(id) {
    const key = ds.key(['Job', ds.int(id)]);
    const data = await ds.get(key);
    const entity = data[0];
    if (!entity) {
      let err = new Error('Invalid job id');
      err.code = 404;
      throw err;
    }
    let job = this._jobFromDSEntity(key, entity);
    job.stateName = STATE_NAMES[job.state];
    job.zombie = false;
    if (job.state < STATE_COMPLETED) {
      job.progress = await MonkeyCAMJobProgress.get(job.progressId);
    } else {
      job.progress = 100;
    }
    return job;
  }
}

module.exports = {
  MonkeyCAMJob,
  MonkeyCAMJobInputs,
  MonkeyCAMJobProgress,
  STATE_FAILED_RETRY,
  STATE_RUNNING,
  STATE_SAVING,
  STATE_COMPLETED,
  STATE_FAILED
};


text:
"use strict";
/*
 * ooiui/static/js/views/aa/ClearAllModalFormView.js
 * View for event clear (resolve) modal
 */

var ClearAllModalFormView = ModalFormView.extend({
  bindings: {

  },
  events: {
    "click #addDef": "onSubmit",
    "change input[name='alertalarm']": "updatedType"
  },
  initialize: function() {
    _.bindAll(this, "render","onSubmit");
    // Intentionally left blank to override parent
  },
  initialRender:function(){
    this.$el.html('<i class="fa fa-spinner fa-spin" style="margin-top:50px;margin-left:50%;font-size:90px;color:#337ab7;"> </i>');
  },
  template: JST['ooiui/static/js/partials/ClearAllModalForm.html'],
  onSubmit: function(e) {
    var self = this;

    //stop the page redirect
    e.preventDefault();

    var clearModel = new ClearAllModel();
    clearModel.set('id', this.model.get('id'));
    clearModel.set('resolved_comment', this.$el.find('#inputDescription').val()); //str

    //console.log(clearModel);
    // Sends ack
    clearModel.save(null,{success: function(model, response, opts) {
      //console.log('Cleared OK!');
      ooi.trigger('alertClearAll:success');
    },
      error: function(model, response, opts) {
        console.log('Clear All Error!');
      }
    });
      this.hide();
  },
  render: function() {
    var self = this;

    this.$el.html(this.template({
      model: this.model
    }));
  }
});


text:
layout: feature
title: 'Tense'
shortdef: 'tense'
udver: '2'
---

  Values:
  Imp
  Past
  Pres

Tense is a feature that specifies the time when the action took /
takes / will take place, in relation to a reference point.
The reference is often the moment of producing the sentence,
but it can be also another event in the context.

Armenian has a complex tense/aspect/mood system.

Note, that we are defining features that apply to a single word. If a tense is constructed periphrastically (two words, e.g. auxiliary verb indicative + participle of the main verb), and none of the participating words are specific to this tense, then the features will probably not directly reveal the tense.

Actions that happened before, during, and after a past or present event are constructed in Armenian by using perfect, resultative, imperfective and future participles.

Note, that actions that happen before or during a future event are expressed using an auxiliary. Imperfect or present finite form of auxilariy in periphrastic future (accompanied by future participle of main verb) is tagged `Tense=Imp` or `Tense=Pres` accordingly, not `Tense=Fut`, while there will be an [aspect](Aspect) (`Aspect=Prosp`) information at the participle (e.g. _վազելու էի տուն _ "I should run home" or _վազելու եմ տուն _ "I shall run home").

Note also, that there is an elaborate system of parallel tense/aspect/mood forms (traditionally called "secondary compound tenses") formed periphrastically by forms of the auxiliary _լինեմ _ and the resultative, processual and future-I participles of the main verb. They express actions that happened before, during, and after past or present reference. For these periphrastical forms we use `Tense=Imp` or `Tense=Pres` with perfect (`Perf`), progressive (`Prog`) or prospective (`Prosp`) aspect.

### `Past`: past tense / aorist

The past tense denotes actions that happened before a reference point. The reference point is the moment of producing
the sentence and the (completed) past event happened before the speaker speaks about
it. In Armenian this is aorist.

#### Examples

* _նա գնաց տուն _ "he went.`Ind` home",
* _եղա _ "I was.`Ind`"

### `Imp`: imperfect

Imperfect is a special case of the past tense. It denotes actions that are happening during some past moment. These actions might continue after the moment of speaking, but also might not, i.e. the evidence is not in the form itself, but it is in the context.

Note, that a limited set of verbs (_գիտեմ, ունեմ, արժեմ_) form imperfect and present morphologically with unmarked vs. marked stems (traditionally called "Defective Verbs with Secondary Expanded Stems"). The difference between them are related also to [Mood]() and `Aspect` rather than `Tense`. The verbs with unmarked ("non-expanded") stems will have `Tense=Imp` and `Tense=Pres` in indicative mood.

#### Examples

* _էի _ "I was.`Ind`",
* _կայի _ "I was / existed.`Ind`",
* _գնայի տուն _ "I wish I went.`Sub` home",
* _վազեի տուն _ "I wish I ran.`Sub` home",
* _պիտի գնայի տուն _ "I should go.`Nec` home",
* _ունեի _ "I had".`Ind`,
* _գիտեի _ "I knew".`Ind`,
* _արժեի _ "I was worth.`Ind`",
* _ունենայի _ "I wish I had.`Sub`",
* _գիտենայի _ "I wish I knew.`Sub`",
* _արժենայի _ "I wish I were worth.`Sub`"
* _լինեի _ "I wish I were.`Sub`",

### `Pres`: present tense

The present tense denotes actions that are in progress (or states that
are valid) in a reference point; it may also describe events that usually
happen. In the prototypical case, the reference point is the moment of producing
the sentence.

Present forms of verbs in subjunctive, conditional and necessitative moods have future meaning. These forms are tagged `Tense=Pres` not `Tense=Fut` as in traditional grammars.

#### Examples

* _եմ _ "I am.`Ind`",
* _կամ _ "I am / exist.`Ind`",
* _գնամ տուն _ "I'd better go.`Sub` home",
* _կգնամ տուն _ "I shall go.`Cnd` home",
* _պիտի գնամ տուն _ "I must go.`Nec` home",
* _ունեմ _ "I have.`Ind`",
* _գիտեմ _ "I know.`Ind`",
* _արժեմ _ "I am worth.`Ind`",
* _ունենամ _ "(if) I have.`Sub`",
* _գիտենամ _ "(if) I know.`Sub`",
* _արժենամ _ "(if) I am worth.`Sub`"
* _լինեմ _ "I am / I will be.`Sub`",


text:
set(JSON_TEST_DATA_URL     https://github.com/nlohmann/json_test_data)
set(JSON_TEST_DATA_VERSION 3.0.0)

# if variable is set, use test data from given directory rather than downloading them
if(JSON_TestDataDirectory)
    message(STATUS "Using test data in ${JSON_TestDataDirectory}.")
    add_custom_target(download_test_data)
    file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${JSON_TestDataDirectory}\"\n")
else()
    find_package(Git)
    # target to download test data
    add_custom_target(download_test_data
        COMMAND test -d json_test_data || ${GIT_EXECUTABLE} clone -c advice.detachedHead=false --branch v${JSON_TEST_DATA_VERSION} ${JSON_TEST_DATA_URL}.git --quiet --depth 1
        COMMENT "Downloading test data from ${JSON_TEST_DATA_URL} (v${JSON_TEST_DATA_VERSION})"
        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
    )
    # create a header with the path to the downloaded test data
    file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${CMAKE_BINARY_DIR}/json_test_data\"\n")
endif()

# determine the operating system (for debug and support purposes)
find_program(UNAME_COMMAND uname)
find_program(VER_COMMAND ver)
find_program(LSB_RELEASE_COMMAND lsb_release)
find_program(SW_VERS_COMMAND sw_vers)
set(OS_VERSION_STRINGS "${CMAKE_SYSTEM}")
if (VER_COMMAND)
    execute_process(COMMAND ${VER_COMMAND} OUTPUT_VARIABLE VER_COMMAND_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE)
    set(OS_VERSION_STRINGS "${OS_VERSION_STRINGS}; ${VER_COMMAND_RESULT}")
endif()
if (SW_VERS_COMMAND)
    execute_process(COMMAND ${SW_VERS_COMMAND} OUTPUT_VARIABLE SW_VERS_COMMAND_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
    string(REGEX REPLACE "[ ]*\n" "; " SW_VERS_COMMAND_RESULT "${SW_VERS_COMMAND_RESULT}")
    set(OS_VERSION_STRINGS "${OS_VERSION_STRINGS}; ${SW_VERS_COMMAND_RESULT}")
endif()
if (LSB_RELEASE_COMMAND)
    execute_process(COMMAND ${LSB_RELEASE_COMMAND} -a OUTPUT_VARIABLE LSB_RELEASE_COMMAND_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
    string(REGEX REPLACE "[ ]*\n" "; " LSB_RELEASE_COMMAND_RESULT "${LSB_RELEASE_COMMAND_RESULT}")
    set(OS_VERSION_STRINGS "${OS_VERSION_STRINGS}; ${LSB_RELEASE_COMMAND_RESULT}")
endif()
if (UNAME_COMMAND)
    execute_process(COMMAND ${UNAME_COMMAND} -a OUTPUT_VARIABLE UNAME_COMMAND_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
    set(OS_VERSION_STRINGS "${OS_VERSION_STRINGS}; ${UNAME_COMMAND_RESULT}")
endif()

message(STATUS "Operating system: ${OS_VERSION_STRINGS}")

# determine the compiler (for debug and support purposes)
if (MSVC)
    execute_process(COMMAND ${CMAKE_CXX_COMPILER} OUTPUT_VARIABLE CXX_VERSION_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_VARIABLE CXX_VERSION_RESULT ERROR_STRIP_TRAILING_WHITESPACE)
    set(CMAKE_CXX_COMPILER "${CXX_VERSION_RESULT}; MSVC_VERSION=${MSVC_VERSION}; MSVC_TOOLSET_VERSION=${MSVC_TOOLSET_VERSION}")
else()
    execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE CXX_VERSION_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
string(REGEX REPLACE "[ ]*\n" "; " CXX_VERSION_RESULT "${CXX_VERSION_RESULT}")
message(STATUS "Compiler: ${CXX_VERSION_RESULT}")


text:
import scala.tools.partest._

// Test that static methods in Java interfaces (new in Java 8)
// are callable from jointly compiler Scala code.
object Test extends CompilerTest {
  import global._

  override lazy val units: List[CompilationUnit] = {
    // This test itself does not depend on JDK8.
    javaCompilationUnits(global)(staticMethodInInterface) ++
    compilationUnits(global)(scalaClient)
  }

  private def staticMethodInInterface = """
public interface Interface {
  public static int staticMethod() {
    return 42;
  }
}

  """

  private def scalaClient = """
object Test {
  val x: Int = Interface.staticMethod()
}

class C extends Interface // expect no errors about unimplemented members.

  """

  // We're only checking we can compile it.
  def check(source: String, unit: global.CompilationUnit): Unit = ()
}


text:
package context

import (
	"golang.org/x/net/context"
)

type key int

const (
	mdKey = key(0)
)

type Metadata map[string]string

func GetMetadata(ctx context.Context) (Metadata, bool) {
	md, ok := ctx.Value(mdKey).(Metadata)
	return md, ok
}

func WithMetadata(ctx context.Context, md Metadata) context.Context {
	return context.WithValue(ctx, mdKey, md)
}


text:
@class IDEDataModelPredicateTextField, NSPopUpButton;

@interface IDEPredicateEditorFallbackRowTemplate : NSPredicateEditorRowTemplate
{
    NSPopUpButton *popup;
    IDEDataModelPredicateTextField *textField;
}

+ (id)rowTemplate;
- (void).cxx_destruct;
- (void)setPredicate:(id)arg1;
- (id)templateViews;
- (id)textField;
- (id)popup;
- (id)predicateWithSubpredicates:(id)arg1;
- (id)displayableSubpredicatesOfPredicate:(id)arg1;
- (double)matchForPredicate:(id)arg1;

@end


text:
<link rel="stylesheet" href="{{ url_for('static', filename='css/metro.css') }}">

{{ session['apIp'] }}

IP Address


text:
package util

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/runtime"
	kdeplutil "k8s.io/kubernetes/pkg/util/deployment"

	deployapi "github.com/openshift/origin/pkg/deploy/api"
	"github.com/openshift/origin/pkg/util/namer"
)

// LatestDeploymentNameForConfig returns a stable identifier for config based on its version.
func LatestDeploymentNameForConfig(config *deployapi.DeploymentConfig) string {
	return fmt.Sprintf("%s-%d", config.Name, config.Status.LatestVersion)
}

// LatestDeploymentInfo returns info about the latest deployment for a config,
// or nil if there is no latest deployment. The latest deployment is not
// always the same as the active deployment.
func LatestDeploymentInfo(config *deployapi.DeploymentConfig, deployments []api.ReplicationController) (bool, *api.ReplicationController) {
	if config.Status.LatestVersion == 0 || len(deployments) == 0 {
		return false, nil
	}
	sort.Sort(ByLatestVersionDesc(deployments))
	candidate := &deployments[0]
	return DeploymentVersionFor(candidate) == config.Status.LatestVersion, candidate
}

// ActiveDeployment returns the latest complete deployment, or nil if there is
// no such deployment. The active deployment is not always the same as the
// latest deployment.
func ActiveDeployment(config *deployapi.DeploymentConfig, input []api.ReplicationController) *api.ReplicationController {
	// we need to create our own copy of the input slice so that our sort here
	// doesn't change the caller's slice
	var deployments []api.ReplicationController
	for _, deployment := range input {
		deployments = append(deployments, deployment)
	}

	sort.Sort(ByLatestVersionDesc(deployments))
	var activeDeployment *api.ReplicationController
	for _, deployment := range deployments {
		if DeploymentStatusFor(&deployment) == deployapi.DeploymentStatusComplete {
			activeDeployment = &deployment
			break
		}
	}
	return activeDeployment
}

// DeployerPodSuffix is the suffix added to pods created from a deployment
const DeployerPodSuffix = "deploy"

// DeployerPodNameForDeployment returns the name of a pod for a given deployment
func DeployerPodNameForDeployment(deployment string) string {
	return namer.GetPodName(deployment, DeployerPodSuffix)
}

// LabelForDeployment builds a string identifier for a Deployment.
func LabelForDeployment(deployment *api.ReplicationController) string {
	return fmt.Sprintf("%s/%s", deployment.Namespace, deployment.Name)
}

// LabelForDeploymentConfig builds a string identifier for a DeploymentConfig.
func LabelForDeploymentConfig(config *deployapi.DeploymentConfig) string {
	return fmt.Sprintf("%s/%s", config.Namespace, config.Name)
}

// DeploymentNameForConfigVersion returns the name of the version-th deployment
// for the config that has the provided name
func DeploymentNameForConfigVersion(name string, version int64) string {
	return fmt.Sprintf("%s-%d", name, version)
}

// ConfigSelector returns a label Selector which can be used to find all
// deployments for a DeploymentConfig.
//
// TODO: Using the annotation constant for now since the value is correct
// but we could consider adding a new constant to the public types.
func ConfigSelector(name string) labels.Selector {
	return labels.Set{deployapi.DeploymentConfigAnnotation: name}.AsSelector()
}

// DeployerPodSelector returns a label Selector which can be used to find all
// deployer pods associated with a deployment with name.
func DeployerPodSelector(name string) labels.Selector {
	return labels.Set{deployapi.DeployerPodForDeploymentLabel: name}.AsSelector()
}

// AnyDeployerPodSelector returns a label Selector which can be used to find
// all deployer pods across all deployments, including hook and custom
// deployer pods.
func AnyDeployerPodSelector() labels.Selector {
	sel, _ := labels.Parse(deployapi.DeployerPodForDeploymentLabel)
	return sel
}

// HasChangeTrigger returns whether the provided deployment configuration has
// a config change trigger or not
func HasChangeTrigger(config *deployapi.DeploymentConfig) bool {
	for _, trigger := range config.Spec.Triggers {
		if trigger.Type == deployapi.DeploymentTriggerOnConfigChange {
			return true
		}
	}
	return false
}

func DeploymentConfigDeepCopy(dc *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
	objCopy, err := api.Scheme.DeepCopy(dc)
	if err != nil {
		return nil, err
	}
	copied, ok := objCopy.(*deployapi.DeploymentConfig)
	if !ok {
		return nil, fmt.Errorf("expected DeploymentConfig, got %#v", objCopy)
	}
	return copied, nil
}

func DeploymentDeepCopy(rc *api.ReplicationController) (*api.ReplicationController, error) {
	objCopy, err := api.Scheme.DeepCopy(rc)
	if err != nil {
		return nil, err
	}
	copied, ok := objCopy.(*api.ReplicationController)
	if !ok {
		return nil, fmt.Errorf("expected ReplicationController, got %#v", objCopy)
	}
	return copied, nil
}

// DecodeDeploymentConfig decodes a DeploymentConfig from controller using codec. An error is returned
// if the controller doesn't contain an encoded config.
func DecodeDeploymentConfig(controller *api.ReplicationController, decoder runtime.Decoder) (*deployapi.DeploymentConfig, error) {
	encodedConfig := []byte(EncodedDeploymentConfigFor(controller))
	decoded, err := runtime.Decode(decoder, encodedConfig)
	if err == nil {
		if config, ok := decoded.(*deployapi.DeploymentConfig); ok {
			return config, nil
		}
		return nil, fmt.Errorf("decoded object from controller is not a DeploymentConfig")
	}
	return nil, fmt.Errorf("failed to decode DeploymentConfig from controller: %v", err)
}

// EncodeDeploymentConfig encodes config as a string using codec.
func EncodeDeploymentConfig(config *deployapi.DeploymentConfig, codec runtime.Codec) (string, error) {
	if bytes, err := runtime.Encode(codec, config); err == nil {
		return string(bytes[:]), nil
	} else {
		return "", err
	}
}

// MakeDeployment creates a deployment represented as a ReplicationController and based on the given
// DeploymentConfig. The controller replica count will be zero.
func MakeDeployment(config *deployapi.DeploymentConfig, codec runtime.Codec) (*api.ReplicationController, error) {
	var err error
	var encodedConfig string

	if encodedConfig, err = EncodeDeploymentConfig(config, codec); err != nil {
		return nil, err
	}

	deploymentName := LatestDeploymentNameForConfig(config)

	podSpec := api.PodSpec{}
	if err := api.Scheme.Convert(&config.Spec.Template.Spec, &podSpec, nil); err != nil {
		return nil, fmt.Errorf("couldn't clone podSpec: %v", err)
	}

	controllerLabels := make(labels.Set)
	for k, v := range config.Labels {
		controllerLabels[k] = v
	}
	// Correlate the deployment with the config.
	// TODO: Using the annotation constant for now since the value is correct
	// but we could consider adding a new constant to the public types.
	controllerLabels[deployapi.DeploymentConfigAnnotation] = config.Name

	// Ensure that pods created by this deployment controller can be safely associated back
	// to the controller, and that multiple deployment controllers for the same config don't
	// manipulate each others' pods.
	selector := map[string]string{}
	for k, v := range config.Spec.Selector {
		selector[k] = v
	}
	selector[deployapi.DeploymentConfigLabel] = config.Name
	selector[deployapi.DeploymentLabel] = deploymentName

	podLabels := make(labels.Set)
	for k, v := range config.Spec.Template.Labels {
		podLabels[k] = v
	}
	podLabels[deployapi.DeploymentConfigLabel] = config.Name
	podLabels[deployapi.DeploymentLabel] = deploymentName

	podAnnotations := make(labels.Set)
	for k, v := range config.Spec.Template.Annotations {
		podAnnotations[k] = v
	}
	podAnnotations[deployapi.DeploymentAnnotation] = deploymentName
	podAnnotations[deployapi.DeploymentConfigAnnotation] = config.Name
	podAnnotations[deployapi.DeploymentVersionAnnotation] = strconv.FormatInt(config.Status.LatestVersion, 10)

	deployment := &api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Name: deploymentName,
			Annotations: map[string]string{
				deployapi.DeploymentConfigAnnotation:        config.Name,
				deployapi.DeploymentStatusAnnotation:        string(deployapi.DeploymentStatusNew),
				deployapi.DeploymentEncodedConfigAnnotation: encodedConfig,
				deployapi.DeploymentVersionAnnotation:       strconv.FormatInt(config.Status.LatestVersion, 10),
				// This is the target replica count for the new deployment.
				deployapi.DesiredReplicasAnnotation:    strconv.Itoa(int(config.Spec.Replicas)),
				deployapi.DeploymentReplicasAnnotation: strconv.Itoa(0),
			},
			Labels: controllerLabels,
		},
		Spec: api.ReplicationControllerSpec{
			// The deployment should be inactive initially
			Replicas: 0,
			Selector: selector,
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels:      podLabels,
					Annotations: podAnnotations,
				},
				Spec: podSpec,
			},
		},
	}
	if config.Status.Details != nil && len(config.Status.Details.Message) > 0 {
		deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = config.Status.Details.Message
	}
	if value, ok := config.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok {
		deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation] = value
	}

	return deployment, nil
}

// GetReplicaCountForDeployments returns the sum of all replicas for the
// given deployments.
func GetReplicaCountForDeployments(deployments []api.ReplicationController) int32 {
	totalReplicaCount := int32(0)
	for _, deployment := range deployments {
		totalReplicaCount += deployment.Spec.Replicas
	}
	return totalReplicaCount
}

// GetStatusReplicaCountForDeployments returns the sum of the replicas reported in the
// status of the given deployments.
func GetStatusReplicaCountForDeployments(deployments []api.ReplicationController) int32 {
	totalReplicaCount := int32(0)
	for _, deployment := range deployments {
		totalReplicaCount += deployment.Status.Replicas
	}
	return totalReplicaCount
}

// GetAvailablePods returns all the available pods from the provided pod list.
func GetAvailablePods(pods []api.Pod, minReadySeconds int32) int32 {
	available := int32(0)
	for i := range pods {
		pod := pods[i]
		if kdeplutil.IsPodAvailable(&pod, minReadySeconds, time.Now()) {
			available++
		}
	}
	return available
}

func DeploymentConfigNameFor(obj runtime.Object) string {
	return annotationFor(obj, deployapi.DeploymentConfigAnnotation)
}

func DeploymentNameFor(obj runtime.Object) string {
	return annotationFor(obj, deployapi.DeploymentAnnotation)
}

func DeployerPodNameFor(obj runtime.Object) string {
	return annotationFor(obj, deployapi.DeploymentPodAnnotation)
}

func DeploymentStatusFor(obj runtime.Object) deployapi.DeploymentStatus {
	return deployapi.DeploymentStatus(annotationFor(obj, deployapi.DeploymentStatusAnnotation))
}

func DeploymentStatusReasonFor(obj runtime.Object) string {
	return annotationFor(obj, deployapi.DeploymentStatusReasonAnnotation)
}

func DeploymentDesiredReplicas(obj runtime.Object) (int32, bool) {
	return int32AnnotationFor(obj, deployapi.DesiredReplicasAnnotation)
}

func DeploymentReplicas(obj runtime.Object) (int32, bool) {
	return int32AnnotationFor(obj, deployapi.DeploymentReplicasAnnotation)
}

func EncodedDeploymentConfigFor(obj runtime.Object) string {
	return annotationFor(obj, deployapi.DeploymentEncodedConfigAnnotation)
}

func DeploymentVersionFor(obj runtime.Object) int64 {
	v, err := strconv.ParseInt(annotationFor(obj, deployapi.DeploymentVersionAnnotation), 10, 64)
	if err != nil {
		return -1
	}
	return v
}

func IsDeploymentCancelled(deployment *api.ReplicationController) bool {
	value := annotationFor(deployment, deployapi.DeploymentCancelledAnnotation)
	return strings.EqualFold(value, deployapi.DeploymentCancelledAnnotationValue)
}

func HasSynced(dc *deployapi.DeploymentConfig) bool {
	return dc.Status.ObservedGeneration >= dc.Generation
}

// IsOwnedByConfig checks whether the provided replication controller is part of a
// deployment configuration.
// TODO: Switch to use owner references once we got those working.
func IsOwnedByConfig(deployment *api.ReplicationController) bool {
	_, ok := deployment.Annotations[deployapi.DeploymentConfigAnnotation]
	return ok
}

// IsTerminatedDeployment returns true if the passed deployment has terminated (either
// complete or failed).
func IsTerminatedDeployment(deployment *api.ReplicationController) bool {
	current := DeploymentStatusFor(deployment)
	return current == deployapi.DeploymentStatusComplete || current == deployapi.DeploymentStatusFailed
}

// IsFailedDeployment returns true if the passed deployment failed.
func IsFailedDeployment(deployment *api.ReplicationController) bool {
	current := DeploymentStatusFor(deployment)
	return current == deployapi.DeploymentStatusFailed
}

// CanTransitionPhase returns whether it is allowed to go from the current to the next phase.
func CanTransitionPhase(current, next deployapi.DeploymentStatus) bool {
	switch current {
	case deployapi.DeploymentStatusNew:
		switch next {
		case deployapi.DeploymentStatusPending,
			deployapi.DeploymentStatusRunning,
			deployapi.DeploymentStatusFailed,
			deployapi.DeploymentStatusComplete:
			return true
		}
	case deployapi.DeploymentStatusPending:
		switch next {
		case deployapi.DeploymentStatusRunning,
			deployapi.DeploymentStatusFailed,
			deployapi.DeploymentStatusComplete:
			return true
		}
	case deployapi.DeploymentStatusRunning:
		switch next {
		case deployapi.DeploymentStatusFailed, deployapi.DeploymentStatusComplete:
			return true
		}
	}
	return false
}

// annotationFor returns the annotation with key for obj.
func annotationFor(obj runtime.Object, key string) string {
	meta, err := api.ObjectMetaFor(obj)
	if err != nil {
		return ""
	}
	return meta.Annotations[key]
}

func int32AnnotationFor(obj runtime.Object, key string) (int32, bool) {
	s := annotationFor(obj, key)
	if len(s) == 0 {
		return 0, false
	}
	i, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		return 0, false
	}
	return int32(i), true
}

// DeploymentsForCleanup determines which deployments for a configuration are relevant for the
// revision history limit quota
func DeploymentsForCleanup(configuration *deployapi.DeploymentConfig, deployments []api.ReplicationController) []api.ReplicationController {
	// if the past deployment quota has been exceeded, we need to prune the oldest deployments
	// until we are not exceeding the quota any longer, so we sort oldest first
	sort.Sort(ByLatestVersionAsc(deployments))

	relevantDeployments := []api.ReplicationController{}
	activeDeployment := ActiveDeployment(configuration, deployments)
	if activeDeployment == nil {
		// if cleanup policy is set but no successful deployments have happened, there will be
		// no active deployment. We can consider all of the deployments in this case except for
		// the latest one
		for _, deployment := range deployments {
			if DeploymentVersionFor(&deployment) != configuration.Status.LatestVersion {
				relevantDeployments = append(relevantDeployments, deployment)
			}
		}
	} else {
		// if there is an active deployment, we need to filter out any deployments that we don't
		// care about, namely the active deployment and any newer deployments
		for _, deployment := range deployments {
			if &deployment != activeDeployment || DeploymentVersionFor(&deployment) < DeploymentVersionFor(activeDeployment) {
				relevantDeployments = append(relevantDeployments, deployment)
			}
		}
	}

	return relevantDeployments
}

// ByLatestVersionAsc sorts deployments by LatestVersion ascending.
type ByLatestVersionAsc []api.ReplicationController

func (d ByLatestVersionAsc) Len() int      { return len(d) }
func (d ByLatestVersionAsc) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByLatestVersionAsc) Less(i, j int) bool {
	return DeploymentVersionFor(&d[i]) < DeploymentVersionFor(&d[j])
}

// ByLatestVersionDesc sorts deployments by LatestVersion descending.
type ByLatestVersionDesc []api.ReplicationController

func (d ByLatestVersionDesc) Len() int      { return len(d) }
func (d ByLatestVersionDesc) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d ByLatestVersionDesc) Less(i, j int) bool {
	return DeploymentVersionFor(&d[j]) < DeploymentVersionFor(&d[i])
}

// ByMostRecent sorts deployments by most recently created.
type ByMostRecent []*api.ReplicationController

func (s ByMostRecent) Len() int      { return len(s) }
func (s ByMostRecent) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ByMostRecent) Less(i, j int) bool {
	return !s[i].CreationTimestamp.Before(s[j].CreationTimestamp)
}


text:
<?php
namespace Thrace\DataGridBundle\Tests\Doctrine\Handler;

use Thrace\DataGridBundle\Tests\Fixture\Entity\GridDependent;

use Thrace\DataGridBundle\Tests\Fixture\Entity\GridMaster;

use Thrace\DataGridBundle\Doctrine\ORM\DataGridHandler;

use Symfony\Component\DependencyInjection\Container;

use Doctrine\ORM\QueryBuilder;

use Thrace\ComponentBundle\Test\Tool\BaseTestCaseORM;

class DataGridHandlerORMTest extends BaseTestCaseORM
{
    const GRID_MASTER = 'Thrace\DataGridBundle\Tests\Fixture\Entity\GridMaster';

    const GRID_DEPENDANT = 'Thrace\DataGridBundle\Tests\Fixture\Entity\GridDependent';

    protected function setUp()
    {
        $this->createMockEntityManager();
    }

    public function testBuildQueryWithInvalidQueryBuilder()
    {
        $options = array(
            'page' => 1,
            'records' => 10,
        );

        $this->setExpectedException('InvalidArgumentException', 'Value must be instance of Doctrine\ORM\QueryBuilder.');

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(0))
            ->setDataGrid($this->getMockDataGridForInvalidTest())
            ->resolveOptions($options)
            ->buildQuery()
        ;

    }

    public function testBuildQueryWithDependentGrid()
    {
        $options = array(
            'page' => 1,
            'records' => 10,
            'masterGridRowId' => 20
        );

        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(2))
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilderForDependentGrid(), true))
            ->resolveOptions($options)
            ->buildQuery()
        ;

        $this->assertSame(3, $dataGridHandler->getCount());
        $this->assertInstanceOf('Doctrine\ORM\Query', $dataGridHandler->getQuery());

    }

    public function testBuildQueryWithSortableGrid()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(3))
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder(), false, true))
            ->resolveOptions($this->getOptions('ne', 'test'))
            ->buildQuery()
            ->buildData()
        ;

        $this->assertSame(20, $dataGridHandler->getCount());

    }

    public function testBuildDataWithOperEq()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('eq', 'name_1'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(1, $dataGridHandler->getCount());
        $this->assertCount(1, $data);
        $this->assertSame('name_1', $data['0']['name']);
        $this->assertNotEmpty($dataGridHandler->getProcessedData());

    }

    public function testBuildDataWithOperNe()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('ne', 'name_1'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(19, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_20', $data['0']['name']);

    }

    public function testBuildDataWithOperLt()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('lt', 15, 'm.rank'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(14, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_14', $data['0']['name']);

    }

    public function testBuildDataWithOperLe()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('le', 15, 'm.rank'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(15, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_15', $data['0']['name']);

    }

    public function testBuildDataWithOperGt()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('gt', 5, 'm.rank'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(15, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_20', $data['0']['name']);

    }

    public function testBuildDataWithOperGe()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('ge', 5, 'm.rank'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(16, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_20', $data['0']['name']);

    }

    public function testBuildDataWithOperBw()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('bw', 'name_1', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(11, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_19', $data['0']['name']);

    }

    public function testBuildDataWithOperBn()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('bn', 'name_1', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(9, $dataGridHandler->getCount());
        $this->assertCount(9, $data);
        $this->assertSame('name_20', $data['0']['name']);

    }

    public function testBuildDataWithOperEw()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('ew', '5', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(2, $dataGridHandler->getCount());
        $this->assertCount(2, $data);
        $this->assertSame('name_15', $data['0']['name']);

    }

    public function testBuildDataWithOperEn()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('en', '5', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(18, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_20', $data['0']['name']);

    }

    public function testBuildDataWithOperCn()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('cn', '5', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(2, $dataGridHandler->getCount());
        $this->assertCount(2, $data);
        $this->assertSame('name_15', $data['0']['name']);
    }

    public function testBuildDataWithOperNc()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('nc', '5', 'm.name', 'OR'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(18, $dataGridHandler->getCount());
        $this->assertCount(10, $data);
        $this->assertSame('name_20', $data['0']['name']);
    }

    public function testBuildDataWithInvalidOperator()
    {
        $this->populate();
        $this->setExpectedException('InvalidArgumentException');

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(0))
            ->setDataGrid($this->getMockDataGridForInvalidTest($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('invalid', '5', 'm.name'))
            ->buildQuery()
            ->buildData()
        ;
    }

    public function testBuildDataWithInvalidGroupOperator()
    {
        $this->setExpectedException('InvalidArgumentException');

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(0))
            ->setDataGrid($this->getMockDataGridForInvalidTest($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('eq', '5', 'm.name', 'INVALID'))
            ->buildQuery()
            ->buildData()
        ;
    }

    public function testBuildDataWithInvalidGroupOp()
    {

        $options = array(
            'search' => true,
            'filters' => array('groupOp' => 'INVALID'),
            'page' => 1,
            'orderBy' => 'm.rank',
            'sort' => 'DESC',
            'records' => 10

        );

        $this->setExpectedException('InvalidArgumentException', 'Operator does not match OR | AND');

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(0))
            ->setDataGrid($this->getMockDataGridForInvalidTest($this->getQueryBuilder()))
            ->resolveOptions($options)
            ->buildQuery()
        ;
    }

    public function testBuildDataWithInvalidRules()
    {

        $options = array(
            'search' => true,
            'filters' => array('groupOp' => 'AND', 'rules' => 'INVALID'),
            'page' => 1,
        );

        $this->setExpectedException('InvalidArgumentException', 'Rules are not set.');

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher(0))
            ->setDataGrid($this->getMockDataGridForInvalidTest($this->getQueryBuilder()))
            ->resolveOptions($options)
            ->buildQuery()
        ;
    }

    public function testAggregatedFieldWithEqAnd()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('eq', 3, 'slaveCount', 'AND'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(1, $dataGridHandler->getCount());
        $this->assertCount(1, $data);
        $this->assertSame('name_20', $data['0']['name']);
        $this->assertSame('3', $data['0']['slaveCount']);
    }

    public function testAggregatedFieldWithEqOr()
    {
        $this->populate();

        $dataGridHandler = new DataGridHandler();
        $dataGridHandler
            ->setDispatcher($this->getMockEventDispatcher())
            ->setDataGrid($this->getMockDataGrid($this->getQueryBuilder()))
            ->resolveOptions($this->getOptions('eq', 3, 'slaveCount', 'OR'))
            ->buildQuery()
            ->buildData()
        ;

        $data = $dataGridHandler->getData();
        $this->assertSame(1, $dataGridHandler->getCount());
        $this->assertCount(1, $data);
        $this->assertSame('name_20', $data['0']['name']);
        $this->assertSame('3', $data['0']['slaveCount']);
    }

    protected function getUsedEntityFixtures()
    {
        return array(self::GRID_MASTER, self::GRID_DEPENDANT);
    }

    protected function getQueryBuilder()
    {
        $qb =  $this->em->createQueryBuilder();
        $qb
            ->select(array('m.id, m.name, m.rank, COUNT(s.id) as slaveCount, m'))
            ->from('\Thrace\DataGridBundle\Tests\Fixture\Entity\GridMaster', 'm')
            ->leftJoin('m.grids', 's')
            ->groupBy('m.id')
        ;

        return $qb;
    }

    private function getQueryBuilderForDependentGrid()
    {
        $qb =  $this->em->createQueryBuilder();
        $qb
            ->select(array('s.id, s.name, s'))
            ->from('\Thrace\DataGridBundle\Tests\Fixture\Entity\GridDependent', 's')
            ->join('s.master', 'm')
            ->where('m.id = :masterGridRowId')
        ;

        return $qb;
    }

    private function populate()
    {
        for($i = 1; $i <= 20; $i++){
            $gridMaster = new GridMaster();
            $gridMaster->setName('name_' . $i);
            $gridMaster->setRank($i);
            $this->em->persist($gridMaster);
        }

        for($i = 1; $i <= 3; $i++){
            $gridDependant = new GridDependent();
            $gridDependant->setName('name_' . $i);
            $this->em->persist($gridDependant);
            $gridMaster->addGrid($gridDependant);
        }

        $this->em->flush();
        $this->em->clear();

    }

    protected function getMockEventDispatcher($expects = 3)
    {
        $mock = $this->getMock('Symfony\Component\EventDispatcher\EventDispatcherInterface');

        $mock
            ->expects($this->exactly($expects))
            ->method('dispatch')
            ->withAnyParameters()
        ;

        return $mock;
    }

    protected function getMockDataGrid(QueryBuilder $qb, $dependent = false, $sortable = false)
    {
        $mock = $this->getMock('Thrace\DataGridBundle\DataGrid\DataGridInterface');

        $mock
            ->expects($this->any())
            ->method('getName')
            ->will($this->returnValue('test'))
        ;

        $mock
            ->expects($this->once())
            ->method('isDependentGrid')
            ->will($this->returnValue($dependent))
        ;

        $mock
            ->expects($this->once())
            ->method('isSortableEnabled')
            ->will($this->returnValue($sortable))
        ;

        $mock
            ->expects($this->once())
            ->method('getQueryBuilder')
            ->will($this->returnValue($qb))
        ;

        $mock
            ->expects($this->any())
            ->method('getColModel')
            ->will($this->returnValue(array(
                array('index' => 'm.name', 'name' => 'name'),
                array('index' => 'slaveCount', 'name' => 'slaveCount', 'aggregated' => true),
            )))
        ;

        return $mock;
    }

    protected function getMockDataGridForInvalidTest(QueryBuilder $qb = null)
    {
        $mock = $this->getMock('Thrace\DataGridBundle\DataGrid\DataGridInterface');

        $mock
            ->expects($this->any())
            ->method('getQueryBuilder')
            ->will($this->returnValue($qb))
        ;

        return $mock;
    }

    protected function getOptions($op, $value, $field = 'm.name', $groupOp = 'AND')
    {
        $options = array(
            'search' => true,
            //'filters' => (array) json_decode('{"groupOp":"'.$groupOp.'","rules":[{"field":"'. $field .'","op":"'.$op.'","data":"'.$value.'"}]}'),
            'filters' => array('groupOp' => $groupOp, 'rules' => array(
                array(
                    'field' => $field,
                    'op' => $op,
                    'data' => $value
                ) ,
            )),
            'page' => 1,
            'orderBy' => 'm.rank',
            'sort' => 'DESC',
            'records' => 10
        );

        return $options;
    }

}