<|start_filename|>src/edu/mit/cci/adapters/csv/DiscussionThreadImpl.java<|end_filename|>
package edu.mit.cci.adapters.csv;

import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
* Created by josh on 1/20/14.
*/
public class DiscussionThreadImpl implements DiscussionThread {

    private String threadid;
    private List<Post> posts = new ArrayList<Post>();
    private boolean sorted = false;


    public DiscussionThreadImpl(String threadid) {
       this.threadid = threadid;
    }


    public String getThreadId() {
       return threadid;
    }

    public void addPost(Post p) {
        ((PostImpl)p).setThreadId(this.getThreadId());
        this.posts.add(p);
        sorted = false;
    }

    public List<Post> getPosts() {
        if (!sorted) {
            Collections.sort(posts, new Comparator<Post>() {
                public int compare(Post post, Post post1) {
                    return post.getTime().compareTo(post1.getTime());
                }
            });
        }
        return posts;
    }

    public List<String> getSentiment(Post p) {
         return Collections.emptyList();
    }
}


<|start_filename|>src/edu/mit/cci/teva/serialization/CommunityFrameMapJaxbAdapter.java<|end_filename|>
package edu.mit.cci.teva.serialization;

import edu.mit.cci.teva.engine.CommunityFrame;

import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * User: jintrone
 * Date: 10/8/12
 * Time: 10:19 PM
 */

@XmlRootElement(name = "frames")
public class CommunityFrameMapJaxbAdapter extends XmlAdapter<CommunityFrameMapJaxbAdapter,Map<Integer,CommunityFrame>> {

    @XmlElement(name="frame")
    List<CommunityFrame> frames;

    public CommunityFrameMapJaxbAdapter() {

    }

     public CommunityFrameMapJaxbAdapter(List<CommunityFrame> frames) {
         this.frames = new ArrayList<CommunityFrame>(frames);

    }

    @Override
    public Map<Integer, CommunityFrame> unmarshal(CommunityFrameMapJaxbAdapter communityFrameMapJaxbAdapter) throws Exception {
        Map<Integer,CommunityFrame> result = new HashMap<Integer, CommunityFrame>();
        for (CommunityFrame f:communityFrameMapJaxbAdapter.frames) {
          result.put(f.getWindow(),f);
        }
        return result;
    }

    @Override
    public CommunityFrameMapJaxbAdapter marshal(Map<Integer, CommunityFrame> integerCommunityFrameMap) throws Exception {
        List<CommunityFrame> f = new ArrayList<CommunityFrame>(integerCommunityFrameMap.values());
        Collections.sort(f,new Comparator<CommunityFrame>() {
            public int compare(CommunityFrame communityFrame, CommunityFrame communityFrame1) {
                return communityFrame.getWindow()-(communityFrame1.getWindow());
            }
        });
        CommunityFrameMapJaxbAdapter adapter =  new CommunityFrameMapJaxbAdapter(f);
        return adapter;
    }
}


<|start_filename|>src/edu/mit/cci/teva/util/ScoringMethod.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.engine.ConversationChunk;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 10:27 PM
 */
public interface ScoringMethod {

    public float score(Network one, Network two);
    public float fit(ConversationChunk chunk);

    public enum Method implements ScoringMethod {
        SIMILARITY() {
            public float score(Network one, Network two) {
                return NetworkUtils.similarity(one,two);
            }
            public float fit(ConversationChunk chunk) {return chunk.similarity;}

        },

        COVERAGE() {
            public float score(Network one, Network two) {
                return NetworkUtils.coverage(one,two);
            }
            public float fit(ConversationChunk chunk) {return chunk.coverage;}
        }


    }

}


<|start_filename|>src/edu/mit/cci/teva/model/ConversationImpl.java<|end_filename|>
package edu.mit.cci.teva.model;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
 * User: jintrone
 * Date: 9/29/11
 * Time: 8:44 AM
 */
public class ConversationImpl implements Conversation {

    List<DiscussionThread> threads = new ArrayList<DiscussionThread>();
    Post first = null;
    Post last = null;
    List<String> users = new ArrayList<String>();
    String name = null;

    public ConversationImpl(String name) {
        this.name = name;
    }

    public ConversationImpl(String name, List<? extends DiscussionThread> threads) {
        this.name = name;
        for (DiscussionThread t:threads) {
          addThread(t);
       }
    }


    public void addThread(DiscussionThread t) {
        int insertion = Collections.binarySearch(threads,t,new Comparator<DiscussionThread>() {
            public int compare(DiscussionThread discussionThread, DiscussionThread discussionThread1) {
                return discussionThread.getPosts().get(0).getTime().compareTo(discussionThread1.getPosts().get(0).getTime());
            }
        });
        if (insertion<0) {
            int idx = -insertion-1;
            if (idx == 0 ) {
                first = t.getPosts().get(0);
            }
            Post tlast =  t.getPosts().get(t.getPosts().size()-1);
            if (last == null || tlast.getTime().after(last.getTime())) {
                last = tlast;
            }
            threads.add(idx, t);

        }

        for (Post p:t.getPosts()) {
            if (!users.contains(p.getUserid())) users.add(p.getUserid());
        }


    }


    public String getName() {
        return name;
    }

    public Post getFirstPost() {
        return first;
    }

    public Post getLastPost() {
       return last;
    }

    public List<DiscussionThread> getAllThreads() {
        return threads;
    }

    public List<String> getUsers() {
        return users;
    }
}


<|start_filename|>src/edu/mit/cci/teva/serialization/ConnectionMapJaxb.java<|end_filename|>
package edu.mit.cci.teva.serialization;

import edu.mit.cci.teva.engine.CommunityModel;

import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.Map;

/**
 * User: jintrone
 * Date: 10/8/12
 * Time: 11:05 PM
 */
public class ConnectionMapJaxb extends XmlAdapter<ConnectionMapJaxb,Map<Integer,CommunityModel.Connection>> {



    @Override
    public Map<Integer, CommunityModel.Connection> unmarshal(ConnectionMapJaxb connectionMapJaxb) throws Exception {
        return null;  //To change body of implemented methods use File | Settings | File Templates.
    }

    @Override
    public ConnectionMapJaxb marshal(Map<Integer, CommunityModel.Connection> integerConnectionMap) throws Exception {
        return null;  //To change body of implemented methods use File | Settings | File Templates.
    }



}


<|start_filename|>src/edu/mit/cci/teva/util/MessageToModelMap.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.engine.ConversationChunk;
import edu.mit.cci.text.windowing.Windowable;

import java.util.*;

/**
 * User: jintrone
 * Date: 5/13/13
 * Time: 3:32 PM
 * <p/>
 * Provides a number of useful lookup functions to help map between a message and the CommunityModel
 */
public class MessageToModelMap {

    Map<String, MessageMeta> data = new HashMap<String, MessageMeta>();

    Map<String, Set<MessageMeta>> communityToMessageMap = new HashMap<String, Set<MessageMeta>>();

    Map<String, Community> communityMap = new HashMap<String, Community>();


    private final CommunityModel model;


    public MessageToModelMap(CommunityModel model) {


        this.model = model;
        for (Community c : model.getCommunities()) {
            communityMap.put(c.getId(), c);
            if (!communityToMessageMap.containsKey(c.getId())) {
                communityToMessageMap.put(c.getId(), new HashSet<MessageMeta>());
            }
            for (ConversationChunk chunk : c.getAssignments()) {
                for (Windowable w : chunk.messages) {

                    if (!data.containsKey(w.getId())) {
                        MessageMeta m = new MessageMeta(w.getId(), w.getStart());
                        data.put(w.getId(), m);
                        communityToMessageMap.get(c.getId()).add(m);

                    }
                    data.get(w.getId()).addChunk(c.getId(), chunk);
                }

            }
        }

    }

    public CommunityWindow getBestCommunityWindow(String messageId) {
        if (!data.containsKey(messageId)) return null;
        else {
            MessageMeta meta = data.get(messageId);
            String community = null;
            ConversationChunk chunk = null;
            for (Map.Entry<String, List<ConversationChunk>> ent : meta.communityAppearances.entrySet()) {
                if (community == null) {
                    community = ent.getKey();
                    chunk = ent.getValue().get(0);
                }
                else {
                    for (ConversationChunk c:ent.getValue()) {
                        if (c.coverage > chunk.coverage ||
                                (c.coverage == chunk.coverage && c.similarity > chunk.similarity)) {
                            chunk = c;
                            community = ent.getKey();
                        }
                    }


                }
            }
            return new CommunityWindow(community,chunk.window);
        }
    }



    public Map.Entry<String, List<ConversationChunk>> _getMostRepresentativeCommunity(String messageId) {
        if (!data.containsKey(messageId)) return null;
        else {
            MessageMeta meta = data.get(messageId);
            Map.Entry<String, List<ConversationChunk>> best = null;
            for (Map.Entry<String, List<ConversationChunk>> ent : meta.communityAppearances.entrySet()) {
                if (best == null || (ent.getValue().size() > best.getValue().size())) {
                    best = ent;
                }
            }
            return best;
        }
    }

    public String getMostRepresentativeCommunity(String msgid) {
        return _getMostRepresentativeCommunity(msgid).getKey();
    }

    /**public CommunityWindow getClosestCommunityWindow(String messageId) {
        if (!data.containsKey(messageId)) return null;
        else {
            MessageMeta meta = data.get(messageId);
            Map<String,ConversationChunk> chunks = new HashMap<String, ConversationChunk>();
            for (Map.Entry<String, List<ConversationChunk>> ent : meta.communityAppearances.entrySet()) {
                if (!chunks.containsKey(ent.getKey())) {
                    chunks.put(ent.getKey());
                }
            }
            return best;
        }
    }
     **/


    public CommunityWindow placeMessageInLastWindow(String messageId) {
        if (!data.containsKey(messageId)) return null;
        Map.Entry<String, List<ConversationChunk>> ent = _getMostRepresentativeCommunity(messageId);
        ConversationChunk best = null;
        for (ConversationChunk chunk : ent.getValue()) {
            if (best == null || chunk.window > best.window) {
                best = chunk;
            }
        }
        return new CommunityWindow(ent.getKey(), best.window);
    }

    public CommunityWindow placeMessageInFirstWindow(String messageId) {
        if (!data.containsKey(messageId)) return null;
        Map.Entry<String, List<ConversationChunk>> ent = _getMostRepresentativeCommunity(messageId);
        ConversationChunk best = null;
        for (ConversationChunk chunk : ent.getValue()) {
            if (best == null || chunk.window < best.window) {
                best = chunk;
            }
        }
        return new CommunityWindow(ent.getKey(), best.window);
    }

    public CommunityWindows getAllWindowsForBestCommunity(String messageId) {
        if (!data.containsKey(messageId)) return null;
        Map.Entry<String, List<ConversationChunk>> ent = _getMostRepresentativeCommunity(messageId);
        CommunityWindows result = new CommunityWindows(ent.getKey());
        for (ConversationChunk chunk : ent.getValue()) {
            result.add(chunk.window);
        }
        return result;
    }

    public List<? extends CommunityWindow> getRankedCommunityWindows(String messageId) {
        if (!data.containsKey(messageId)) return null;
        List<WeightedCommunityWindow> result = new ArrayList<WeightedCommunityWindow>();
        MessageMeta meta = data.get(messageId);
        for (Map.Entry<String,List<ConversationChunk>> ent:meta.communityAppearances.entrySet()) {
            for (ConversationChunk chunk:ent.getValue()){
                result.add(new WeightedCommunityWindow(ent.getKey(),chunk.window,chunk.similarity,chunk.coverage));
            }
        }
        Collections.sort(result, new Comparator<WeightedCommunityWindow>() {
            public int compare(WeightedCommunityWindow o1, WeightedCommunityWindow o2) {
                if (o1.coverage > o2.coverage) {
                    return -1;
                } else if (o1.coverage == o2.coverage) {
                    if (o1.similarity > o2.similarity) {
                        return -1;
                    }  else if (o1.similarity == o2.similarity) {
                        return 0;
                    }  else return 1;
                } else return 1;

            }
        });
        return result;
    }


    private Set<MessageMeta> _findMessagesMetaAt(String community, int window) {
        Set<MessageMeta> meta = communityToMessageMap.get(community);
        Set<MessageMeta> messages = new HashSet<MessageMeta>();
        for (MessageMeta m : meta) {
            if (m.windowAppearances.containsKey(window)) {
                messages.add(m);
            }

        }
        return messages;
    }

    private Set<String> findeMessagesAt(String community, int window) {
        Set<MessageMeta> meta = _findMessagesMetaAt(community, window);
        Set<String> result = new HashSet<String>();
        for (MessageMeta m : meta) {
            result.add(m.msgid);
        }
        return result;
    }

    public String getLastMessageAt(String community, int window) {
        Set<MessageMeta> meta = _findMessagesMetaAt(community, window);
        MessageMeta result = null;
        for (MessageMeta m : meta) {
            if (result == null || result.time.before(m.time)) {
                result = m;
            }
        }
        return result.msgid;
    }

    public String getFirstMessageAt(String community, int window) {
        Set<MessageMeta> meta = _findMessagesMetaAt(community, window);
        MessageMeta result = null;
        for (MessageMeta m : meta) {
            if (result == null || result.time.after(m.time)) {
                result = m;
            }
        }
        return result.msgid;
    }


    private static class MessageMeta {

        Map<String, List<ConversationChunk>> communityAppearances = new HashMap<String, List<ConversationChunk>>();
        Map<Integer, List<CommunityConversationChunk>> windowAppearances = new TreeMap<Integer, List<CommunityConversationChunk>>();
        String msgid = null;
        Date time;

        MessageMeta(String msgid, Date time) {
            this.msgid = msgid;
            this.time = time;
        }

        public void addChunk(String community, ConversationChunk chunk) {
            if (!communityAppearances.containsKey(community)) {
                communityAppearances.put(community, new ArrayList<ConversationChunk>());

            }
            communityAppearances.get(community).add(chunk);
            if (!windowAppearances.containsKey(chunk.window)) {
                windowAppearances.put(chunk.window, new ArrayList<CommunityConversationChunk>());

            }
            windowAppearances.get(chunk.window).add(new CommunityConversationChunk(chunk, community));

        }
    }

    private static class WeightedCommunityWindow extends CommunityWindow {

        float similarity;
        float coverage;

        public WeightedCommunityWindow(String community, int window, float similarity, float coverage) {
            super(community, window);
            this.similarity = similarity;
            this.coverage = coverage;
        }


    }

    private static class CommunityConversationChunk extends ConversationChunk {

        private final ConversationChunk chunk;

        public String community;

        public CommunityConversationChunk(ConversationChunk chunk, String community) {
            super();
            this.chunk = chunk;
            this.community = community;
        }


    }

}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityModel.java<|end_filename|>
package edu.mit.cci.teva.engine;




import edu.mit.cci.teva.serialization.MapJaxbAdapter;
import org.apache.log4j.Logger;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlIDREF;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 10:50 AM
 */

@XmlRootElement(name = "model")
public class
        CommunityModel {

    private static Logger logger = Logger.getLogger(CommunityModel.class);
    
    @XmlElement(name = "communities")
    List< Community > communities = new ArrayList<Community>();

    @XmlElement(name = "windows")
    Date[][] windows;

    public TevaParameters getParameters() {
        return parameters;
    }

    @XmlElement(name = "parameters")
    TevaParameters parameters;

    @XmlJavaTypeAdapter(MapJaxbAdapter.class)
    @XmlElement(name="apawners")
    public Map<Integer,Set<Connection>> spawners = new HashMap<Integer,Set<Connection>>();

    @XmlJavaTypeAdapter(MapJaxbAdapter.class)
    @XmlElement(name="consumedBy")
    public Map<Integer,Set<Connection>> consumers = new HashMap<Integer,Set<Connection>>();

    @XmlJavaTypeAdapter(MapJaxbAdapter.class)
    @XmlElement(name="informs")
    public Map<Integer,Set<Connection>> informs = new HashMap<Integer,Set<Connection>>();

    public String getCorpusName() {
        return corpusName;
    }

    @XmlElement(name = "corpus")
    private String corpusName;


    private CommunityModel() {

    }

    public CommunityModel(TevaParameters params, Date[][] windows, String corpusName) {
        this.windows = windows;
        this.parameters = params;
        this.corpusName = corpusName;

    }


    public void addConnection(int bin, float weight, ConnectionType type, Community from, Community to) {


        if (type == ConnectionType.CONSUMES && to.getMaxBin() < bin) {
            logger.warn("Older community "+to.getId()+"consuming a newer one! "+from.getId());
        }

        Set<Connection> conn = map(type).get(bin);
        if (conn == null) {
            map(type).put(bin,conn = new HashSet<Connection>());
        }
        conn.add(new Connection(from,to,weight));
    }

    public int getMaxPostCount() {
           int maxposts = 0;
           for (Community c : communities) {
               for (ConversationChunk chunk:c.getAssignments()) {
                   if (chunk.messages.size() > maxposts) maxposts = chunk.messages.size();
               }

           }
           return maxposts;

       }


    public Set<Connection> getConnection(int bin, ConnectionType  type) {
        return map(type)!=null?map(type).containsKey(bin)?map(type).get(bin):Collections.<Connection>emptySet(): Collections.<Connection>emptySet();
    }

    public Date[][] getWindows() {
        return windows;
    }

    private Map<Integer,Set<Connection>> map(ConnectionType type) {
        if (type == ConnectionType.SPAWNS) {
            return spawners;
        } else if (type == ConnectionType.INFORMS) {
            return informs;
        } else if (type == ConnectionType.CONSUMES) {
            return consumers;
        }
        else return null;
    }

    public void addCommunity(Community c) {
        communities.add(c);
    }

    public List<Community> getCommunities() {
        return communities;
    }

    @XmlRootElement(name = "Connection")
    @XmlAccessorType(XmlAccessType.NONE)
    public static class Connection {

        @XmlAttribute
        public float weight;

        @XmlIDREF
        @XmlAttribute
        public Community source;

        @XmlIDREF
        @XmlAttribute
        public Community target;

        public Connection(){}


        public Connection(Community source, Community target, float weight) {
            this.source = source;
            this.target = target;
            this.weight = weight;
        }

        public boolean equals(Connection c) {
            return weight == c.weight && source == c.source && target == c.target;
        }

        public String toString() {
            return source.getId()+"->"+target.getId();
        }
    }

    public enum ConnectionType {

        SPAWNS, CONSUMES, INFORMS



    }

}


<|start_filename|>src/edu/mit/cci/teva/engine/NetworkProvider.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 11:19 AM
 */
public interface NetworkProvider {

    public int getNumberWindows();
    public Network getNetworkAt(int i);
}


<|start_filename|>src/edu/mit/cci/sna/jung/UndirectedJungNetwork.java<|end_filename|>
package edu.mit.cci.sna.jung;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.uci.ics.jung.graph.UndirectedSparseMultigraph;
import org.apache.log4j.Logger;

import java.util.Collection;
import java.util.Collections;

/**
 * User: jintrone
 * Date: 9/21/12
 * Time: 8:50 PM
 */
public class UndirectedJungNetwork extends UndirectedSparseMultigraph<Node, Edge> implements Network {

    public UndirectedJungNetwork() {
    }

    Logger log = Logger.getLogger(UndirectedJungNetwork.class);

    public UndirectedJungNetwork(DirectedJungNetwork copy) {
        for (Edge edge : copy.getEdges()) {
            Node[] nodes = edge.getEndpoints();
            this.addEdge(new EdgeImpl(nodes[0], nodes[1], edge.getWeight(), true), copy.getEndpoints(edge));
        }
    }

    public Collection<Node> getNodes() {
        return getVertices();
    }


    public boolean isDirected() {
        return false;
    }

    public void remove(Edge e) {
        this.removeEdge(e);
    }

    public void remove(Node n) {
        this.removeVertex(n);
    }

    public void add(Node n) {
        this.addVertex(n);
    }

    public void add(Edge e) {
        if (!containsEdge(e)) {
            this.addEdge(e, e.getEndpoints()[0], e.getEndpoints()[1]);
        }
    }

    public Collection<Edge> getEdges() {
        return Collections.unmodifiableSet(edges.keySet());
    }

    public Collection<Node> getVertices() {
        return Collections.unmodifiableSet(vertices.keySet());
    }

    public Edge addEdge(Node node1, Node node2, float weight) {
        Edge e = findEdge(node1, node2);
        if (e != null) {
            log.debug("Already added edge, updating weight");
        } else {
            e = new EdgeImpl(node1, node2, weight, false);
            this.addEdge(e, node1, node2);
        }
        return e;
    }

    public Edge addAndSum(Node node1, Node node2, float weight) {
        Edge e = findEdge(node1, node2);
        if (e != null) {
            e.setWeight(e.getWeight()+weight);
        } else {
            e = new EdgeImpl(node1, node2, weight, false);
            this.addEdge(e, node1, node2);
        }
        return e;
    }


}


<|start_filename|>src/edu/mit/cci/teva/util/CFinderBatchRunner.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.cpm.cfinder.CFinderRunner;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Properties;

/**
 * User: jintrone
 * Date: 7/26/12
 * Time: 1:10 PM
 */
public class CFinderBatchRunner {

    public static String DEFAULT_PROPS = "resources/cfinderbatch.properties";

    public static String INPUTDIR_PROP = "inputdir";
    public static String EXTENSION_PROP = "extension";
    public static String IGNORE_EXISTING_PROP = "ignoreexisting";
    public static String CFINDER_APP_PROP = "cfinderapplication";
    public static String CFINDER_LICENSE_PROP = "cfinderlicense";

    public static Logger log = Logger.getLogger(CFinderBatchRunner.class);

    Properties props;


    public CFinderBatchRunner() throws IOException {
      this(DEFAULT_PROPS);
    }

    public CFinderBatchRunner(Properties p) {
        this.props = p;
        process();
    }

    public CFinderBatchRunner(String propsFileName) throws IOException {
        props = new Properties();
        props.load(new FileReader(propsFileName));
        process();
    }

    public void process() {
       File inputdir = new File(props.getProperty(INPUTDIR_PROP,"."));
       final String extension = props.getProperty(EXTENSION_PROP,null);
       if (!inputdir.exists() || !inputdir.isDirectory()) {
           throw new RuntimeException("Could not locate input directory: "+inputdir.getAbsolutePath());
       }

       File[] files = inputdir.listFiles(new FilenameFilter() {
           public boolean accept(File file, String s) {
               return extension == null || s.endsWith(extension);
           }
       });
      // MIN_WEIGHT("w"), MAX_WEIGHT("W"), CLIQUE_SIZE("k"), INTENSITY("I"), MAX_TIME("t"), DIRECTED("D");

        List<String> params = new ArrayList<String>();
        for (Enumeration e =props.propertyNames();e.hasMoreElements();) {
            String prop = (String)e.nextElement();

            if (CFinderRunner.CommandLineParams.resolve(prop)!=null) {
                params.add(prop);
                params.add(props.getProperty(prop));
            }
        }

        String license = props.getProperty(CFINDER_LICENSE_PROP,null);
                String app = props.getProperty(CFINDER_APP_PROP,null);

        CFinderRunner cfinder = new CFinderRunner(app,license,params.toArray(new String[params.size()]));


        for (File f:files) {
            try {
                cfinder.process(f,Boolean.parseBoolean(props.getProperty(IGNORE_EXISTING_PROP,"false")));
            } catch (IOException e) {
                log.error(e);
            } catch (InterruptedException e) {
                log.error(e);
            }
        }



    }




}


<|start_filename|>src/edu/mit/cci/teva/util/TevaUtils.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.DirectedJungNetwork;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityFrame;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.engine.ConversationChunk;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.serialization.CommunityFrameJaxbAdapter;
import edu.mit.cci.util.F;
import edu.mit.cci.util.U;
import edu.uci.ics.jung.algorithms.scoring.EigenvectorCentrality;
import org.apache.commons.collections15.Transformer;
import org.apache.log4j.Logger;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.helpers.DefaultValidationEventHandler;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/8/12
 * Time: 3:11 PM
 */

public class TevaUtils {

    private static Logger logger = Logger.getLogger(TevaUtils.class);

    /**
     * Iterate through a conversation and return a list of all posts sorted by time
     *
     * @param c The conversation
     * @return A list of sorted posts
     */
    public static List<Post> getAllSortedPosts(Conversation c) {
        final List<Post> result = new ArrayList<Post>();
        U.map(c.getAllThreads()).forEach(new F<DiscussionThread>() {
            public void apply(DiscussionThread discussionThread) {
                U.map(discussionThread.getPosts()).forEach(new F<Post>() {
                    public void apply(Post post) {
                        result.add(post);
                    }
                });
            }
        });
        Collections.sort(result,new Comparator<Post>() {
            @Override
            public int compare(Post o1, Post o2) {
                return o1.getTime().compareTo(o2.getTime());
            }
        });
        return result;
    }

    /**
     * Iterate through a conversation and return a list of all posts sorted by time
     *
     * @param c The conversation
     * @return A list of sorted posts
     */
    public static Map<String,Post> getAllMappedPosts(Conversation c) {
        final Map<String,Post> result = new HashMap<>();
        U.map(c.getAllThreads()).forEach(new F<DiscussionThread>() {
            public void apply(DiscussionThread discussionThread) {
                U.map(discussionThread.getPosts()).forEach(new F<Post>() {
                    public void apply(Post post) {
                       result.put(post.getPostid(),post);
                    }
                });
            }
        });
        return result;
    }

    public static void serialize(Clique clique, File f) throws JAXBException, IOException {
        JAXBContext jc = JAXBContext.newInstance(Clique.class);
        Marshaller m = jc.createMarshaller();
        m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT,
                Boolean.TRUE);

//        edu.mit.cci.snatools.topicevolution.Utils.analyzeDrainagePatterns(c);
        FileOutputStream output = new FileOutputStream(f);
        m.marshal(clique, output);
        output.flush();
        output.close();

    }

    public static void serialize(CommunityFrame communityFrame, File f) throws JAXBException, IOException {
        JAXBContext jc = JAXBContext.newInstance(CommunityFrameJaxbAdapter.class, CommunityFrame.class);
        Marshaller m = jc.createMarshaller();
        m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT,
                Boolean.TRUE);

//        edu.mit.cci.snatools.topicevolution.Utils.analyzeDrainagePatterns(c);
        FileOutputStream output = new FileOutputStream(f);
        m.marshal(communityFrame, output);
        output.flush();
        output.close();

    }

    public static CommunityModel getCommunityModelFromFile(File file) throws JAXBException, FileNotFoundException {
        return deserialize(file, CommunityModel.class);
    }

    public static List<String> getCommunityRepresentation(Network c, int size) {
        UndirectedJungNetwork graph = new UndirectedJungNetwork();
        List<String> s = new ArrayList<String>();

        for (Edge e : c.getEdges()) {
            graph.add(e);
        }

        final EigenvectorCentrality<Node, Edge> ev = new EigenvectorCentrality<Node, Edge>(graph, new Transformer<Edge, Float>() {
            public Float transform(Edge e) {
                return e.getWeight();
            }
        });
        ev.evaluate();
        List<Node> sortedlist = new ArrayList<Node>(graph.getVertices());
        Collections.sort(sortedlist, new Comparator<Node>() {
            public int compare(Node defaultJungNode, Node defaultJungNode1) {
                double score1 = ev.getVertexScore(defaultJungNode);
                double score2 = ev.getVertexScore(defaultJungNode1);
                if (score1 < score2) return 1;
                else if (score2 == score1) return 0;
                else return -1;

            }
        }
        );

        for (int i = 0; i < Math.min(size, sortedlist.size()); i++) {
            s.add(sortedlist.get(i).getLabel());
        }
        return s;


    }

    /**
     * Creates a graph of all topics / time windows, with each step connected, and each of the link types between topics included
     *
     * @param model
     * @param spawns
     * @param consumes
     * @param informs
     * @return
     */
    public static DirectedJungNetwork createCommunityGraph(CommunityModel model, boolean spawns, boolean consumes, boolean informs) {
        Map<String, List<Node>> nodemap = new HashMap<String, List<Node>>();
        DirectedJungNetwork result = new DirectedJungNetwork();
        for (Community cs : model.getCommunities()) {
            List<Node> nodes = new ArrayList<Node>();
            nodemap.put(cs.getId(), nodes);


            Node last = null;
            for (int i = 0; i < model.getWindows().length; i++) {
                CommunityFrame s = cs.getCommunityAtBin(i);
                if (s == null) {

                    nodes.add(null);

                } else {

                    Node n = new NodeImpl("C" + cs.getId() + "W" + i);
                    n.setProperty("Size", s.getNodes().size());
                    n.setProperty("Window", i);
                    n.setProperty("CommunityId", cs.getId());
                    int numMessages =0;
                    for (ConversationChunk chunk:cs.getAssignments()) {
                        if (chunk.window == i) {
                            numMessages+=chunk.messages.size();
                        }
                    }
                    n.setProperty("Messages",numMessages);
                    //n.setProperty("Intensity", (int) (300.0f * (s..getIntegration() * s.getStats().getThreadFocus() * (s.getStats().getNumPosts() / (float) maxposts))));

                    nodes.add(n);
                    result.addVertex(n);
                    if (last != null) {
                        Edge e = new EdgeImpl(last, n, 1.0f, true);
                        result.add(e);
                    }
                    last = n;
                }

            }
            while (nodes.size() > 0 && nodes.get(nodes.size() - 1) == null) {
                nodes.remove(nodes.size() - 1);
            }

        }


        for (int i = 0; i < model.getWindows().length; i++) {

            if (consumes && model.consumers.containsKey(i)) {
                for (CommunityModel.Connection c : model.consumers.get(i)) {
                    List<Node> nodes = nodemap.get(c.source.getId());
                    Node n1 = nodes.get(nodes.size() - 1);
                    Node n2 = nodemap.get(c.target.getId()).get(i);
                    if (n1.getProperty("CommunityId").equals(n2.getProperty("CommunityId"))) {
                        logger.warn("BUG - consumes links should not be to the same community "+c.source.getId()+"->"+c.target.getId()+" in "+i);
                        continue;
                    }
                    Edge e = new EdgeImpl(n1, n2, c.weight, true);
                    e.setProperty("Type", "CONSUMES");
                    result.add(e);

                }
            }
            if (spawns && model.spawners.containsKey(i)) {
                for (CommunityModel.Connection c : model.spawners.get(i)) {
                    Node n1 = nodemap.get(c.source.getId()).get(i - 1);
                    Node n2 = nodemap.get(c.target.getId()).get(i);
                    Edge e = new EdgeImpl(n1, n2, c.weight, true);
                    e.setProperty("Type", "SPAWNS");
                    result.add(e);
                }
            }
            if (informs && model.informs.containsKey(i)) {
                for (CommunityModel.Connection c : model.informs.get(i)) {
                    List<Node> nodes = nodemap.get(c.source.getId());
                    Node n1 = nodes.get(nodes.size() - 1);
                    Node n2 = nodemap.get(c.target.getId()).get(i);
                    Edge e = new EdgeImpl(n1, n2, c.weight, true);
                    e.setProperty("Type", "INFORMS");
                    result.add(e);
                }
            }
        }

        return result;


    }


    public static void addDrainageScoresForCommunityGraph(DirectedJungNetwork graph) {

        for (Node node : graph.getVertices()) {


            node.setProperty("Centrality", getIncidentCommunities(graph, node).size() - 1);


        }

    }

    public static Set<String> getIncidentCommunities(DirectedJungNetwork graph, Node node) {
        Set<String> result = new HashSet<String>();
        //logger.info("In Node: "+node.getProperty("CommunityId")+" Window:"+node.getProperty("Window"));
        for (Edge e : graph.getInEdges(node)) {
            result.addAll(getIncidentCommunities(graph, e.getEndpoints()[0]));
        }
        result.add((String) node.getProperty("CommunityId"));
        return result;
    }


    public static Clique deserializeClique(File f) throws JAXBException, FileNotFoundException {
        JAXBContext context = JAXBContext.newInstance(Clique.class);
        Clique x = (Clique) context.createUnmarshaller().unmarshal(new FileInputStream(f));
        return x;
    }

    public static <T> T deserialize(File f, Class... support) throws JAXBException, FileNotFoundException {
        JAXBContext context = JAXBContext.newInstance(support);
        Unmarshaller m = context.createUnmarshaller();
        m.setEventHandler(new DefaultValidationEventHandler());
        T x = (T) m.unmarshal(new FileInputStream(f));
        return x;
    }

    public static <T> void serialize(File f, T obj, Class... support) throws JAXBException, IOException {
        JAXBContext context = JAXBContext.newInstance(support);
        Marshaller m = context.createMarshaller();
        m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT,
                Boolean.TRUE);
        FileOutputStream output = new FileOutputStream(f);
        m.marshal(obj, output);
        output.flush();
        output.close();

    }

    /**
     * Creates a graph of all topics / time windows, with each step connected, and each of the link types between topics included
     *
     * @param jaxb
     * @param spawns
     * @param consumes
     * @param informs
     * @return public static UndirectedJungNetwork createCommunityGraph(CommunityModel jaxb, boolean spawns, boolean consumes, boolean informs) {
    Map<String, List<Node>> nodemap = new HashMap<String, List<Node>>();
    UndirectedJungNetwork result = new UndirectedJungNetwork();
    for (Community cs : jaxb.getCommunities()) {
    List<Node> nodes = new ArrayList<Node>();
    nodemap.put(cs.getId(), nodes);
    int maxposts = jaxb.getMaxPostCount();
    for (int i = 0; i < jaxb.getWindows().length; i++) {
    CommunityFrame s = cs.getCommunityAtBin(i);
    if (s == null) continue;
    else {
    NodeImpl n = new NodeImpl("C" + cs.getId());
    n.setProperty("Size", s.getNodes().size());
    n.setProperty("Age", i);
    n.setProperty("CommunityId", cs.getId());
    //TODO add community intensity

    //n.setProperty("Intensity", (int) (300.0f * (s.getStats().getIntegration() * s.getStats().getThreadFocus() * (s.getStats().getNumPosts() / (float) maxposts))));

    nodes.add(n);
    result.addVertex(n);
    }
    }
    }

    for (Community cs : jaxb.getCommunities()) {
    Set<CommunityModel.Connection> destinations = new HashSet<CommunityModel.Connection>();
    if (consumes) {
    for (Map.Entry<Integer,Set<CommunityModel.Connection>> ent:jaxb.consumers.entrySet()) {
    for (CommunityModel.Connection con:ent.getValue()) {
    Node src = null;
    Node dest = null;
    for (Node ns: nodemap.get(cs.getId())) {
    if (ns.getProperty("Age").equals(ent.getKey()-1)) {
    src = ns;
    }   break;
    }
    for (Node ns:nodemap.get(con.target.getId())) {
    if ()
    }
    }
    }
    };
    if (spawns) destinations.addAll(cs.getSpawns());


    for (DestinationJaxb d : destinations) {
    DefaultJungNode src = null;
    DefaultJungNode dest = null;
    for (DefaultJungNode ns : nodemap.get(cs.getCommunityId())) {
    if (ns.getAttribute("Age").equals(d.getWindow() - 1)) {
    src = ns;
    break;
    }
    }

    for (DefaultJungNode nd : nodemap.get(d.getCommunityId())) {
    if (nd.getAttribute("Age").equals(d.getWindow())) {
    dest = nd;
    break;
    }
    }
    result.addEdge(src, dest, 1.0f);
    }

    if (informs) {
    for (DestinationJaxb d : cs.getInforms()) {
    int idx = nodemap.get(cs.getCommunityId()).size() - 1;
    DefaultJungNode src = nodemap.get(cs.getCommunityId()).get(idx);
    DefaultJungNode dest = null;
    for (DefaultJungNode nd : nodemap.get(d.getCommunityId())) {
    if (nd.getAttribute("Age").equals(d.getWindow())) {
    dest = nd;
    break;
    }
    }
    result.addEdge(src, dest, 1.0f);
    }
    }


    }

    for (List<DefaultJungNode> nodes : nodemap.values()) {
    for (int i = 1; i < nodes.size(); i++) {
    result.addEdge(nodes.get(i - 1), nodes.get(i), 1.0f);
    }
    }
    return result;


    }
     */


}


<|start_filename|>src/edu/mit/cci/teva/util/NetWriterStrategy.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Network;

import java.io.File;
import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 6/3/13
 * Time: 2:37 PM
 * To change this template use File | Settings | File Templates.
 */
public interface NetWriterStrategy {

    public File writeNetwork(Network network, int window) throws IOException;
    public File getNetworkFilename(int window);
}


<|start_filename|>src/edu/mit/cci/util/U.java<|end_filename|>
package edu.mit.cci.util;

import javax.swing.*;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.List;

/**
 * User: jintrone
 * Date: 9/24/12
 * Time: 11:34 PM
 */
public class U {

    //Collection methods

    public static <T> T last(List<T> l) {
        return (l==null || l.isEmpty())?null:l.get(l.size()-1);
    }

    public static <T> T first(List<T> l) {
        return (l==null || l.isEmpty())?null:l.get(0);
    }

    public static <T> T first(Collection<T> c) {
        return (c==null || c.isEmpty())?null:c.iterator().next();
    }

    //Slow!
    public static <T> T last(Collection<T> c) {
        return (c==null || c.isEmpty())?null:last(new ArrayList<>(c));
    }


    public static <T, V> int binarySearch(List<? extends T> list, V key, Comparator<? super V> c, Adapter<? super T, V> adapter) {

        if (list instanceof RandomAccess || list.size() < 5000)
            return indexedBinarySearch(list, key, c, adapter);
        else
            return iteratorBinarySearch(list, key, c, adapter);
    }

    private static <T, V> int indexedBinarySearch(List<? extends T> l, V key, Comparator<? super V> c, Adapter<? super T, V> adapter) {
        int low = 0;
        int high = l.size() - 1;
        while (low <= high) {
            int mid = (low + high) >>> 1;
            T midVal = l.get(mid);
            int cmp = c.compare(adapter.adapt(midVal), key);
            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else
                return mid; // key found
        }
        return -(low + 1);  // key not found
    }

    private static <T, V> int iteratorBinarySearch(List<? extends T> l, V key, Comparator<? super V> c, Adapter<? super T, V> adapter) {
        int low = 0;
        int high = l.size() - 1;
        ListIterator<? extends T> i = l.listIterator();
        while (low <= high) {
            int mid = (low + high) >>> 1;
            T midVal = get(i, mid);
            int cmp = c.compare(adapter.adapt(midVal), key);
            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else
                return mid; // key found
        }
        return -(low + 1);  // key not found
    }

    /**
     * 308        * Gets the ith element from the given list by repositioning the specified
     * 309        * list listIterator.
     * 310
     */
    private static <T> T get(ListIterator<? extends T> i, int index) {
        T obj = null;
        int pos = i.nextIndex();
        if (pos <= index) {
            do {
                obj = i.next();
            } while (pos++ < index);
        } else {
            do {
                obj = i.previous();
            } while (--pos > index);
        }
        return obj;
    }

    public static void delete(File f) throws IOException {
        if (f.isDirectory()) {
            for (File c : f.listFiles()) {
                delete(c);
            }
        }
        f.delete();
//        if (!f.delete())
//            throw new FileNotFoundException("Failed to delete file: " + f);
    }

    public static void cleanDirectory(File f) throws IOException {
        if (!f.isDirectory()) return;
        else {
            for (File f2:f.listFiles()) {
                delete(f2);
            }
        }
    }

    public static interface Adapter<T, V> {
        public V adapt(T obj);
    }


    public static Map<String, Object> mapify(Object... s) {
        Map<String, Object> result = new HashMap();
        for (int i = 1; i < s.length; i += 2) {
            result.put(s[i - 1].toString(), s[i]);
        }
        return result;
    }

    public static <T> Iterable<T> multiIterator(final Iterable<T>... cs) {
        return new Iterable<T>() {


            public Iterator<T> iterator() {
                return new Iterator<T>() {

                    List<Iterator<T>> its = new ArrayList<Iterator<T>>();

                    {
                        for (Iterable<T> i : cs) {
                            its.add(i.iterator());
                        }
                    }

                    public boolean hasNext() {
                        while (its.size() > 0 && !its.get(0).hasNext()) {
                            its.remove(0);

                        }
                        return its.size() > 0 && its.get(0).hasNext();
                    }

                    public T next() {
                        while (its.size() > 0 && !its.get(0).hasNext()) {
                            its.remove(0);

                        }
                        if (its.size() > 0) return its.get(0).next();
                        else throw new NoSuchElementException();
                    }

                    public void remove() {
                        throw new UnsupportedOperationException("Remove is not supported");
                    }
                };
            }
        };
    }

    /**
     * Splits, just like the underlying String.split, but skips any empty tokens in the result
     *
     * @param str
     * @param regx
     * @return
     */
    public static String[] mysplit(String str, String regx) {
        String[] tmp = str.split(regx);
        List<String> result = new ArrayList<String>();
        for (String s : tmp) {
            if (!s.isEmpty()) {
                result.add(s);
            }
        }
        return result.toArray(new String[result.size()]);
    }


    public static File getAnyFileNative(String message, String file, int type) {
        //JFileChooser chooser = new JFileChooser(file);

        JFrame frame = new JFrame();
        frame.setVisible(true);
        FileDialog dialog = new FileDialog(frame);
        dialog.setTitle(message);
        dialog.setFile(file);
        dialog.setMode(type);
        dialog.setVisible(true);
        File result =  dialog.getFiles()[0];
        frame.dispose();
        return result;

//        chooser.setFileSelectionMode(type);
//        if (message != null && !message.isEmpty()) chooser.setDialogTitle(message);
//        int returnVal = chooser.showOpenDialog(frame);
//        if (returnVal == JFileChooser.APPROVE_OPTION) {
//            File f = chooser.getSelectedFile();
//            frame.dispose();
//            return f;
//        } else return null;
    }

    public static File getAnyFile(String message, String file, int type) {
        //JFrame frame = new JFrame();
        JFileChooser chooser = new JFileChooser(file);
        chooser.setFileSelectionMode(type);
        if (message != null && !message.isEmpty()) chooser.setDialogTitle(message);
        int returnVal = chooser.showOpenDialog(null);
        File f = null;
        if (returnVal == JFileChooser.APPROVE_OPTION) {
            f = chooser.getSelectedFile();

        }
        //frame.dispose();
        return f;
    }

    public static boolean move(File f, File dest) {
        File actualdest;
        if (dest.isFile()) {
            actualdest = dest;
        } else {
            actualdest = new File(dest, f.getName());
        }
        return f.renameTo(actualdest);

    }

    public static <T> List<T> filter(List<T> filtered, Filter<T> f) {
        List<T> result = new ArrayList<T>();
        if (filtered != null && !filtered.isEmpty()) {
            for (T obj : filtered) {
                if (f.accept(obj)) {
                    result.add(obj);
                }
            }
        }
        return result;

    }

    public static <T> MappingIterator<T> map(Collection<T> x) {
        return new MappingIterator<T>(x);

    }

    public static class MappingIterator<T> {

        Collection<T> c;

        public MappingIterator(Collection<T> c) {
            this.c = c;
        }

        public void forEach(F<T> fx) {
            for (T a : c) {
                fx.apply(a);
            }
        }

    }


    public static <K,V> void lput (Map<K,List<V>> map, K key, V val) {
        List<V> l = map.get(key);
        if (l == null) {
            map.put(key, l= new ArrayList<V>());
        }
        l.add(val);

    }


}


<|start_filename|>src/edu/mit/cci/teva/engine/MergeStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;

import java.util.Collection;
import java.util.List;

/**
 * User: jintrone
 * Date: 7/25/12
 * Time: 5:14 PM
 */
public interface MergeStrategy {
    List<Network> process(Network from, Collection<CommunityFrame> cdfrom, Network to, Collection<CommunityFrame> cdto, int window);
}


<|start_filename|>src/edu/mit/cci/teva/TevaFactory.java<|end_filename|>
package edu.mit.cci.teva;

import com.sun.java.swing.plaf.windows.resources.windows;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.WindowStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.CorpusToNetworkGenerator;
import edu.mit.cci.text.wordij.TextToNetworkGenerator;

import java.io.IOException;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/12/12
 * Time: 11:22 PM
 */
public interface TevaFactory {
    public List<List<Windowable>> getConversationData() throws IOException;

    public Tokenizer<String> getTokenizer() throws IOException;

    public MergeStrategy getMerger();

    public EvolutionStepStrategy getStepper(CommunityModel model);

    public WindowStrategy.Factory<Windowable> getTopicWindowingFactory();

    public WindowStrategy.Factory<Windowable> getPostAssignmentWindowingFactory();

    public TextToNetworkGenerator getNetworkCalculator();

    public CommunityFinder getFinder();

    public BinningStrategy<Windowable> getTopicBinningStrategy(List<List<Windowable>> data, WindowStrategy.Factory<Windowable> windows);

    public BinningStrategy<Windowable> getMembershipBinningStrategy(List<List<Windowable>> data, WindowStrategy.Factory<Windowable>windows);

    public TopicMembershipEngine getMembershipEngine(CommunityModel model, Conversation conversation);

    public CorpusToNetworkGenerator<Windowable> getNetworkGenerator(BinningStrategy<Windowable> binningStrategy);

    public CommunityMembershipStrategy getMembershipMatchingStrategy();
}


<|start_filename|>src/edu/mit/cci/text/windowing/SingleThreadTokenWidthWindowingStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.List;

/**
 * Created by josh on 1/14/14.
 */
public class SingleThreadTokenWidthWindowingStrategy extends PrecomputedTimeWindowStrategy {


    public SingleThreadTokenWidthWindowingStrategy(List<Windowable> current, int numTokensWidth, int numTokensDelta) {
       super(WindowingUtils.analyzeSingleThreadBySize(current, numTokensWidth, numTokensDelta));
    }
}


<|start_filename|>src/edu/mit/cci/teva/engine/BasicStepStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.cpm.cfinder.CFinderCommunityFinder;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * User: jintrone
 * Date: 7/24/12
 * Time: 6:18 AM
 */
public class BasicStepStrategy implements EvolutionStepStrategy {


    private static Logger log = Logger.getLogger(BasicStepStrategy.class);

    private CommunityModel model;
    private TevaParameters params;


    public BasicStepStrategy(CommunityModel model, TevaParameters parameters) {
        this.model = model;
        this.params = parameters;

    }

    public void setModel(CommunityModel model) {
        this.model = model;
    }

    public void processStep(int i, List<CommunityFrame> fromcliques, List<Network> mergedcliques, List<CommunityFrame> tocliques) {
        log.debug(i + ". From: " + fromcliques);
        log.debug(i + ". Thru: " + mergedcliques.size() + " merged cliques");
        log.debug(i + ". To: " + tocliques);
        if (fromcliques == null && tocliques == null) {
            log.debug(i + ". NO COMMUNITIES FOR WINDOW, skipping...");
            return;
        }
        List<ScoredCommunityFramePair> evolution = new ArrayList<ScoredCommunityFramePair>();

        Set<CommunityFrame> accounting = new HashSet<CommunityFrame>();
        for (Network vinfo : mergedcliques) {
            log.debug("Processing Window:" + i + " Merged communities:" + NetworkUtils.simpleString(vinfo));

            List<CommunityFrame> f = new ArrayList<CommunityFrame>();
            List<CommunityFrame> t = new ArrayList<CommunityFrame>();

            if (fromcliques != null) {
                for (CommunityFrame info : fromcliques) {

                    if (vinfo.getEdges().containsAll(info.getEdges())) {
                        f.add(info);
                        accounting.add(info);
                        log.info("Adding FROM as candidate: " + NetworkUtils.simpleString(info));
                    } else {

                    }
                }
            }


            if (tocliques != null) {
                for (CommunityFrame info : tocliques) {

                    if (vinfo.getEdges().containsAll(info.getEdges())) {
                        t.add(info);
                        accounting.add(info);
                        log.info("Adding TO as candidate: " + NetworkUtils.simpleString(info));
                    } else {


                    }
                }
            }


            if (t.isEmpty()) {
                log.debug(i + ". NO TARGETS " + f);
                for (CommunityFrame info1 : f) {
                    evolution.add(new ScoredCommunityFramePair(info1, CommunityFrame.EMPTY, params.getScoringMethod().score(info1, CommunityFrame.EMPTY)));
                }


            } else if (f.isEmpty()) {
                log.debug(i + ". NO SOURCES " + t);
                for (CommunityFrame info1 : t) {
                    evolution.add(new ScoredCommunityFramePair(CommunityFrame.EMPTY, info1, params.getScoringMethod().score(CommunityFrame.EMPTY, info1)));
                }

            } else {

                for (CommunityFrame info1 : f) {
                    for (CommunityFrame info2 : t) {
                        evolution.add(new ScoredCommunityFramePair(info1, info2, params.getScoringMethod().score(info1, info2)));
                    }
                }
            }

        }
        if (!accounting.containsAll(tocliques)) {
            Set<CommunityFrame> test = new HashSet<CommunityFrame>(accounting);
            test.removeAll(tocliques);

            throw new RuntimeException("To cliques should all be accounted for, there is a problem");
        }

        Collections.sort(evolution, new Comparator<ScoredCommunityFramePair>() {
            public int compare(ScoredCommunityFramePair scoredCliquePair, ScoredCommunityFramePair scoredCliquePair1) {
                if (scoredCliquePair.getScore() < scoredCliquePair1.getScore()) return 1;
                else if (scoredCliquePair.getScore() > scoredCliquePair1.getScore()) return -1;
                else return 0;
            }
        });
        log.debug(i + " cliques to assign:" + accounting);
        log.info(i + ". VIABLE PAIRS: " + evolution);

        for (ScoredCommunityFramePair pair : evolution) {

            //first clique not added, so add pair
            log.debug(i + ". Processing pair: " + pair);
            if (pair.getPair()[1] == CommunityFrame.EMPTY) {
                if (pair.getPair()[0].getCommunity() == null) {
                    addCommunityForFrame(pair.getPair()[0]);
                    log.warn(i + ". ADDING SPURIOUS COMMUNITY with no children (SHOULDN\'T BE HERE!) " + pair.getPair()[0].getCommunity());
                } else {
                    log.debug("NO MATCH FOR " + pair.getPair()[0].getCommunity());
                }

                accounting.remove(pair.getPair()[0]);


            } else if (pair.getPair()[0] == CommunityFrame.EMPTY) {
                addCommunityForFrame(pair.getPair()[1]);
                accounting.remove(pair.getPair()[1]);
                log.info(i + ". CREATE NEW COMMUNITY (NO PARENTS) " + pair.getPair()[1].getCommunity() + "->" + pair.getPair()[1].getNodes());


            } else if (accounting.contains(pair.getPair()[0])) {
                if (pair.getPair()[0].getCommunity() == null) {
                    addCommunityForFrame(pair.getPair()[0]);
                    log.warn(i + ". CREATE COMMUNITY (SHOULDN\'T BE HERE!): " + pair.getPair()[0].getCommunity());
                }

                //placement of this line will determine the policy regarding progeny
                int size = accounting.size();
                accounting.remove(pair.getPair()[0]);
                if (accounting.size() == size) {
                    log.warn("Did not remove frame from accounting: "+pair.getPair()[0]);
                }

                //pair.getPair()[0].getCommunity().setBin(i, pair.getPair()[0]);
                //logProcess.debug(i + ". Set data:" + pair.getPair()[0].getCommunity().getName() + pair.getPair()[0].nodes);
                //have not yet encountered second clique, so just add it, and add the members to the next step
                //in the community
                if (accounting.contains(pair.getPair()[1])) {
                    pair.getPair()[0].getCommunity().addFrame(pair.getPair()[1]);
                    log.debug(i + ". Evolve community: " + pair.getPair()[0].getCommunity().getName() + ":" + pair.getPair()[1]);
                    //  STOPPED
                    size = accounting.size();
                    accounting.remove(pair.getPair()[1]);
                    if (size == accounting.size()) {
                        log.warn("Accounting did not remove a frame!");
                    }

                } else {
                    //have already encountered the second clique, as part of another community, so expire the first of the pair
                    //logProcess.debug(i + ". Already assigned to clique:" + pair.getPair()[1]);
                    log.debug(i + ". Expire " + pair.getPair()[0]);
                    log.debug(i + ". SET PROGENITOR of " + pair.getPair()[1].getCommunity() + " to " + pair.getPair()[0].getCommunity() + " at " + i);
                    if (pair.getPair()[1].getCommunity().getId().equals(pair.getPair()[0].getCommunity().getId())) {
                        log.warn("Community shouldn't consume itself!");
                    }
                    model.addConnection(i, NetworkUtils.coverage(pair.getPair()[0], pair.getPair()[1]), CommunityModel.ConnectionType.CONSUMES, pair.getPair()[0].getCommunity(), pair.getPair()[1].getCommunity());


                    if (params.expireConsumedCommunities()) {
                        log.debug(i + "Expire:" + pair.getPair()[0].getCommunity() + " becomes " + pair.getPair()[1]);
                        pair.getPair()[0].getCommunity().expire();
                    }

                }


            }

            //we've already identified the evolution for the first community, but not second
            //second community is "born"
            else if (accounting.contains(pair.getPair()[1])) {

//                logProcess.debug(i + ". Already processed from-clique " + pair.getPair()[0]);
//                logProcess.debug(i + ". Processing to-clique: " + pair.getPair()[1]);
                accounting.remove(pair.getPair()[1]);
                if (pair.getPair()[1].getCommunity() == null) {
                    //this clause would allows us to determine that this pairing, even though it is not as good a pairing should
                    //still be allowed - merges results
//                    if (pair.getScore() >= params.getScoreForMerge()) {
//                        log.debug(i + ". REMERGE clique " + pair.getPair()[1] + " to mapped community " + pair.getPair()[0].getCommunity());
//                        //for bookeeping purposes in case we encounter another clique that would be consumed by this community.
//                        pair.getPair()[1].getCommunity() = pair.getPair()[0].getCommunity();
//                        pair.getPair()[0].merge(pair.getPair()[1]);
//
//                    } else {

                    addCommunityForFrame(pair.getPair()[1]);
                    log.info(i + ". CREATE COMMUNITY (spawned): " + pair.getPair()[1] + "->" + pair.getPair()[1].getNodes());
                    log.debug(i + ". SET NODES at " + i + " for " + pair.getPair()[1].getCommunity() + pair.getPair()[1].getNodes());
                    model.addConnection(i, NetworkUtils.coverage(pair.getPair()[0], pair.getPair()[1]), CommunityModel.ConnectionType.SPAWNS, pair.getPair()[0].getCommunity(), pair.getPair()[1].getCommunity());
                    log.debug(i + ". SET SPAWNER of " + pair.getPair()[1].getCommunity() + " to " + pair.getPair()[0].getCommunity() + " at " + i);
                    // }
                } else {
                    log.warn("Invalid state - target frame is attached to a community");
                }

            }

            log.debug(i + ". Remaining cliques: " + accounting);

            if (accounting.isEmpty()) {
                break;
            }

        }
    }


    private void addCommunityForFrame(CommunityFrame frame) {
        Community c = Community.create();
        c.addFrame(frame);
        model.addCommunity(c);
    }

    public static void main(String[] args) throws IOException, CommunityFinderException {


        TevaParameters param = new TevaParameters();
        param.setProperty(TevaParameters.WORKING_DIRECTORY, "../REASONTEVA/work");
        BasicStepStrategy step = new BasicStepStrategy(new CommunityModel(param, new Date[][]{}, "TestCorpus"), param);

        CFinderCommunityFinder finder = new CFinderCommunityFinder(false, false, new TevaParameters());
        File one = new File("../REASONTEVA/work/CFinderNetwork.TEvA.0.net");
        File two = new File("../REASONTEVA/work/CFinderNetwork.TEvA.1.net");

        Network none = NetworkUtils.readNetworkFile(one);
        Network ntwo = NetworkUtils.readNetworkFile(two);

        List<CommunityFrame> frames1 = finder.findCommunities(one, 4, 0);
        List<CommunityFrame> frames2 = finder.findCommunities(two, 4, 1);

        FastMergeStrategy strategy = new FastMergeStrategy(4);
        List<Network> result = strategy.process(none, frames1, ntwo, frames2, 0);
        for (Network n : result) {
            log.debug("Network: " + n.getNodes());
        }

        step.processStep(0, frames1, result, frames2);


    }


}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityFinderException.java<|end_filename|>
package edu.mit.cci.teva.engine;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 4:25 PM
 */
public class CommunityFinderException extends Exception {

    public CommunityFinderException(String msg) {
        super(msg);
    }

    public CommunityFinderException(String s, Throwable throwable) {
        super(s, throwable);
    }
}


<|start_filename|>src/edu/mit/cci/teva/engine/SlowMergeStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;

import java.util.Collection;
import java.util.List;

/**
 * User: jintrone
 * Date: 5/5/13
 * Time: 10:21 PM
 */
public class SlowMergeStrategy implements MergeStrategy {

    private CommunityFinder finder;
    private TevaParameters params;

    public SlowMergeStrategy(CommunityFinder finder, TevaParameters params) {
        this.finder = finder;
        this.params = params;
    }

    public List<Network> process(Network from, Collection<CommunityFrame> cdfrom, Network to, Collection<CommunityFrame> cdto, int window) {
        Network combined = NetworkUtils.combine(from,to);
        for (Network n:cdfrom) {
            combined = NetworkUtils.combine(combined,n);
        }
        List<? extends Network> result;
        try {
            result = finder.findCommunities(combined,params.getFixedCliqueSize(),window,"combined");
        } catch (CommunityFinderException e) {
            throw new RuntimeException("Error processing communities in merged graph",e);
        }
        return (List<Network>) result;

    }
}


<|start_filename|>src/edu/mit/cci/teva/engine/ScoredCommunityFramePair.java<|end_filename|>
package edu.mit.cci.teva.engine;

/**
* User: jintrone
* Date: 10/2/12
* Time: 10:29 PM
*/
public class ScoredCommunityFramePair {

    CommunityFrame[] pair;
    float score;

    public ScoredCommunityFramePair(CommunityFrame one, CommunityFrame two, float score) {
      pair = new CommunityFrame[] {one,two};
        this.score = score;
    }

    public CommunityFrame[] getPair() {
        return pair;
    }
    public float getScore() {
        return score;
    }
}


<|start_filename|>src/edu/mit/cci/teva/cpm/cfinder/CFinderCommunityFinder.java<|end_filename|>
package edu.mit.cci.teva.cpm.cfinder;

import edu.mit.cci.sna.CliqueDecoratedNetwork;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.engine.CommunityFinder;
import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.teva.engine.CommunityFrame;
import edu.mit.cci.teva.engine.TevaParameters;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/3/12
 * Time: 10:08 PM
 */
public class CFinderCommunityFinder implements CommunityFinder {


    private static Logger log = Logger.getLogger(CFinderCommunityFinder.class);
    private static String CFINDER_NET_NAME = "CFinderNetwork";

    private File outputdir;


    private CFinderRunner cFinderRunner;
    private boolean overwriteAnalysis = false;
    private boolean overwriteNetworks = false;



    public CFinderCommunityFinder(boolean overwriteNetworks, boolean overwriteAnalysis, TevaParameters params) {
        this.overwriteNetworks = overwriteNetworks;
        this.overwriteAnalysis = overwriteAnalysis;
        this.outputdir = new File(params.getWorkingDirectory());
        cFinderRunner = new CFinderRunner(params.getCFinderExecutable(), params.getCFinderLicensePath());
        cFinderRunner.setParam(CFinderRunner.CommandLineParams.CLIQUE_SIZE, "" + params.getFixedCliqueSize());
        cFinderRunner.setParam(CFinderRunner.CommandLineParams.MIN_WEIGHT, "" + params.getMinimumLinkWeight());
    }


    public static File getNetworkFile(File outputdir, int window, String id) {
        return new File(outputdir, CFINDER_NET_NAME + "." + id + "." + window+".net");
    }

    public File getAnalysisOutputDirectory(File networkFile) {
        return new File(networkFile.getAbsolutePath() + "_files");
    }

    public List<CommunityFrame> findCommunities(File networkFile, int cliqueSizeAtWindow, int window) throws CommunityFinderException {
        File baseAnalysisDir = getAnalysisOutputDirectory(networkFile);
        File analysisDir = new File(baseAnalysisDir, "/k=" + cliqueSizeAtWindow);
        try {
            if (analysisDir.exists() && overwriteAnalysis) {
                U.delete(analysisDir);
            }
            if (!analysisDir.exists()) {
                cFinderRunner.setParam(CFinderRunner.CommandLineParams.OUTPUT, baseAnalysisDir.getAbsolutePath());
                cFinderRunner.process(networkFile, false);
            }
        } catch (IOException ex) {
            throw new CommunityFinderException("Error processing network files", ex);
        } catch (InterruptedException e) {
            throw new CommunityFinderException("Error running cfinder", e);
        }
        List<CliqueDecoratedNetwork> networks = null;
        try {
            networks = CFinderReader.readCommunities(cliqueSizeAtWindow, baseAnalysisDir);
        } catch (IOException ex) {
            throw new CommunityFinderException("Error reading analysis files");
        }

        List<CommunityFrame> result = new ArrayList<CommunityFrame>();
        for (CliqueDecoratedNetwork n : networks) {
            result.add(new CommunityFrame(window, n));
        }
        return result;

    }


    public List<CommunityFrame> findCommunities(Network currentGraph, int cliqueSizeAtWindow, int window, String id) throws CommunityFinderException {
        File networkFile = getNetworkFile(this.outputdir,window, id);
        if (!networkFile.exists() || overwriteNetworks) {
            try {
                U.delete(networkFile);
                log.debug("Attempting to create file at: "+networkFile.getAbsolutePath());
                NetworkUtils.createNetworkFile(currentGraph, networkFile, true);
            } catch (IOException e) {
                throw new CommunityFinderException("Error creating network file",e);
            }

        }
        return findCommunities(networkFile, cliqueSizeAtWindow, window);
    }



}


<|start_filename|>src/edu/mit/cci/teva/analysis/AnalysisConsole.java<|end_filename|>
package edu.mit.cci.teva.analysis;

import com.Ostermiller.util.ExcelCSVPrinter;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.jung.DirectedJungNetwork;
import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.util.CommunityWindow;
import edu.mit.cci.teva.util.MessageToModelMap;
import edu.mit.cci.teva.util.TevaUtils;
import edu.mit.cci.util.ConsoleBase;
import edu.mit.cci.util.ConsoleDocumentation;
import edu.mit.cci.util.U;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/12/12
 * Time: 9:33 AM
 */
public abstract class AnalysisConsole extends ConsoleBase {

    private Properties props;
    File pfile;


    public AnalysisConsole() throws IOException {
       this( "./.teva-analysis.properties");

    }

    public AnalysisConsole(String filename) throws IOException {
        pfile =  new File(filename);
        props = new Properties();
        if (!pfile.exists()) {
            pfile.createNewFile();
            updateProperties(new File("."));

        } else {
            props.load(new FileReader(pfile));
        }



    }


    private void updateProperties(File f) {
        try {
            PrintWriter writer = new PrintWriter(pfile);
            props.setProperty("last_file", f.getAbsolutePath());
            props.store(writer, "Teva Analysis Properties");
        } catch (FileNotFoundException e) {
            e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        } catch (IOException e) {
            e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
        }

    }

    public File getAnyFile(String message) {
        File f = U.getAnyFile(message, props.getProperty("last_file"), javax.swing.JFileChooser.FILES_ONLY);
        if (f != null) updateProperties(f);
        return f;
    }




    @ConsoleDocumentation(value= "Create community graph")
    public void runCreateCommunityGraph() throws IOException, JAXBException {
        CommunityModel model = TevaUtils.getCommunityModelFromFile(U.getAnyFile("Select community file", ".", javax.swing.JFileChooser.FILES_ONLY));

        String filename = askUser("Enter a filename:");
        boolean informs = false;
        boolean spawns = false;
        System.out.print("Include informs links (y/n)? ");
        String response = s.nextLine();
        if ("y".equals(response.trim())) {
            informs = true;
        }
        System.out.print("Include spawns links (y/n)? ");
        response = s.nextLine();
        if ("y".equals(response.trim())) {
            spawns = true;
        }
        DirectedJungNetwork network = TevaUtils.createCommunityGraph(model, spawns, true, informs);
        TevaUtils.addDrainageScoresForCommunityGraph(network);
        JungUtils.writeGraphML(network, U.mapify("Size", 0, "Messages",0,"Window", 0, "CommunityId", "", "Centrality", 0), filename + ".graphml");
    }

    @ConsoleDocumentation(value = "Convert to CSV")
    public void runCreateCSV() throws IOException, JAXBException, ParseException {

        File f = U.getAnyFile("Choose a community model", ".", javax.swing.JFileChooser.FILES_ONLY);
        Conversation conversation = getConversation();

        CommunityModel c = TevaUtils.getCommunityModelFromFile(f);
        DirectedJungNetwork network = TevaUtils.createCommunityGraph(c, false, true, true);
        MessageToModelMap mmap = new MessageToModelMap(c);

        for (Community comm : c.getCommunities()) {
            comm.getAssignments();

        }
        ExcelCSVPrinter printer = new ExcelCSVPrinter(new FileOutputStream(f.getName() + ".messages.csv"));
        String[] headers = new String[]{"timestamp", "authorid", "postid", "replytoid", "topic"};
        printer.println(headers);

        DateFormat format = DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM);
        Set<String> topics = new HashSet<String>();
        for (DiscussionThread t : conversation.getAllThreads()) {
            for (Post p : t.getPosts()) {
                CommunityWindow cw = mmap.getBestCommunityWindow(p.getPostid());
                if (cw != null) topics.add(cw.community);
                String[] line = new String[]{format.format(p.getTime()),
                        p.getUserid(), p.getPostid(), p.getReplyToId(), cw == null ? null : cw.community};
                printer.println(line);
            }
        }

        printer.flush();
        printer.close();

        printer = new ExcelCSVPrinter(new FileOutputStream(f.getName() + ".links.csv"));
        printer.println(new String[]{"from_topic", "to_topic", "from_time", "to_time"});
        for (Node from : network.getNodes()) {
            Collection<Edge> edges = network.getInEdges(from);
            for (Edge e : edges) {
                Node to = network.getOpposite(from, e);
                String type = (String) e.getProperty("Type");
                if (type != null && !type.isEmpty()) {

                    if (topics.contains((String) from.getProperty("CommunityId")) && topics.contains((String) to.getProperty("CommunityId"))) {
                        String[] line = new String[]{
                                (String) from.getProperty("CommunityId"),
                                format.format(c.getWindows()[(Integer) from.getProperty("Window")][1]),
                                (String) to.getProperty("CommunityId"),
                                format.format(c.getWindows()[(Integer) to.getProperty("Window")][0])
                        };
                        printer.println(line);
                    }
                }


            }


        }

        printer.flush();
        printer.close();
    }

    public abstract Conversation getConversation() throws IOException, ParseException;


    @ConsoleDocumentation(value = "Write stacked topic CSV.")
    public void runNarrowTopicCSV() throws IOException, JAXBException {

        File f = getAnyFile("Choose a community analysis file");
        System.out.print("Enter an output filename:");
        String name = s.nextLine();
        if (!name.endsWith(".csv")) {
            name += ".csv";
        }
        new DumpTopicsToCsv(TevaUtils.getCommunityModelFromFile(f), new FileOutputStream(name)).write();

    }

    @ConsoleDocumentation(value = "Write long topic CSV.")
    public void runWideTopicCSV() throws IOException, JAXBException {

        File f = getAnyFile("Choose a community analysis file");
        System.out.print("Enter an output filename:");
        String name = s.nextLine();
        if (!name.endsWith(".csv")) {
            name += ".csv";
        }
        new DumpTopicsToCsvLong(TevaUtils.getCommunityModelFromFile(f), new FileOutputStream(name)).write();

    }



   /**
    @ConsoleDocumentation(value = "Create a static graphml representation of a community evolution file")
    public void runCreateGraphMLCommunityGraph() throws JAXBException, IOException {
        CommunityModel model = TevaUtils.getCommunityModelFromFile(U.getAnyFile("Select community file", "."));


        boolean informs = false;
        boolean spawns = false;
        System.out.print("Include informs links (y/n)? ");
        String response = s.nextLine();
        if ("y".equals(response.trim())) {
            informs = true;
        }
        System.out.print("Include spawns links (y/n)? ");
        response = s.nextLine();
        if ("y".equals(response.trim())) {
            spawns = true;
        }
        createCommunityGraphMLGraph(model, informs, spawns);
    }
    **/
   /**
    private void createCommunityGraphMLGraph(CommunityModel model, boolean informs, boolean spawns) throws JAXBException, IOException {
        UndirectedJungNetwork graph = Utils.createCommunityGraph(jaxb, spawns, true, informs);
        Utils.addDrainageScoresForCommunityGraph(graph);
        String filename = jaxb.getCorpusName() + ".static.graphml";
        Utils.writeGraphML(graph, Utils.mapify("Size", 0, "Intensity", 0, "Age", 0, "CommunityId", 0, "Centrality", 0), filename);
    }
     **/

}


<|start_filename|>src/edu/mit/cci/sna/CliqueDecorater.java<|end_filename|>
package edu.mit.cci.sna;

import java.util.Collection;
import java.util.HashSet;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/3/12
 * Time: 10:58 PM
 */
public class CliqueDecorater implements CliqueDecoratedNetwork {

    public void add(Edge e) {
        network.add(e);
    }

    public void add(Node n) {
        network.add(n);
    }

    private Network network;
    private Set<Clique> cliques = new HashSet<Clique>();

    public CliqueDecorater(Network n) {
       this.network  = n;
    }

    public Collection<Edge> getEdges() {
        return network.getEdges();
    }

    public Collection<Node> getNodes() {
        return network.getNodes();
    }

    public boolean isDirected() {
        return network.isDirected();
    }

    public void remove(Edge e) {
        network.remove(e);
    }

    public void remove(Node n) {
        network.remove(n);
    }


    public Set<Clique> getCliques() {
        return cliques;
    }

    public void addCliques(Collection<Clique> cliques) {
        this.cliques.addAll(cliques);
    }


}


<|start_filename|>src/edu/mit/cci/teva/util/CommunityRelationships.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.util.Filter;
import edu.mit.cci.util.U;

import java.util.*;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 5/22/13
 * Time: 2:45 PM
 * To change this template use File | Settings | File Templates.
 */
public class CommunityRelationships {


    private Map<String, List<ResolvedConnection>> incoming = new HashMap<String, List<ResolvedConnection>>();


    public CommunityRelationships(CommunityModel model) {

        for (int i = 0; i < model.getWindows().length; i++) {
            Set<CommunityModel.Connection> conns = model.getConnection(i, CommunityModel.ConnectionType.CONSUMES);
            if (conns != null) {
                for (CommunityModel.Connection c : conns) {

                    ResolvedConnection in = new ResolvedConnection(c.source.getMaxBin(), i, c);
                    List<ResolvedConnection> list = incoming.get(c.target.getId());
                    if (list == null) {
                        incoming.put(c.target.getId(), list = new ArrayList<ResolvedConnection>());
                    }
                    list.add(in);

                }
            }
        }
    }


    public List<ResolvedConnection> getAllIncoming(String communityId) {
        return incoming.get(communityId);
    }


    public List<CommunityWindow> getAllIncoming(String communityId, final int window) {
        final List<CommunityWindow> result = new ArrayList<CommunityWindow>();

        U.filter(incoming.get(communityId), new Filter<ResolvedConnection>() {
            public boolean accept(ResolvedConnection obj) {
                if (window == obj.getTo()) {
                    result.add(new CommunityWindow(obj.getConnection().source.getId(), obj.getFrom()));
                    return true;
                } else return false;
            }
        });
        return result;
    }


    public List<CommunityWindow> getAllIncomingBefore(String communityId, final int window) {
        final List<CommunityWindow> result = new ArrayList<CommunityWindow>();

        U.filter(incoming.get(communityId), new Filter<ResolvedConnection>() {
            public boolean accept(ResolvedConnection obj) {
                if (window <= obj.getTo()) {
                    result.add(new CommunityWindow(obj.getConnection().source.getId(), obj.getFrom()));
                    return true;
                } else return false;
            }
        });
        return result;
    }
}


<|start_filename|>src/edu/mit/cci/teva/analysis/DumpCentralityRankedConversationToCsv.java<|end_filename|>
package edu.mit.cci.teva.analysis;

import com.Ostermiller.util.ExcelCSVPrinter;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.jung.DirectedJungNetwork;
import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.engine.ConversationChunk;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.util.CommunityWindow;
import edu.mit.cci.teva.util.Stringer;
import edu.mit.cci.teva.util.TevaUtils;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.text.DateFormat;
import java.util.*;

/**
 * User: jintrone
 * Date: 5/14/13
 * Time: 3:44 PM
 */
public class DumpCentralityRankedConversationToCsv {

    public String[] headers = new String[]{"Topic", "Date", "ThreadId", "MessageId","ReplyTo", "Consumes", "MaxCentrality", "Author", "Text"};



    protected ExcelCSVPrinter printer;
    protected CommunityModel model;
    protected Conversation conversation;
    protected Stringer<String> textStringer;

    private static Logger log = Logger.getLogger(DumpCentralityRankedConversationToCsv.class);

    public DumpCentralityRankedConversationToCsv(CommunityModel model, Conversation conversation, OutputStream stream) throws UnsupportedEncodingException {
        printer = new ExcelCSVPrinter(stream);
        this.model = model;
        this.conversation = conversation;
    }

    public void setTextStringer(Stringer<String> stringer) {
        this.textStringer = stringer;
    }


    public void write() throws IOException {

        DirectedJungNetwork network = TevaUtils.createCommunityGraph(model, false, true, false);
        TevaUtils.addDrainageScoresForCommunityGraph(network);

        Map<String, Community> cmap = new HashMap<String, Community>();
        Map<String,Post> postmap = new HashMap<String,Post>();
        Map<String,List<CommunityWindow>> windowConsumers = new HashMap<String,List<CommunityWindow>>();

        for (DiscussionThread thread:conversation.getAllThreads()) {
            for (Post p:thread.getPosts()) {
                postmap.put(p.getPostid(),p);
            }
        }

        final Map<String, Integer> centrality = new HashMap<String, Integer>();
        for (Community c : model.getCommunities()) {
            cmap.put(c.getId(), c);
            windowConsumers.put(c.getId(),new ArrayList<CommunityWindow>());

        }
        for (Node n : network.getNodes()) {
            String cid = (String) n.getProperty("CommunityId");
            if (!centrality.containsKey(cid) || centrality.get(cid) < (Integer) n.getProperty("Centrality")) {
                centrality.put(cid, (Integer) n.getProperty("Centrality"));
            }

            for (Edge e:network.getInEdges(n)) {
                Node src = e.getEndpoints()[0];
                if (!cid.equals(src.getProperty("CommunityId"))) {
                    windowConsumers.get(cid).add(new CommunityWindow((String)src.getProperty("CommunityId"),(Integer)n.getProperty("Window")));
                }
            }

        }
        List<String> communities = new ArrayList();

        communities.addAll(centrality.keySet());

        Collections.sort(communities,new Comparator<String>() {
            public int compare(String s, String s2) {
                return -1*(centrality.get(s).compareTo(centrality.get(s2)));
            }
        });
        printLine(headers);

        for (String community:communities) {
            Community c = cmap.get(community);
            Set<Windowable> posts = new TreeSet<Windowable>(new Comparator<Windowable>() {
                public int compare(Windowable windowable, Windowable windowable2) {
                    return windowable.getStart().compareTo(windowable2.getStart());
                }
            });
            if (c!=null && c.getAssignments()!=null) {
                log.info("Found "+c.getAssignments().size()+" chunks for "+c.getId());
                for (ConversationChunk chunk :c.getAssignments()) {
                    posts.addAll(chunk.messages);
                    log.info("Adding "+chunk.messages.size()+" messages ");

                }
            }
            List<CommunityWindow> consumes = windowConsumers.get(c.getId());

            for (Windowable w:posts) {
                String consumesString = "--";
                if (!consumes.isEmpty()) {
                    StringBuilder builder = new StringBuilder();
                    for (CommunityWindow cw:consumes) {
                        Date[] window = model.getWindows()[cw.window];
                        if (inWindow(w.getStart(),window)) {
                            builder.append("[").append(cw.community).append("]");
                        }

                    }
                    consumesString = builder.toString();

                }
                printLine(c.getId(),w.getStart(),postmap.get(w.getId()).getThreadid(),w.getId(),postmap.get(w.getId()).getReplyToId(),consumesString,centrality.get(community),postmap.get(w.getId()).getUserid(),w.getRawData());
            }
        }
        printer.flush();
        printer.close();




    }

    private boolean inWindow(Date d, Date[] bounds) {
        return (d.equals(bounds[0]) || d.after(bounds[0])) && d.before(bounds[1]);
    }

    public void printLine(Object... r) {
        String[] result = new String[r.length];
        for (int i = 0;i<r.length;i++) {
            result[i] = getString(r[i]);
        }

        printer.println(result);

    }

    public String getString(Object o) {
        if (o == null) {
            return "";
        } else if (o instanceof String) {
           // String rep = ((String) o).replace("\"", "\\\"");
           // return "\"" + rep + "\"";
            return (textStringer!=null)?textStringer.stringify((String)o):((String)o);

        } else if (o instanceof Date) {
            return getString(DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM).format((Date) o));
        } else if (o instanceof Number) {
            return o.toString();
        } else if (o instanceof Collection) {
            return getString(flatten((Collection) o, null));
        } else return getString(o.toString());
    }

    public String flatten(Collection c, Stringer s) {
        StringBuilder builder = new StringBuilder();
        String sep = "";
        for (Object o : c) {
            builder.append(sep).append(s != null ? s.stringify(o) : o == null ? "" : o.toString());
            sep = ",";
        }
        return builder.toString();
    }




}


<|start_filename|>src/edu/mit/cci/util/ConsoleDocumentation.java<|end_filename|>
package edu.mit.cci.util;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
 * User: jintrone
 * Date: 5/16/12
 * Time: 7:53 AM
 */

@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface ConsoleDocumentation {
    public String value();
}


<|start_filename|>src/edu/mit/cci/sna/Network.java<|end_filename|>
package edu.mit.cci.sna;

import java.util.Collection;

/**
 * Simple wrapper around a generic network
 *
 * User: jintrone
 * Date: 4/19/12
 * Time: 3:46 PM
 */
public interface Network {

    public Collection<Node> getNodes();

    public Collection<Edge> getEdges();

    public boolean isDirected();

    public void remove(Edge e);

    public void remove(Node n);

    public void add(Node n);

    public void add(Edge e);

    public static interface Factory {

        public void addEdge(String[] ends, float weight);
        public Network getAdapter();
        public Network getAdapter(boolean b);
        public boolean hasData();

    }

}


<|start_filename|>src/edu/mit/cci/text/windowing/SingleThreadBinningStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
 * User: jintrone
 * Date: 5/23/11
 * Time: 5:33 PM
 */
public class SingleThreadBinningStrategy implements BinningStrategy<Windowable> {


    WindowStrategy<Windowable> windowStrategy;

    public SingleThreadBinningStrategy(List<Windowable> thread, WindowStrategy.Factory<Windowable> factory) {
        List<Windowable> data = new ArrayList<Windowable>(thread);
        Collections.sort(data, new Comparator<Windowable>() {
            public int compare(Windowable argument, Windowable argument1) {
                return argument.getStart().compareTo(argument1.getStart());
            }
        });

        windowStrategy = factory.getStrategy();
        windowStrategy.setData(data);

    }


    public List<Bin<Windowable>> getDataAtWindow(int window) {
         return Collections.singletonList((Bin<Windowable>)new BasicBin<Windowable>(windowStrategy.getWindow(window)));
    }

    public int getNumBins() {
        return 1;
    }

    public int getNumWindows() {
        return windowStrategy.getNumberWindows();
    }


}


<|start_filename|>src/edu/mit/cci/teva/cpm/cos/CosCommunityFinder.java<|end_filename|>
package edu.mit.cci.teva.cpm.cos;

import edu.mit.cci.sna.CliqueDecoratedNetwork;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.engine.CommunityFinder;
import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.teva.engine.CommunityFrame;
import edu.mit.cci.teva.engine.TevaParameters;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * User: jintrone
 * Date: 11/13/12
 * Time: 4:01 PM
 */
public class CosCommunityFinder implements CommunityFinder {

    private static Logger log = Logger.getLogger(CosCommunityFinder.class);

    private static final String COS_NET_NAME = "CosNetwork";
    private boolean overwriteNetworks = false;
    private boolean overwriteAnalysis = false;
    private File outputdir;
    private CosRunner cosRunner;

    public CosCommunityFinder(TevaParameters params) {
        this.overwriteNetworks = params.getOverwriteNetworks();
        this.overwriteAnalysis = params.getOverwriteAnalyses();
        this.outputdir = new File(params.getWorkingDirectory());
        this.cosRunner = new CosRunner(params.getCosMaxCliquesExecutable(), params.getCosExecutable(), params.getCpmParameters(), params.getWorkingDirectory());

    }


    public List<CommunityFrame> findCommunities(File networkFile, int cliqueSizeAtWindow, int window) throws CommunityFinderException {
        try {
             cosRunner.process(networkFile, overwriteAnalysis);

        } catch (IOException e) {
            throw new CommunityFinderException("Error processing network", e);
        } catch (InterruptedException e) {
            throw new CommunityFinderException("Error running cos", e);
        }

        List<CliqueDecoratedNetwork> networks = null;

        try {
            networks = CosFileReader.readCommunities(networkFile.getName(), cliqueSizeAtWindow, networkFile.getParentFile());
        } catch (CommunityFinderException ex) {
            log.error(ex);
            log.info("No communities identified; continuing");
            networks = Collections.emptyList();
        }

        List<CommunityFrame> result = new ArrayList<CommunityFrame>();
        for (CliqueDecoratedNetwork n : networks) {
            result.add(new CommunityFrame(window, n));
        }
        return result;

    }

    public List<CommunityFrame> findCommunities(Network currentGraph, int cliqueSizeAtWindow, int window, String id) throws CommunityFinderException {
        File networkFile = getInputNetworkFile(this.outputdir, window, id);
        File outputDir = getOutputDir(networkFile);
        if (!outputDir.exists()) {
            outputDir.mkdirs();
        }
        File outputFile = new File(getOutputDir(networkFile), networkFile.getName());
        if (!outputFile.exists() || overwriteNetworks) {
            try {
               // U.delete(networkFile);
                log.debug("Attempting to create file at: " + networkFile.getAbsolutePath());
                NetworkUtils.createNetworkFile(currentGraph, outputFile, false);
            } catch (IOException e) {
                throw new CommunityFinderException("Error creating network file", e);
            }

        }
        return findCommunities(outputFile, cliqueSizeAtWindow, window);
    }

    public static File getInputNetworkFile(File outputdir, int window, String id) {
        return new File(outputdir, COS_NET_NAME + "." + id + "." + window + ".net");
    }

    public static File getOutputDir(File inputfile) {
        return new File(inputfile.getAbsolutePath() + "_files");

    }
}


<|start_filename|>src/edu/mit/cci/text/preprocessing/Tokenizer.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

import java.util.List;

/**
 * Most clients will use the tokenizer to split a data stream into tokens.  The tokenizer insulates the client
 * from details about the underlying text transformation routines.
 *
 * User: jintrone
 * Date: 9/21/12
 * Time: 5:00 PM
 */
public interface Tokenizer<T> {


    public List<String> tokenize(T text);
}


<|start_filename|>src/edu/mit/cci/teva/util/CommunityWindows.java<|end_filename|>
package edu.mit.cci.teva.util;

import java.util.Collection;
import java.util.HashSet;
import java.util.Set;

/**
* User: jintrone
* Date: 5/14/13
* Time: 6:19 AM
*/
public class CommunityWindows {
    public Set<Integer> windows = new HashSet<Integer>();
    public String community = null;

    public CommunityWindows(String community, int... window) {
        this.community = community;
        for (int w : window) {
            windows.add(w);
        }
    }

    public CommunityWindows(String community, Collection<Integer> w) {
        this.community = community;
        windows.addAll(windows);

    }

    public void add(int win) {
        windows.add(win);
    }

}


<|start_filename|>src/edu/mit/cci/text/windowing/PrecomputedTimeWindowStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import edu.mit.cci.util.U;

import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;

/**
 * User: jintrone
 * Date: 9/28/12
 * Time: 9:55 PM
 */
public class PrecomputedTimeWindowStrategy implements WindowStrategy<Windowable> {

    Date[][] windows;
    List<? extends Windowable> data;

    public PrecomputedTimeWindowStrategy(Date[]... windows) {
        this.windows = windows;
    }

    public int getNumberWindows() {
        return windows.length;
    }

    public List<Windowable> getWindow(int i) {
        int start = U.binarySearch(data, windows[i][0], new Comparator<Date>() {

                    public int compare(Date post, Date post1) {
                        return post.compareTo(post1);
                    }
                }, new U.Adapter<Windowable, Date>() {

            public Date adapt(Windowable obj) {
                return obj.getStart();
            }
        });
        if (start < 0) {
            start = -1 * (++start);
        }
        if (start == data.size()) {
            return Collections.emptyList();
        } else {
            int end = start;
            while (end < data.size() && data.get(end).getStart().compareTo(windows[i][1]) <= 0) {
                end++;
            }
            return (List<Windowable>) data.subList(start, end);
        }


    }

    @Override
    public int[] getWindowIndices(int i) {
        return new int[0];
    }

    public Date[][] getWindowBoundaries() {
        return windows;
    }



    public void setData(List<? extends Windowable> data) {
        this.data = data;
    }


}


<|start_filename|>src/edu/mit/cci/teva/util/TimeWindowDiscussionThreadAdapter.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;

import java.util.*;

/**
 * User: jintrone
 * Date: 5/14/13
 * Time: 1:25 PM
 */
public class TimeWindowDiscussionThreadAdapter implements DiscussionThread {

    private final DiscussionThread thread;

    private List<Post> result = null;

    private Date start;
    private Date end;

    public TimeWindowDiscussionThreadAdapter(DiscussionThread t, Date start, Date end) {
        this.thread = t;
        this.start = start;
        this.end = end;
    }

    public List<Post> getPosts() {


        if (result == null) {
            result = new ArrayList<Post>(thread.getPosts());

            for (Iterator<Post> p =result.iterator();p.hasNext();) {
                Post post = p.next();
                if (start!=null && start.after(post.getTime())) {
                    p.remove();
                    continue;
                }
                if (end!=null && end.before(post.getTime())) {
                    p.remove();

                }
            }
        }
        return result;
    }

    public List<String> getSentiment(Post p) {
        return thread.getSentiment(p);
    }

    public String getThreadId() {
        return thread.getThreadId();
    }
}


<|start_filename|>src/edu/mit/cci/teva/cpm/cos/CosRunner.java<|end_filename|>
package edu.mit.cci.teva.cpm.cos;


import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.util.StreamGobbler;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;

/**
 * User: jintrone
 * Date: 11/13/12
 * Time: 4:17 PM
 */
public class CosRunner {

    public static final String DEFAULT_MAX_CLIQUES_EXECUTABLE = "/usr/local/bin/maximal_cliques";
    public static final String DEFAULT_COS_EXECUTABLE = "/usr/local/bin/cos";

    private static Logger log = Logger.getLogger(CosRunner.class);

    private String mCliqueExec;
    private String cosExec;
    private String outputBaseDir;
    private String cosParams = "";

    public CosRunner(String pathToMaxCliquesExecutable, String pathToCosExecutable, String outputBaseDir) {
        this.mCliqueExec = pathToMaxCliquesExecutable;
        this.cosExec = pathToCosExecutable;
        this.outputBaseDir = outputBaseDir;
    }

    public CosRunner(String pathToMaxCliquesExecutable, String pathToCosExecutable, String cosParams, String outputBaseDir) {
        this.mCliqueExec = pathToMaxCliquesExecutable;
        this.cosExec = pathToCosExecutable;
        this.outputBaseDir = outputBaseDir;
        this.cosParams = cosParams;
    }

    public String buildMCliqueCommandLine(File inputFile) {
        return String.format("%s %s", mCliqueExec, inputFile.getAbsolutePath());

    }

    public String buildCosCommandLine(File inputFile) {
        return String.format("%s %s %s", cosExec, cosParams, inputFile.getAbsolutePath());

    }


    public boolean process(File inputFile, boolean overwrite) throws IOException, InterruptedException, CommunityFinderException {
        File outputDir = inputFile.getParentFile();

        if (!new File(inputFile.getAbsolutePath() + ".mcliques").exists() || overwrite) {
            if (!generateMaxCliques(inputFile)) throw new CommunityFinderException("Problem generating max cliques");

            // move files

            boolean b = U.move(new File(inputFile.getName() + ".map"), outputDir);
            b &= U.move(new File(inputFile.getName() + ".mcliques"), outputDir);
            if (!b) {
                log.error("Error moving files");
                throw new CommunityFinderException("Could not move files to output dir: " + outputDir.getAbsolutePath());
            }
        }
        if (!new File(outputDir, "k_num_communities.txt").exists() || overwrite) {
            if (!generateCommunityAnalysis(inputFile, outputDir))
                throw new CommunityFinderException("Problem extracting communities");


            File[] files = new File(".").listFiles(new FilenameFilter() {
                public boolean accept(File file, String s) {
                    return s.endsWith("_communities.txt");
                }
            });
            boolean b = true;
            for (File f : files) {
                b &= U.move(f, outputDir);
            }
            if (!b) {
                throw new CommunityFinderException("Error moving community files");
            }
        }
        return true;


    }

    private boolean generateMaxCliques(File inputFile) throws IOException, InterruptedException {
        log.info("Generating maximal cliques for " + inputFile.getAbsolutePath());
        long current = System.currentTimeMillis();
        String cmd = buildMCliqueCommandLine(inputFile);
        log.info("Run " + cmd);
        Process p = Runtime.getRuntime().exec(cmd);

        StreamGobbler errorGobbler = new
                StreamGobbler(p.getErrorStream(), "ERROR");

        // any output?
        StreamGobbler outputGobbler = new
                StreamGobbler(p.getInputStream(), "OUTPUT");

        // kick them off
        errorGobbler.start();
        outputGobbler.start();

        // any error???
        int exitVal = p.waitFor();
        log.debug("ExitValue: " + exitVal);

        if (log.isDebugEnabled()) {
            BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
            String s = null;
            while ((s = r.readLine()) != null) {
                System.err.println(s);
            }
        }

        log.info("Finished in " + (System.currentTimeMillis() - current) / 1000l + " seconds.");


        return true;
    }

    private boolean generateCommunityAnalysis(File inputFile, File outputdir) throws IOException, InterruptedException {
        log.info("Generating communities for " + inputFile.getName());
        long current = System.currentTimeMillis();

        File mcliques = new File(outputdir, inputFile.getName() + ".mcliques");
        String cmd = buildCosCommandLine(mcliques);
        log.info("Execute: " + cmd);
        Process p = Runtime.getRuntime().exec(cmd);
        StreamGobbler errorGobbler = new
                StreamGobbler(p.getErrorStream(), "ERROR");

        // any output?
        StreamGobbler outputGobbler = new
                StreamGobbler(p.getInputStream(), "OUTPUT");

        // kick them off
        errorGobbler.start();
        outputGobbler.start();

        // any error???
        int exitVal = p.waitFor();
        log.debug("ExitValue: " + exitVal);

        if (log.isDebugEnabled()) {
            BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
            String s = null;
            while ((s = r.readLine()) != null) {
                System.err.println(s);
            }
        }

        log.info("Finished in " + (System.currentTimeMillis() - current) / 1000l + " seconds.");


        return true;
    }

    private File checkExists(File inputFile, String s) {
        File f = new File(inputFile.getParentFile(), inputFile.getName() + s);
        if (f.exists()) return f;
        else return null;
    }


}


<|start_filename|>src/edu/mit/cci/teva/util/ExhaustiveAssignment.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.text.windowing.Bin;
import edu.mit.cci.text.windowing.Windowable;
import org.apache.log4j.Logger;

import java.util.*;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 8/30/13
 * Time: 1:58 PM
 * To change this template use File | Settings | File Templates.
 */
public class ExhaustiveAssignment implements CommunityMembershipStrategy {

    Logger log = Logger.getLogger(ExhaustiveAssignment.class);

    @Override
    public List<Assignment> assignToCommunity(CommunityModel communities, int window, Network net) {
        List<Assignment> scores = new ArrayList<>();
        for (Community c : communities.getCommunities()) {
            EdgesCoverage e = coverage(net, c.getCommunityAtBin(window));
            float similarity = NetworkUtils.similarity(net, c.getCommunityAtBin(window));


            //TODO handle the unlikely possibility there could be more than one match here.

            if (e.coverage > 0) {
                scores.add(new Assignment(c, e.coverage, similarity,e.edges));
            }
        }

        return scores;
    }

    private  EdgesCoverage coverage(Network one, Network two) {
        if (one == null || two == null || one.getNodes().isEmpty() || two.getNodes().isEmpty()) {
            return new EdgesCoverage(Collections.<Edge>emptySet(),0f);
        } else {
            Set<Edge> fromedges = new HashSet<Edge>(one.getEdges());
            float denom = fromedges.size();
            fromedges.retainAll(two.getEdges());

            return new EdgesCoverage(fromedges,(float) fromedges.size() / denom);
        }

    }

    private class EdgesCoverage {

        private final float coverage;
        private final Collection<Edge> edges;

        EdgesCoverage(Collection<Edge> edges, float coverage) {
            this.edges = edges;
            this.coverage = coverage;
        }
    }


}


<|start_filename|>src/edu/mit/cci/sna/Edge.java<|end_filename|>
package edu.mit.cci.sna;

import edu.mit.cci.sna.impl.EdgeImpl;

import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.Map;

/**
 * Simple interface around a generic edge
 *
 * User: jintrone
 * Date: 9/21/12
 * Time: 3:54 PM
 */

@XmlJavaTypeAdapter(EdgeImpl.JaxbAdapter.class)
public interface Edge {

    public Node[] getEndpoints();

    public boolean isDirected();

    public float getWeight();

    public void setWeight(float weight);

    public void setProperty(String property, Object value);

    public Object getProperty(String property);

    public Map<String,Object> getProperties();
}


<|start_filename|>src/edu/mit/cci/teva/engine/TopicMembershipEngine.java<|end_filename|>
package edu.mit.cci.teva.engine;

import java.io.IOException;

/**
 * User: jintrone
 * Date: 5/9/13
 * Time: 11:22 PM
 */
public interface TopicMembershipEngine {

    public void process() throws IOException;
}


<|start_filename|>src/edu/mit/cci/teva/analysis/DumpTopicsToCsv.java<|end_filename|>
package edu.mit.cci.teva.analysis;

import edu.mit.cci.sna.Node;
import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.util.Stringer;
import edu.mit.cci.teva.util.TevaUtils;

import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;

/**
 * User: jintrone
 * Date: 10/12/12
 * Time: 9:18 AM
 */
public class DumpTopicsToCsv {

    public  String[] headers = new String[]{"From", "To", "TopicID", "Short", "Long"};


    protected PrintWriter writer;
    protected CommunityModel model;

    public DumpTopicsToCsv(CommunityModel model, OutputStream stream) {
        writer = new PrintWriter(new OutputStreamWriter(stream));
        this.model = model;
    }


    public void write() {
        Date[][] wins = model.getWindows();
        printLine(headers);
        for (int i = 0; i < wins.length; i++) {
            for (Community c : model.getCommunities()) {
               Collection<String> s = c.getCommunityAtBin(i) == null ? Collections.<String>emptyList() : TevaUtils.getCommunityRepresentation(c.getCommunityAtBin(i), 5);
                String l = flatten(c.getCommunityAtBin(i) == null ? Collections.<Node>emptyList() : c.getCommunityAtBin(i).getNodes(),new Stringer<Node>() {

                    public String stringify(Node object) {
                        return object.getLabel();
                    }
                });
                printLine(wins[i][0], wins[i][1], c.getId(), s, l);

            }
        }
    }

    public void printLine(Object... r) {
        StringBuilder builder = new StringBuilder();
        String sep = "";
        for (Object o:r) {
            builder.append(sep).append(getString(o));
            sep = ",";
        }
        writer.println(builder.toString());

    }

    public String getString(Object o) {
        if (o == null) {
            return "";
        } else if (o instanceof String) {
            String rep = ((String) o).replace("\"", "\\\"");
            return "\""+rep+"\"";
        } else if (o instanceof Date) {
            return getString(DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.MEDIUM).format((Date) o));
        } else if (o instanceof Number) {
            return o.toString();
        } else if (o instanceof Collection) {
            return getString(flatten((Collection)o,null));
        } else return getString(o.toString());
    }

    public String flatten(Collection c, Stringer s) {
        StringBuilder builder = new StringBuilder();
        String sep = "";
        for (Object o:c) {
            builder.append(sep).append(s!=null?s.stringify(o):o==null?"":o.toString());
            sep = ",";
        }
        return builder.toString();
    }


}


<|start_filename|>src/edu/mit/cci/util/ConsoleBase.java<|end_filename|>
package edu.mit.cci.util;

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Map;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * User: jintrone
 * Date: 7/13/12
 * Time: 9:09 AM
 */
public class ConsoleBase {
    protected Scanner s;
    Map<Integer, MethodRunner> methodRunnerMap = new HashMap<Integer, MethodRunner>();
    String docstring = "";


    public void start() {

        s = new Scanner(System.in);
        int i = 1;
        for (Method m : getClass().getMethods()) {
            if (m.getName().startsWith("run")) {
                ConsoleDocumentation a = m.getAnnotation(ConsoleDocumentation.class);
                methodRunnerMap.put(i++, new MethodRunner(m, a.value()));
            }

        }
        Method e = null;
        try {
            e = getClass().getMethod("exit");

        } catch (NoSuchMethodException ex) {
            //this shouldn't happen
            throw (new RuntimeException("Couldn't locate exit method"));
        }


        methodRunnerMap.put(i, new MethodRunner(e, e.getAnnotation(ConsoleDocumentation.class).value()));
        buildDocString();
        waitForInput();
    }


    public void buildDocString() {
        for (int i = 1; i <= methodRunnerMap.size(); i++) {
            docstring += "\n[" + i + "] " + methodRunnerMap.get(i).documentation();

        }


    }

    public String docstring() {
        return docstring;
    }

    @ConsoleDocumentation(value = "Exit.")
    public void exit() {
        System.out.println("Bye!");
        System.exit(1);
    }

    public void handleError(Throwable e) {
        System.out.println("Encountered error while running task:" + e.getMessage());
        e.printStackTrace(System.out);
        System.out.println("\n\n");
    }

    public void waitForInput() {
        Pattern p = Pattern.compile(".*?(\\d+).*$");

        System.out.println(docstring());
        while (s.hasNext()) {
            String l = s.nextLine();
            Matcher m = p.matcher(l);

            if (m.matches()) {
                int i = Integer.parseInt(m.group(1));

                MethodRunner runner = methodRunnerMap.get(i);
                if (runner != null) {
                    try {
                        runner.run();
                    } catch (Throwable e) {
                        handleError(e);
                    }

                }
            } else {
                System.out.println("Unrecognized");
            }

            System.out.println(docstring());
        }
    }

    public String askUser(String question) {
        System.out.print(question + " ");
        return s.nextLine();
    }

    public String askUser(String question, String def) {
        System.out.print(question + " ["+def+"]");
        String result = s.nextLine();
        if (result.trim().isEmpty()) {
            return def;
        } else {
            return result;
        }
    }

    protected class MethodRunner {


        private Method m;
        private String doc;

        public MethodRunner(Method m, String doc) {
            this.m = m;
            this.doc = doc;
        }

        public String documentation() {
            return doc;
        }

        public void run() throws InvocationTargetException, IllegalAccessException {
            m.invoke(ConsoleBase.this);
        }
    }
}


<|start_filename|>src/edu/mit/cci/sna/impl/NodeImpl.java<|end_filename|>
package edu.mit.cci.sna.impl;

import edu.mit.cci.sna.Node;
import org.apache.commons.collections15.Factory;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.HashMap;
import java.util.Map;

/**
* User: jintrone
* Date: 4/20/11
* Time: 2:25 PM
*/

@XmlRootElement(name="Node")
@XmlAccessorType(XmlAccessType.NONE)
public class NodeImpl implements Node {

    private static int idgen = 0;

    private Map<String,Object> attributes = new HashMap<String,Object>();


    private String label;


    private NodeImpl() {

    }

     public NodeImpl(String label) {
         setLabel(label);

    }

    //TODO this needs to be backed out of the api - id's should be invariant
    public void setLabel(String label) {
        this.label = label;
    }
    @XmlAttribute(name = "id")
    public String getLabel() {
        return label;
    }

    public String getId() {
        return label;
    }



    public static Factory<NodeImpl> getFactory() {
        return new Factory<NodeImpl>() {

            public NodeImpl create() {
                return new NodeImpl(""+idgen++);
            }
        };
    }

    public void setProperty(String key, Object val) {
        attributes.put(key,val);
    }


    public Map<String,Object> getProperties() {
        return attributes;
    }

    public Object getProperty(String key) {
       return attributes.get(key);
    }


    public String toString() {
        return "N("+getId()+")";
    }

    public int hashCode() {
        return label.hashCode()*7+13;
    }

    public boolean equals(Object o) {
        return o instanceof Node && ((Node)o).getId().equals(getId());
    }

    public static class JaxbAdapter extends XmlAdapter<NodeImpl,Node> {

        @Override
        public Node unmarshal(NodeImpl node) throws Exception {
            return node;
        }

        @Override
        public NodeImpl marshal(Node node) throws Exception {
            //TODO generalize this
            return (NodeImpl)node;
        }
    }



}


<|start_filename|>src/edu/mit/cci/text/windowing/ReplyBasedBinningStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import edu.mit.cci.teva.util.WindowablePostAdapter;
import edu.mit.cci.util.F;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

/**
 * User: jintrone
 * Date: 4/17/13
 * Time: 3:31 PM
 */
public class ReplyBasedBinningStrategy implements BinningStrategy<Windowable> {

    int maxwin = -1;
    List<List<Bin<Windowable>>> binmodel = new ArrayList<List<Bin<Windowable>>>();
    private static Logger logger = Logger.getLogger(ReplyBasedBinningStrategy.class);


    public ReplyBasedBinningStrategy(List<List<WindowablePostAdapter>> info, WindowStrategy.Factory<Windowable> factory) {
        WindowStrategy<Windowable> strategy = factory.getStrategy();
        final List<WindowablePostAdapter> posts = new ArrayList<WindowablePostAdapter>();


        U.map(info).forEach(new F<List<WindowablePostAdapter>>() {
            public void apply(List<WindowablePostAdapter> windowablePostAdapters) {
                posts.addAll(windowablePostAdapters);
            }
        });

        Collections.sort(posts, new Comparator<Windowable>() {
            public int compare(Windowable post1, Windowable post2) {
                return post1.getStart().compareTo(post2.getStart());
            }
        });

        strategy.setData(posts);
        maxwin = Math.max(strategy.getNumberWindows(), maxwin);


        for (int i = 0; i < maxwin; i++)

        {
            List<Bin<Windowable>> chunks = new ArrayList<Bin<Windowable>>();
            chunks.addAll(analyzeChunk(strategy.getWindow(i)));
            binmodel.add(chunks);

        }

        //debugging
        logger.info("There are "+maxwin+" windows");
        for (int i=0;i<maxwin;i++) {
            List<Bin<Windowable>> bin = getDataAtWindow(i);
            logger.info(i+". "+bin.size());
            final Date[] boundaries = strategy.getWindowBoundaries()[i];
            U.map(bin).forEach(new F<Bin<Windowable>>() {
                public void apply(Bin<Windowable> windowables) {
                    U.map(windowables).forEach(new F<Windowable>() {
                        public void apply(Windowable windowable) {
                            if (windowable.getStart().before(boundaries[0]) ||
                                    windowable.getStart().after(boundaries[1])) {
                                logger.warn("----> Messages outside bin boundaries?");
                            }
                        }
                    });
                }
            });

        }


    }

    protected Collection<Bin<Windowable>> analyzeChunk(List<Windowable> chunk) {
        final Map<String, Bin<Windowable>> data = new HashMap<String, Bin<Windowable>>();
        final Map<String, WindowablePostAdapter> lookup = new HashMap<String, WindowablePostAdapter>();
        U.map(chunk).forEach(new F<Windowable>() {
            public void apply(Windowable p) {
                WindowablePostAdapter pa = (WindowablePostAdapter) p;
                lookup.put(p.getId(), pa);
                if (!data.containsKey(pa.getPost().getReplyToId())) {
                    Bin<Windowable> result = new BasicBin<Windowable>();
                    data.put(pa.getId(), result);
                    result.add(pa);

                } else {
                    Bin<Windowable> existing = data.get(pa.getPost().getReplyToId());

                    if (existing.get(existing.size() - 1).getId().equals(pa.getPost().getReplyToId())) {
                        existing.add(pa);
                        data.put(pa.getId(), existing);
                    } else {
                        Bin<Windowable> bin = new BasicBin<Windowable>();
                        data.put(pa.getId(), bin);
                        bin.add(lookup.get(pa.getPost().getReplyToId()));
                        bin.add(pa);
                        bin.setFirstItemIndex(1);
                    }


                }
            }
        });
        return new HashSet<Bin<Windowable>>(data.values());

    }


    public List<Bin<Windowable>> getDataAtWindow(int window) {
        return binmodel.get(window);
    }




    public int getNumWindows() {
        return maxwin;
    }

}


<|start_filename|>src/edu/mit/cci/teva/FileBasedRunner.java<|end_filename|>
package edu.mit.cci.teva;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.cpm.cfinder.CFinderCommunityFinder;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.util.NetWriterStrategy;
import edu.mit.cci.teva.util.TevaUtils;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.WindowStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.CorpusToNetworkGenerator;
import edu.mit.cci.text.wordij.TextCorpusNetworkGenerator;
import org.apache.log4j.Logger;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/3/12
 * Time: 9:39 PM
 */
public abstract class FileBasedRunner implements TevaRunner{


    Logger log = Logger.getLogger(FileBasedRunner.class);
    TevaParameters params;
    Conversation conversation;
    private WindowStrategy.Factory<Windowable> windowStrategyFactory;
    private TevaFactory factory;


    public FileBasedRunner(Conversation c) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters();
        this.factory = new DefaultTevaFactory(params,conversation);
        //process();
    }

    public FileBasedRunner(Conversation c, InputStream parameters) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters(parameters);
        this.factory = new DefaultTevaFactory(params,conversation);
        //process();
    }

     public FileBasedRunner(Conversation c, InputStream parameters, TevaFactory factory) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters(parameters);
        this.factory =factory;
        //process();
    }

    public FileBasedRunner(Conversation c, TevaParameters parameters, TevaFactory factory) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = parameters;
        this.factory =factory;
        //process();
    }




    public void createNetworkFiles() throws IOException {
        NetWriterStrategy writer = getNetworkWriter();
        BinningStrategy<Windowable> binningStrategy =  factory.getTopicBinningStrategy(factory.getConversationData(), factory.getTopicWindowingFactory());
        CorpusToNetworkGenerator<Windowable> networkGenerator = factory.getNetworkGenerator(binningStrategy);
        networkGenerator.analyzeToFiles(writer);
        log.info("Done creating networks.");
    }

    public NetworkProvider getNetworkProvider() throws IOException {

        NetworkProvider provider;
        File f = new File(params.getWorkingDirectory());
        if (!f.exists()) {
            f.mkdirs();
        }

        final NetWriterStrategy writer = getNetworkWriter();

        if (!params.getSkipNetworkGeneration()) {
           createNetworkFiles();

        }
        WindowStrategy.Factory<Windowable> winFactory = factory.getTopicWindowingFactory();
        final int i = winFactory.getStrategy().getNumberWindows();
        provider = new NetworkProvider() {
            public int getNumberWindows() {
                return i;
            }

            public Network getNetworkAt(int i) {
                File f =  writer.getNetworkFilename(i);
                        //CFinderCommunityFinder.getInputNetworkFile(new File(params.getWorkingDirectory()),i,params.getFilenameIdentifier());
                try {
                    return NetworkUtils.readNetworkFile(f);
                } catch (IOException e) {
                    log.error("Error reading network file "+f.getAbsolutePath()+", cannot proceed");
                    throw new RuntimeException(e);
                }
            }
        };

        return provider;
    }

    public abstract NetWriterStrategy getNetworkWriter();

    public void processAndSave() throws IOException, CommunityFinderException, JAXBException {
        log.info("Begin process.");
        NetworkProvider provider = getNetworkProvider();
        CommunityModel model = new CommunityModel(params,factory.getTopicWindowingFactory().getStrategy().getWindowBoundaries(),conversation.getName());
        EvolutionEngine engine = new EvolutionEngine(model,params, provider, factory.getFinder(), factory.getStepper(model), factory.getMerger());
        engine.process();
        TopicMembershipEngine membershipEngine = factory.getMembershipEngine(model,conversation);
        membershipEngine.process();
        TevaUtils.serialize(new File(params.getWorkingDirectory() + "/CommunityOutput." + conversation.getName() + "." + params.getFilenameIdentifier() + ".xml"), model, CommunityModel.class);

    }


}


<|start_filename|>src/edu/mit/cci/text/windowing/BinningStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.List;

/**
 * User: jintrone
 * Date: 5/5/11
 * Time: 5:32 PM
 **/


/**
 * A "bin" is just a bin of objects in a temporal data stream.  Multiple bins may exist at each window.
 *
 * A bin has no specific clas; in this API, it is merely a collection
 *
 * Consider a matrix of data, where column index reflects temporal index, and rows represent parallel temporal processes.
 * Then, windows indicate columns, and bins indicate rows.
 *
 */
public interface BinningStrategy<T> {

    public List<Bin<T>> getDataAtWindow(int window);
    public int getNumWindows();

}


<|start_filename|>src/edu/mit/cci/util/ThreadPoolRunner.java<|end_filename|>
package edu.mit.cci.util;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class ThreadPoolRunner {
    public static final long serialVersionUID = 1L;

    // Our simulation queue and task pool
    private BlockingQueue<Runnable> tasks = null;
    private ThreadPoolExecutor threadPool = null;
    private int activeJobs = 0;

    private final Lock lock = new ReentrantLock();

    /**
     * The size of the queue of runnables will be this many times larger than
     * the number of threads.
     */
    private static final int queueSizeMultiplier = 10;

    public ThreadPoolRunner(int threads) {
        if (tasks == null) {

            tasks = new ArrayBlockingQueue<Runnable>(threads
                    * queueSizeMultiplier);

            threadPool = new ThreadPoolExecutor(threads, threads,
                    Long.MAX_VALUE, TimeUnit.NANOSECONDS, tasks);

        }
    }


    public synchronized void runModel(Runnable model) {
        // Wait for space to open up in the queue
        while (tasks.remainingCapacity() < 1) {
            try {
                Thread.sleep(10);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }

        ThreadPoolHelper helper = new ThreadPoolHelper(model);

        threadPool.submit(helper);

    }

    public void finish() {
		/*
		 * Check every 0.1 seconds to see if the task queue is empty. Print
		 * status messages for debug purposes.
		 */

        while (activeJobs != 0) {
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
            }
        }
        System.out.println("All Simulations Finished.");
        threadPool.shutdown();
        threadPool = null;

    }

    private class ThreadPoolHelper implements Runnable {

        Runnable model;

        ThreadPoolHelper(final Runnable model) {
            this.model = model;
        }

        @Override
        public void run() {
            lock.lock();
            activeJobs++;
            lock.unlock();

            model.run();

            lock.lock();
            activeJobs--;
            lock.unlock();
        }

    }

}

<|start_filename|>src/edu/mit/cci/text/preprocessing/AlphaNumericTokenizer.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * User: jintrone
 * Date: 9/21/12
 * Time: 5:02 PM
 */
public class AlphaNumericTokenizer implements Tokenizer<String> {

    private static Logger log = Logger.getLogger(AlphaNumericTokenizer.class);
    Munger munger;


    public AlphaNumericTokenizer(Munger... mungers) {

        this.munger = new CompositeMunger(mungers);
    }

    public AlphaNumericTokenizer() {
        this.munger = null;
    }

    /**
     * Default implementation remove quote characters, replaces all other punctuation with spaces, and
     * compresses multiple spaces to single spaces
     * @param input
     * @return
     */
    protected String replace(String input) {
        String data = input.replaceAll("'", "");
        data = data.replaceAll("[^\\p{Alnum}]+", " ");
        data = data.replaceAll("\\s+", " ");
        data = data.replaceAll("\\s\\p{Alnum}\\s"," ");
        return data;
    }

    public List<String> tokenize(String input) {
        String data = replace(input);

        String[] words = data.split("\\s+");
        List<String> tokens = new ArrayList<String>();

        if (munger == null) {
            tokens.addAll(Arrays.asList(words));
        } else {
            log.debug("Input: "+Arrays.asList(words));
            for (String w : words) {

                w = w.trim();
                if (w.isEmpty()) continue;
                log.debug("Chew: " + w.toLowerCase());
                if (munger.read(w.toLowerCase())) {
                    List<String> spit = munger.flush();
                    log.debug("PTtooey! " + spit);
                    tokens.addAll(spit);
                }
            }
            List<String> l = munger.finish();
            log.debug("Finishing: " + l);
            tokens.addAll(l);
        }
        log.debug("Output: "+tokens);
        return tokens;
    }


}


<|start_filename|>src/edu/mit/cci/teva/model/Conversation.java<|end_filename|>
package edu.mit.cci.teva.model;

import java.util.List;

/**
 * User: jintrone
 * Date: 8/11/11
 * Time: 11:06 AM
 */
public interface Conversation {

    public String getName();
    public Post getFirstPost();
    public Post getLastPost();
    public List<DiscussionThread> getAllThreads();
    public List<String> getUsers();

}


<|start_filename|>src/edu/mit/cci/text/preprocessing/CompositeMunger.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/19/11
 * Time: 5:06 PM
 * <p/>
 * //TODO CLASS IS INCOMPLETE - will fail to flush down the whole chain if less than n-1 mungers have fired by last input
 */
public class CompositeMunger implements Munger {


    public Munger[] mungers;

    Logger log = Logger.getLogger(CompositeMunger.class);

    public CompositeMunger(Munger... mungers) {
        this.mungers = mungers;
    }


    public boolean read(String from) {

        List<String> input = new ArrayList<String>();
        input.add(from);
        return _munge(input, 0);
    }

    public boolean _munge(List<String> words, int munger) {
        log.debug("Munging "+words+" at level "+munger);
        Munger current = mungers[munger];
        boolean result = false;
        while (!words.isEmpty()) {
            String next = words.remove(0);
            if (current.read(next)) {
                result = true;
                if (munger < mungers.length -1) {
                    result = _munge(current.flush(),munger+1);
                }
            } else {
                result = false;
            }
        }
        return result;


    }

    public List<String> flush() {
        return mungers[mungers.length-1].flush();
    }

    public List<String> finish() {
        List<String> result = new ArrayList<String>();
        for (Munger m:mungers) {
            List<String> tmp = new ArrayList<String>();
            if (!result.isEmpty()) {
                for (String r:result) {
                    m.read(r);
                }
                tmp.addAll(m.flush());
            }
            tmp.addAll(m.finish());
            result.clear();
            result.addAll(tmp);
        }
        return result;
    }
}


<|start_filename|>src/edu/mit/cci/util/GeneralLauncher.java<|end_filename|>
package edu.mit.cci.util;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
 * Command line utility to launch a class by calling it's constructor with a properties file
 *
 * User: jintrone
 * Date: 7/27/12
 * Time: 12:10 PM
 */
public class GeneralLauncher {

    public static void main(String[] args) throws ClassNotFoundException, IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException, InstantiationException {

        Map<String,String> params = new HashMap<String,String>();
        for (int i = 0;i<(args.length/2) * 2;) {
          params.put(args[i],args[++i]);
            i++;
        }

        Class target = null;
        if (!params.containsKey("-c")) {
            exit("Must specify class file to launch with -c");

        } else {
            target = Class.forName(params.remove("-c"));
        }


        Properties p = new Properties();

        if (params.containsKey("-p")) {
            File f= new File(params.remove("-p"));
            if (!f.exists()) {
              exit("Properties file "+f.getAbsolutePath()+" could not be found");
            } else {
                p.load(new FileReader(f));
            }

        } else {
           for (Map.Entry<String,String> ent:params.entrySet()) {
               if (ent.getKey().startsWith("-")) {
                   p.setProperty(ent.getKey().substring(1),ent.getValue());
               } else {
                   p.setProperty(ent.getKey(),ent.getValue());
               }
           }

        }

        Constructor c = target.getDeclaredConstructor(Properties.class);
        c.newInstance(p);

    }

    public static void exit(String msg) {
        System.err.println(msg);
        System.exit(-1);

    }

}


<|start_filename|>src/edu/mit/cci/text/windowing/BasicWindowable.java<|end_filename|>
package edu.mit.cci.text.windowing;

import edu.mit.cci.text.preprocessing.Tokenizer;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlID;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.Date;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/14/12
 * Time: 11:07 PM
 */

@XmlRootElement(name = "message")
@XmlAccessorType(XmlAccessType.NONE)
public class BasicWindowable implements Windowable {

    @XmlAttribute
    @XmlID
    public String id;

   // @XmlElement
    public String content;

    @XmlAttribute
    private Date start;


    private Tokenizer<String> tokenizer;


    private BasicWindowable() {

    }

    public BasicWindowable(String id, String rawData, Date start) {
        this.start = start;
        this.id = id;
        this.content = rawData;
    }


    public Date getStart() {
        return start;
    }

    public void setTokenizer(Tokenizer<String> t) {
        this.tokenizer = t;
    }

    public List<String> getTokens() {
        return tokenizer.tokenize(content);
    }

    public String getRawData() {
        return content;
    }

    public String getId() {
        return id;
    }

    public boolean equals(Object w) {
       return (w instanceof Windowable && ((Windowable)w).getId().equals(getId()) && ((Windowable)w).getStart().equals(getStart()));

    }

    public int hashCode() {
        return (getId().hashCode()+getStart().hashCode())*13+27;
    }



    public static class JaxbAdapter extends XmlAdapter<BasicWindowable, Windowable> {
        @Override
        public Windowable unmarshal(BasicWindowable w) throws Exception {
            return w;
        }

        @Override
        public BasicWindowable marshal(Windowable windowable) throws Exception {
            return new BasicWindowable(windowable.getId(), windowable.getRawData(), windowable.getStart());
        }
    }
}


<|start_filename|>src/edu/mit/cci/sna/NetworkFactory.java<|end_filename|>
package edu.mit.cci.sna;

/**
 * User: jintrone
 * Date: 9/21/12
 * Time: 8:54 PM
 */
public interface NetworkFactory {

    public Edge addEdge(String from, String to, boolean directed);
    public Network getNetwork();
}


<|start_filename|>src/edu/mit/cci/teva/example/RunSampleCsv.java<|end_filename|>
package edu.mit.cci.teva.example;

import edu.mit.cci.adapters.csv.CsvBasedConversation;
import edu.mit.cci.teva.DefaultTevaFactory;
import edu.mit.cci.teva.MemoryBasedRunner;
import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.teva.engine.TevaParameters;
import edu.mit.cci.teva.model.Conversation;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.HashMap;
import java.util.Map;

/**
 * User: jintrone
 * Date: 12/12/12
 * Time: 11:17 PM
 */
public class RunSampleCsv {

    public static void main(String args[]) throws IOException, ParseException, JAXBException, CommunityFinderException {

        System.err.println(new File(".").getAbsoluteFile());
        Map<String,String> params = getParams(args);
        InputStream props = null;
        InputStream datafile = null;
        if (params.containsKey("p")) {
            props = new FileInputStream(params.get("p"));
        } else {

        }
        String corpus = "Demo";
        if (params.containsKey("f")) {
            File f = new File(params.get("f"));
            datafile = new FileInputStream(f);
            corpus = params.get(f.getName());
        } else {
            datafile = ClassLoader.getSystemClassLoader().getResourceAsStream("sampledata/MM15.csv");
        }
        Conversation c = new CsvBasedConversation(corpus,datafile);


        TevaParameters tevaParams = new TevaParameters(props);
        new MemoryBasedRunner(c,tevaParams,new DefaultTevaFactory(tevaParams,c)).processAndSave();
    }

    public static void usage() {
        String usage = "USAGE:  RunSampleCsv [-p<properties_file>] [-f<input_csv>]";
        System.err.println(usage);
    }

    public static Map<String,String> getParams(String[] args) {
        Map<String,String> result = new HashMap<String,String>();


        for (String s:args) {

            if (s.startsWith("-p")) {
                result.put("p",s.substring(2));
            } else if (s.startsWith("-f")) {
                result.put("f",s.substring(2));

            } else if (s.startsWith("-help") || "-?".equals(s)) {
                usage();
                System.exit(1);
            } else if (s.startsWith("-")) {
                System.out.println("Unknown option: "+s);
            }

        }
        return result;
    }

}


<|start_filename|>src/edu/mit/cci/teva/serialization/MapJaxbAdapter.java<|end_filename|>
package edu.mit.cci.teva.serialization;

import edu.mit.cci.teva.engine.CommunityModel;

import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/9/12
 * Time: 11:28 AM
 */

@XmlRootElement(name = "map")
public class MapJaxbAdapter extends XmlAdapter<MapJaxbAdapter,Map<Integer,Set<CommunityModel.Connection>>> {


    @XmlElementWrapper(name = "data")
    List<IntConnectionPair> map = new ArrayList<IntConnectionPair>();

    public MapJaxbAdapter() {}

    public MapJaxbAdapter(Map<Integer,Set<CommunityModel.Connection>> kvMap) {
      for (Map.Entry<Integer,Set<CommunityModel.Connection>> ent:kvMap.entrySet()) {
          map.add(new IntConnectionPair(ent.getKey(),ent.getValue()));
      }
    }

    @Override
    public Map<Integer,Set<CommunityModel.Connection>> unmarshal(MapJaxbAdapter kvHashMap) throws Exception {
        return kvHashMap.toMap();
    }

    private Map<Integer,Set<CommunityModel.Connection>> toMap() {
        Map<Integer,Set<CommunityModel.Connection>> result = new HashMap<Integer,Set<CommunityModel.Connection>>();
        for (IntConnectionPair ent:map) {
            result.put(ent.key,new HashSet<CommunityModel.Connection>(ent.val));
        }
        return result;
    }

    @Override
    public MapJaxbAdapter marshal(Map<Integer, Set<CommunityModel.Connection>> kvMap) throws Exception {
        return new MapJaxbAdapter(kvMap);
    }

    @XmlRootElement(name = "pair")
    public static class IntConnectionPair {

        @XmlElement
        Integer key;
        @XmlElementWrapper(name = "targets")
        List<CommunityModel.Connection> val;

        public IntConnectionPair(Integer key, Set<CommunityModel.Connection> val) {
            this.key = key;
            this.val = new ArrayList<CommunityModel.Connection>(val);
        }

        public IntConnectionPair() {}
    }
}


<|start_filename|>src/edu/mit/cci/sna/jung/MyGraphMLWriter.java<|end_filename|>
package edu.mit.cci.sna.jung;

import edu.uci.ics.jung.io.GraphMLMetadata;
import edu.uci.ics.jung.io.GraphMLWriter;
import org.apache.commons.collections15.Transformer;

import java.io.BufferedWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

/**
 * User: jintrone
 * Date: 5/17/12
 * Time: 8:09 AM
 */
public class MyGraphMLWriter<V,E> extends GraphMLWriter<V,E> {

    Map<String,String> nodeAttributeClasses = new HashMap<String, String>();
    Map<String,String> edgeAttributeClasses = new HashMap<String,String>();

    protected void writeKeySpecification(String key, String type,
			GraphMLMetadata<?> ds, BufferedWriter bw) throws IOException
	{

        bw.write("<key attr.name=\""+key+"\" attr.type=\""+ (type.equalsIgnoreCase("node")?nodeAttributeClasses.get(key):edgeAttributeClasses.get(key))+"\" id=\"" + key + "\" for=\"" + type + "\"");
		boolean closed = false;
		// write out description if any
		String desc = ds.description;
		if (desc != null)
		{
			if (!closed)
			{
				bw.write(">\n");
				closed = true;
			}
			bw.write("<desc>" + desc + "</desc>\n");
		}
		// write out default if any
		Object def = ds.default_value;
		if (def != null)
		{
			if (!closed)
			{
				bw.write(">\n");
				closed = true;
			}
			bw.write("<default>" + def.toString() + "</default>\n");
		}
		if (!closed)
		    bw.write("/>\n");
		else
		    bw.write("</key>\n");
	}

    public void addVertexData(String id, String description, String clazz, String default_value,
			Transformer<V, String> vertex_transformer)
	{
		addVertexData(id,description,default_value,vertex_transformer);
        this.nodeAttributeClasses.put(id, clazz);
	}

    public void addEdgeData(String id, String description, String clazz, String default_value,
                              Transformer<E, String> vertex_transformer)
    {
        addEdgeData(id, description, default_value, vertex_transformer);
        this.edgeAttributeClasses.put(id, clazz);
    }

}


<|start_filename|>src/edu/mit/cci/teva/util/SocialCommunityFinder.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.TevaFactory;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.text.windowing.*;
import edu.uci.ics.jung.graph.Graph;
import org.apache.log4j.Logger;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 8/22/13
 * Time: 1:29 PM
 * To change this template use File | Settings | File Templates.
 */
public class SocialCommunityFinder {



    private final TevaParameters params;
    private final TevaFactory factory;
    private WindowStrategy<Windowable> strategy;
    private Map<String,String> post2author = new HashMap<>();
    private Conversation conversation;


    Logger log = Logger.getLogger(SocialCommunityFinder.class);

    public SocialCommunityFinder(Conversation c, TevaParameters params, TevaFactory factory) {
         this.params = params;
        this.factory = factory;
        this.strategy = factory.getTopicWindowingFactory().getStrategy();
        this.conversation = c;
    }

    public void analyze() throws CommunityFinderException, IOException, JAXBException {
       List<Windowable> allposts = new ArrayList<Windowable>();
        for (List<Windowable> w:factory.getConversationData()) {
            allposts.addAll(w);
        }
        Collections.sort(allposts,new Comparator<Windowable>() {
            @Override
            public int compare(Windowable o1, Windowable o2) {
                return o1.getStart().compareTo(o2.getStart());
            }
        });
        strategy.setData((List<? extends Windowable>) allposts);

        log.info("Will process " + strategy.getNumberWindows() + " windows");
        final List<Network> results = new ArrayList<>();
        for (int win = 0; win < strategy.getNumberWindows(); win++) {
            log.debug("Process window " + win);

            results.add(analyzeWindow(win));
        }
        NetworkProvider provider = new NetworkProvider() {
            @Override
            public int getNumberWindows() {
                return results.size();
            }

            @Override
            public Network getNetworkAt(int i) {
               return results.get(i);
            }
        };


        log.info("Done processing windows");
        CommunityModel model = new CommunityModel(params,factory.getTopicWindowingFactory().getStrategy().getWindowBoundaries(),conversation.getName());
        EvolutionEngine engine = new EvolutionEngine(model,params, provider, factory.getFinder(), factory.getStepper(model), factory.getMerger());
        engine.process();
        TevaUtils.serialize(new File(params.getWorkingDirectory()+"/SocialOutput."+conversation.getName()+"."+params.getFilenameIdentifier()+".xml"), model, CommunityModel.class);
    }

    public Network analyzeWindow(int win) {
        UndirectedJungNetwork graph = new UndirectedJungNetwork();
        List<Windowable> posts = strategy.getWindow(win);


        for (Windowable w:posts) {
            Post p = ((WindowablePostAdapter)w).getPost();
            post2author.put(p.getPostid(),p.getUserid());
            if (p.getReplyToId() == null) continue;
            else if (!post2author.containsKey(p.getReplyToId())) {
                log.warn("Couldn't resolve reply-to id");
            } else {
                Node from = new NodeImpl(p.getUserid());
                Node to = new NodeImpl(post2author.get(p.getReplyToId()));
                Edge e = graph.findEdge(from,to);
                if (e!=null) {
                    e.setWeight(e.getWeight()+1.0f);
                } else {
                    graph.add(new EdgeImpl(from,to,1.0f,false));
                }
            }
        }
        return graph;
    }


}


<|start_filename|>src/edu/mit/cci/adapters/csv/PostImpl.java<|end_filename|>
package edu.mit.cci.adapters.csv;

import edu.mit.cci.teva.model.Post;

import java.util.Collections;
import java.util.Date;
import java.util.List;

/**
* Created by josh on 1/20/14.
*/
public class PostImpl implements Post {

    public String postId,replyToId,threadId,userId,content;
    public Date creation;



    public PostImpl(String postId, String replyToId, String userId, Date creation) {

        this.postId = postId;
        this.replyToId = replyToId;
        this.userId = userId;
        this.creation = creation;
    }

    public void setContent(String content) {
        this.content = content;
    }

    public void setCreation(Date creation) {
        this.creation = creation;
    }

    public void setPostId(String postId) {
        this.postId = postId;
    }

    public void setReplyToId(String replyToId) {
        this.replyToId = replyToId;
    }


    public void setThreadId(String threadId) {
        this.threadId = threadId;
    }

    public void setUserId(String userId) {
        this.userId = userId;
    }

    public String getThreadid() {
        return threadId;
    }

    public String getPostid() {
        return postId;
    }

    public String getReplyToId() {
       return replyToId;
    }

    public String getUserid() {
       return userId;
    }

    public Date getTime() {
        return creation;
    }

    public List<String> getSentiment() {
        return Collections.emptyList();
    }

    public String getContent() {
        return content;
    }
}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityMembershipStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.text.windowing.Bin;
import edu.mit.cci.text.windowing.Windowable;

import java.util.Collection;
import java.util.List;
import java.util.Map;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 8/30/13
 * Time: 11:48 AM
 * To change this template use File | Settings | File Templates.
 */
public interface CommunityMembershipStrategy {

    public List<Assignment> assignToCommunity(CommunityModel communities, int window, Network net);

    public static class Assignment {

        public float coverage;
        public float similarity;
        public Community community;
        public Collection<Edge> edges;

        public Assignment(Community c, float coverage, float similarity) {
            this.coverage = coverage;
            this.similarity = similarity;
            this.community = c;
        }

        public Assignment(Community c, float coverage, float similarity, Collection<Edge> edges) {
           this(c,coverage,similarity);
            this.edges = edges;
        }

    }


}


<|start_filename|>src/edu/mit/cci/teva/TevaRunner.java<|end_filename|>
package edu.mit.cci.teva;

import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.teva.engine.NetworkProvider;

import javax.xml.bind.JAXBException;
import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 6/3/13
 * Time: 2:56 PM
 * To change this template use File | Settings | File Templates.
 */
public interface TevaRunner {
    NetworkProvider getNetworkProvider() throws IOException;

    void processAndSave() throws IOException, CommunityFinderException, JAXBException;
}


<|start_filename|>src/edu/mit/cci/sna/jung/JungUtils.java<|end_filename|>
package edu.mit.cci.sna.jung;


import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.uci.ics.jung.graph.AbstractGraph;
import edu.uci.ics.jung.graph.Graph;
import edu.uci.ics.jung.graph.util.Pair;
//import it.uniroma1.dis.wsngroup.gexf4j.core.EdgeType;
//import it.uniroma1.dis.wsngroup.gexf4j.core.Gexf;
//import it.uniroma1.dis.wsngroup.gexf4j.core.Mode;
//import it.uniroma1.dis.wsngroup.gexf4j.core.data.Attribute;
//import it.uniroma1.dis.wsngroup.gexf4j.core.data.AttributeClass;
//import it.uniroma1.dis.wsngroup.gexf4j.core.data.AttributeList;
//import it.uniroma1.dis.wsngroup.gexf4j.core.data.AttributeType;
//import it.uniroma1.dis.wsngroup.gexf4j.core.dynamic.Spell;
//import it.uniroma1.dis.wsngroup.gexf4j.core.dynamic.TimeFormat;
//import it.uniroma1.dis.wsngroup.gexf4j.core.impl.GexfImpl;
//import it.uniroma1.dis.wsngroup.gexf4j.core.impl.SpellImpl;
//import it.uniroma1.dis.wsngroup.gexf4j.core.impl.StaxGraphWriter;
//import it.uniroma1.dis.wsngroup.gexf4j.core.impl.data.AttributeListImpl;
import org.apache.commons.collections15.Transformer;
import org.apache.log4j.Logger;



import java.io.*;
import java.util.*;

/**
 * User: jintrone
 * Date: 5/23/11
 * Time: 11:29 AM
 */
public class JungUtils {

//    public static DefaultUndirectedJungGraph makeUndirectedGraph(DefaultJungGraph g) {
//        DefaultUndirectedJungGraph graph =
//    }

    private static Logger log = Logger.getLogger(JungUtils.class);

    public static enum MergePolicy {
        ADD {
            public float merge(Edge one, Edge two) {
                return one.getWeight() + two.getWeight();

            }
        },

        ADD_MAX_1 {
            public float merge(Edge one, Edge two) {
                return Math.min(1.0f, one.getWeight() + two.getWeight());

            }
        },
        MAX {
            public float merge(Edge one, Edge two) {
                return Math.max(one.getWeight(), two.getWeight());
            }
        };

        public abstract float merge(Edge one, Edge two);
    }

    public static void normalizeEdgeWeights(Network net) {
        float max = 0.0f;
        for (Edge e:net.getEdges()) {
            max = Math.max(max,e.getWeight());
        }
        for (Edge e:net.getEdges()) {
            e.setWeight(e.getWeight()/max);
        }
    }


    public static AbstractGraph<Node, Edge> copy(AbstractGraph<NodeImpl, EdgeImpl> graph, boolean directed) {
        AbstractGraph<Node, Edge> result = directed ? new DirectedJungNetwork() : new UndirectedJungNetwork();
        for (NodeImpl node : graph.getVertices()) {
            result.addVertex(node);
        }
        for (EdgeImpl edge : graph.getEdges()) {
            Pair<NodeImpl> pair = graph.getEndpoints(edge);
            result.addEdge(edge, pair.getFirst(), pair.getSecond());
        }
        return result;
    }


    public static void merge(Graph<Node, Edge> to, Graph<Node, Edge> from, MergePolicy policy) {
        if (from == null || from.getVertexCount() == 0) return;
        Map<String, Node> labelmap = new HashMap<String, Node>();
        for (Node node : to.getVertices()) {
            labelmap.put(node.getLabel(), node);
        }

        for (Node node : from.getVertices()) {
            if (!labelmap.containsKey(node.getLabel())) {
                to.addVertex(node);
                labelmap.put(node.getLabel(), node);
            }
        }

        for (Edge edge : from.getEdges()) {
            //double check if this is using equality or what
            Pair<Node> p = from.getEndpoints(edge);
            Node src = labelmap.get(p.getFirst().getLabel());
            Node dest = labelmap.get(p.getSecond().getLabel());
            Edge e = to.findEdge(src, dest);
            if (e != null) {
                e.setWeight(policy.merge(e, edge));
            } else {
                EdgeImpl nedge = new EdgeImpl(src, dest);
                nedge.setWeight(edge.getWeight());
                to.addEdge(nedge, src, dest);
            }
        }
    }

    public static void merge(Graph<Node, Edge> to, Graph<Node, Edge> from) {
        merge(to, from, MergePolicy.ADD);
    }


    public static void writeGenericFile(Writer out, AbstractGraph<NodeImpl, EdgeImpl> graph) throws IOException {
        for (EdgeImpl e : graph.getEdges()) {
            Pair<NodeImpl> pair = graph.getEndpoints(e);
            out.write(pair.getFirst().getLabel() + " " + pair.getSecond().getLabel() + " " + e.getWeight() + "\n");
        }
        out.flush();
    }

    public static void writeGraphML(DirectedJungNetwork graph, Map<String, Object> nodeatts, String filename) throws IOException {
        writeGraphML(graph,nodeatts, Collections.<String, Object>emptyMap(),filename);
    }

    public static void writeGraphML(DirectedJungNetwork graph, Map<String, Object> nodeatts,Map<String,Object> edgeatts, String filename) throws IOException {
        MyGraphMLWriter<Node, Edge> writer = new MyGraphMLWriter<Node, Edge>();
        for (Map.Entry<String, Object> ent : nodeatts.entrySet()) {
            final String key = ent.getKey();
            final String[] clazz = new String[]{"string"};
            if (ent.getValue() instanceof Integer) {
                clazz[0] = "int";
            } else if (ent.getValue() instanceof Float || ent.getValue() instanceof Double) {
                clazz[0] = "float";
            }


            writer.addVertexData(ent.getKey(), "", clazz[0], ent.getValue().toString(), new Transformer<Node, String>() {

                public String transform(Node defaultJungNode) {
                    Object o = defaultJungNode.getProperty(key);
                    if (o == null) {
                        log.warn("Property [" + key + "] for node " + defaultJungNode + " is null");
                        log.warn("Other props: " + defaultJungNode.getProperties());
                    }
                    return defaultJungNode.getProperty(key).toString();
                }
            });
        }
        for (Map.Entry<String, Object> ent : edgeatts.entrySet()) {
            final String key = ent.getKey();
            final String[] clazz = new String[]{"string"};
            if (ent.getValue() instanceof Integer) {
                clazz[0] = "int";
            } else if (ent.getValue() instanceof Float || ent.getValue() instanceof Double) {
                clazz[0] = "float";
            }


            writer.addEdgeData(ent.getKey(), "", clazz[0], ent.getValue().toString(), new Transformer<Edge, String>() {

                public String transform(Edge defaultJungEdge) {
                    Object o = defaultJungEdge.getProperty(key);
                    if (o == null) {
                        log.warn("Property [" + key + "] for edge " + defaultJungEdge + " is null");
                        log.warn("Other props: " + defaultJungEdge.getProperties());
                        return "";
                    }
                    return o.toString();
                }
            });
        }

        FileWriter out = new FileWriter(filename);
        writer.save(graph, out);

    }


    public static void createDynamicGraph(File out, String description, List<Network> data, boolean directed, List<AttributeHolder> atts) throws IOException {
        LinkedHashMap<Date, Network> result = new LinkedHashMap<Date, Network>();
        long start = System.currentTimeMillis();
        int i = 0;
        for (Network a : data) {
            result.put(new Date(start + (1000l * 60l * 60l * 24l * i)), a);
            i++;

        }
        result.put(new Date(start + (1000l * 60l * 60l * 24l * i)), null);
   //     createDynamicGraph(out, description, result, directed, atts);
    }

    public static void createDynamicGraph(File out, String description, List<Network> data, boolean directed) throws IOException {
        createDynamicGraph(out, description, data, directed, null);
    }

    public static class AttributeHolder {
        public String[] attnames;
        public Map<String, String[]> values = new HashMap<String, String[]>();

        public AttributeHolder(String[] attnames) {
            this.attnames = attnames;

        }

        public void setAttributeValues(String node, String[] vals) {
            values.put(node, vals);
        }
    }


//    public static void createDynamicGraph(File out, String description, LinkedHashMap<Date, Network> data, boolean directed, List<AttributeHolder> atts) throws IOException {
//        Gexf gexf = new GexfImpl();
//        gexf.getMetadata().setLastModified(new Date()).setCreator("<NAME>").setDescription(description);
//        gexf.getGraph().setMode(Mode.DYNAMIC).setTimeType(TimeFormat.XSDDATETIME);
//        gexf.getGraph().setDefaultEdgeType(directed ? EdgeType.DIRECTED : EdgeType.UNDIRECTED);
//        AttributeList aList = new AttributeListImpl(AttributeClass.EDGE);
//        aList.setMode(Mode.DYNAMIC);
//        Attribute weight = aList.createAttribute("weight", AttributeType.FLOAT, "Weight");
//
//        AttributeList nList = new AttributeListImpl(AttributeClass.NODE);
//        nList.setMode(Mode.DYNAMIC);
//
//        gexf.getGraph().getAttributeLists().add(aList);
//        gexf.getGraph().getAttributeLists().add(nList);
//        List<Attribute> nAtts = new ArrayList<Attribute>();
//        if (atts != null) {
//            AttributeHolder h = atts.get(0);
//            for (String s : h.attnames) {
//                nAtts.add(nList.createAttribute(AttributeType.STRING, s));
//            }
//        }
//
//        Map<Collection<String>, it.uniroma1.dis.wsngroup.gexf4j.core.Edge> edges = new HashMap<>();
//        Map<String, it.uniroma1.dis.wsngroup.gexf4j.core.Node> nodes = new HashMap<>();
//        Map<String, List<Integer>> lifetime = new HashMap<String, List<Integer>>();
//        List<Network> networks = new ArrayList<>(data.values());
//        List<Date> dates = new ArrayList<>(data.keySet());
//        for (int i = 0; i < networks.size() - 1; i++) {
//            AttributeHolder a = atts != null ? atts.get(i) : null;
//            Set<String> processed = new HashSet<>();
//            for (edu.mit.cci.sna.Edge e : networks.get(i).getEdges()) {
//                Node[] ends = e.getEndpoints();
//
//
//                it.uniroma1.dis.wsngroup.gexf4j.core.Node gfrom = nodes.get(ends[0].getId());
//                List<Integer> active = lifetime.get(ends[0].getId());
//                if (active == null) {
//                    lifetime.put(ends[0].getId(), active = new ArrayList<>());
//                }
//                if (!active.contains(i)) active.add(i);
//
//                if (gfrom == null) {
//                    if (ends[0].getId().trim().isEmpty()) {
//                        System.err.println("What is up?");
//                    }
//                    gfrom = gexf.getGraph().createNode(ends[0].getId());
//                    gfrom.setLabel(gfrom.getId());
//                    nodes.put(ends[0].getId(), gfrom);
//                }
//                if (a != null && !processed.contains(gfrom.getLabel()) && a.values.containsKey(gfrom.getLabel())) {
//                    processed.add(gfrom.getLabel());
//                    for (int ai = 0; ai < nAtts.size(); ai++) {
//                        gfrom.getAttributeValues().createValue(nAtts.get(ai), a.values.get(gfrom.getLabel())[ai]).setStartValue(dates.get(i)).setEndValue(dates.get(i + 1));
//                    }
//                }
//
//
//                it.uniroma1.dis.wsngroup.gexf4j.core.Node gto = nodes.get(ends[1].getId());
//                active = lifetime.get(ends[1]);
//                if (active == null) {
//                    lifetime.put(ends[1].getId(), active = new ArrayList<Integer>());
//                }
//                if (!active.contains(i)) active.add(i);
//
//                if (gto == null) {
//                    gto = gexf.getGraph().createNode(ends[1].getId());
//                    gto.setLabel(gto.getId());
//                    nodes.put(ends[1].getId(), gto);
//                }
//
//                if (a != null && !processed.contains(gto.getLabel()) && a.values.containsKey(gto.getLabel())) {
//                    processed.add(gto.getLabel());
//                    for (int ai = 0; ai < nAtts.size(); ai++) {
//                        gto.getAttributeValues().createValue(nAtts.get(ai), a.values.get(gto.getLabel())[ai]).setStartValue(dates.get(i)).setEndValue(dates.get(i + 1));
//                    }
//                }
//                Collection cE = Arrays.asList(ends[0].getId(),ends[1].getId());
//                if (!directed) {
//                   cE = new HashSet(cE);
//                }
//
//                it.uniroma1.dis.wsngroup.gexf4j.core.Edge ge = edges.get(cE);
//                if (ge == null) {
//                    ge = gfrom.connectTo(gto);
//                    ge.setEdgeType(directed ? EdgeType.DIRECTED : EdgeType.UNDIRECTED);
//                    edges.put(cE, ge);
//
//                }
//
//                ge.getAttributeValues().createValue(weight, e.getWeight() + "").setStartValue(dates.get(i)).setEndValue(dates.get(i + 1));
//
//
//            }
//
//        }
//
//
//        for (String s : nodes.keySet()) {
//            it.uniroma1.dis.wsngroup.gexf4j.core.Node gnode = nodes.get(s);
//
//            Spell current = null;
//            int last = -1;
//            if (lifetime.get(s) == null) continue;
//
//            Collections.sort(lifetime.get(s));
//            for (Integer i : lifetime.get(s)) {
//                if (current == null) {
//                    gnode.getSpells().add(current = new SpellImpl());
//                    current.setStartValue(dates.get(i));
//                    last = i;
//                } else if (i - last > 1) {
//                    current.setEndValue(dates.get(last + 1));
//                    gnode.getSpells().add(current = new SpellImpl());
//                    current.setStartValue(dates.get(i));
//                    last = i;
//                } else {
//                    last = i;
//                }
//
//            }
//            if (current != null && !current.hasEndDate()) {
//                current.setEndValue(dates.get(last + 1));
//            }
//
//
//        }
//
//        new StaxGraphWriter().writeToStream(gexf, new FileOutputStream(out),"UTF8");
//
//
//    }


}


<|start_filename|>src/edu/mit/cci/adapters/csv/CsvBasedConversation.java<|end_filename|>
package edu.mit.cci.adapters.csv;

import com.csvreader.*;
import edu.mit.cci.teva.model.ConversationImpl;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;


import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 12/12/12
 * Time: 3:25 PM
 */
public class CsvBasedConversation extends ConversationImpl {

    private static DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    private static String[] cols = {"id","replyTo","created","author","text"};

    public static enum Column {
        ID(0), REPLY(1), CREATED(2), AUTHOR(3), TEXT(4);

        private String colname;

        Column(int i) {
            this.colname = cols[i];
        }

        public String getColumnName() {
            return colname;
        }
    }

    public CsvBasedConversation(String corpusname, InputStream inputFile) throws IOException, ParseException {
        this(cols, corpusname, inputFile, ',', true);
    }

    public CsvBasedConversation(String[] cols, String corpusname, InputStream inputFile, char delim, boolean useTextQual) throws IOException, ParseException {
        super(corpusname);
        CsvBasedConversation.cols = cols;
        CsvReader reader = new CsvReader(new InputStreamReader(inputFile),delim);
        reader.setUseTextQualifier(useTextQual);
        if (!reader.readHeaders()) {
            throw new RuntimeException("Empty input file?");
        };
        for (Column c:Column.values()) {
            if (reader.getIndex(c.getColumnName())<0) {
                throw new RuntimeException("Could not locate header: "+c.getColumnName());
            }
        }
        Map<Post,DiscussionThread> threadMap = new HashMap<Post, DiscussionThread>();
        Map<String,Post> postMap = new HashMap<String, Post>();

        while (reader.readRecord()) {



            if (skip(reader)) continue;
            PostImpl p = new PostImpl(processString(Column.ID, reader),processString(Column.REPLY, reader),
                    processString(Column.AUTHOR, reader),processDate(Column.CREATED,reader));
            p.setContent(reader.get(Column.TEXT.getColumnName()));


            postMap.put(p.getPostid(),p);

            if (p.getReplyToId()==null || p.getReplyToId().isEmpty()) {
                DiscussionThreadImpl d = new DiscussionThreadImpl("thread-"+p.getPostid());
                d.addPost(p);
                threadMap.put(p,d);
            }

        }

        Map<Post,Set<Post>> childmap = new HashMap<Post, Set<Post>>();
        for (Post p:postMap.values()) {
            if (threadMap.containsKey(p)) continue;
            Post parent = postMap.get(p.getReplyToId());
            Set<Post> children = childmap.get(parent);
            if (children == null) {
                children = new HashSet<Post>();
                childmap.put(parent,children);
            }
            children.add(p);
        }

        for (Map.Entry<Post,DiscussionThread> ent:threadMap.entrySet()) {
            addChildren(ent.getKey(), (DiscussionThreadImpl) ent.getValue(),childmap);
        }

        for (DiscussionThread thread:threadMap.values()) {
            this.addThread(thread);
        }


    }

    public boolean skip(CsvReader reader) throws IOException {
        return false;
    }

    public String processString(Column field, CsvReader reader) throws IOException {
        return (String)reader.get(field.getColumnName());
    }

    public Date processDate(Column field, CsvReader reader) throws ParseException, IOException {
        return format.parse(processString(field,reader));
    }


    public void addChildren(Post top, DiscussionThreadImpl thread, Map<Post,Set<Post>> childmap) {
        if (!childmap.containsKey(top)) return;
        Set<Post> children = childmap.get(top);
        for (Post child:children) {
            thread.addPost(child);
            addChildren(child, thread, childmap);
        }
    }


}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityScore.java<|end_filename|>
package edu.mit.cci.teva.engine;

/**
* User: jintrone
* Date: 10/14/12
* Time: 11:38 PM
*/
public class CommunityScore {
    public float coverage = 0.0f;
    public float similarity = 0.0f;
    int win = -1;
    public Community community = null;

    public CommunityScore(int win, Community community, float coverage, float similarity) {
        this.coverage = coverage;
        this.similarity = similarity;
        this.win = win;
        this.community = community;
    }

}


<|start_filename|>src/edu/mit/cci/teva/engine/EvolutionEngine.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import org.apache.log4j.Logger;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * User: jintrone
 * Date: 4/19/12
 * Time: 7:36 PM
 */
public class EvolutionEngine {


    private static Logger log = Logger.getLogger(EvolutionEngine.class);
    private static Logger logProcess = log;


    EvolutionStepStrategy stepStrategy;
    MergeStrategy mergeStrategy;

    private NetworkProvider provider;
    private CommunityFinder finder;


    private CommunityModel model;
    private TevaParameters params;


    public EvolutionEngine(CommunityModel model, TevaParameters params, NetworkProvider provider, CommunityFinder finder, EvolutionStepStrategy stepper, MergeStrategy merger) {
        this.model = model;
        this.provider = provider;
        this.stepStrategy = stepper;
        this.mergeStrategy = merger;
        this.finder = finder;
        this.params = params;
    }


    public CommunityModel getCommunityModel() {
        return model;
    }


    /**
     * At each step, uses the "maximal communities" (leading edge) of the current topic model
     * and tries to develop a mapping to the next slice of conversation, using Palla's approach
     * of merging slices and mapping through cliques in the merged layer
     *
     * @throws CommunityFinderException
     * @throws java.io.IOException
     */
    public void process() throws CommunityFinderException, IOException {


        Network lastGraph = null;
        for (int i = 0; i < provider.getNumberWindows(); i++) {

            Network currentGraph = provider.getNetworkAt(i);
            log.info("Removing edges weight < " + params.getMinimumLinkWeight() + " from " + currentGraph.getEdges().size());
            NetworkUtils.filterEdges(currentGraph, params.getMinimumLinkWeight());
            log.info(currentGraph.getEdges().size() + " edges remain");
            if (currentGraph.getEdges().size() == 0) {
                log.info("Skipping network at step " + i);
                continue;
            }
            if (!step(i, lastGraph, currentGraph)) {
                break;
            }
            lastGraph = currentGraph;

        }
    }

    protected int getCliqueSizeAtWindow(int i) {
        return params.getFixedCliqueSize();
    }

    protected List<CommunityFrame> getInputFrames() {
        List<CommunityFrame> result = new ArrayList<CommunityFrame>();
        Set<String> communities = new HashSet<String>();
        for (Community c : model.getCommunities()) {

            if (c.isExpired()) continue;
            if (communities.contains(c.getId())) {
                log.warn("Adding frame that already points to community: " + c.getId());
            }
            communities.add(c.getId());
            CommunityFrame data = c.getCommunityAtBin(c.getMaxBin());
            result.add(data);

        }
        return result;
    }

    private boolean step(int i, Network lastGraph, Network currentGraph) throws CommunityFinderException {

        List<CommunityFrame> to = finder.findCommunities(currentGraph, getCliqueSizeAtWindow(i), i, params.getFilenameIdentifier());
        if (to == null) return false;

        if (lastGraph == null) {


            for (CommunityFrame a : to) {
                Community c = Community.create();
                c.addFrame(a);
                model.addCommunity(c);
                log.info("CREATE INIT COMMUNITY " + c + " -> " + a.getNodes());
            }

        } else {


            List<CommunityFrame> from = getInputFrames();
            List<Network> merged = mergeStrategy.process(lastGraph, from, currentGraph, to, i);
            stepStrategy.processStep(i, from, merged, to);

        }
        return true;
    }


}


<|start_filename|>src/edu/mit/cci/sna/jung/Loader.java<|end_filename|>
package edu.mit.cci.sna.jung;

import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;

import edu.mit.cci.util.UnicodeReader;

import edu.uci.ics.jung.algorithms.util.MapSettableTransformer;
import edu.uci.ics.jung.graph.DirectedSparseMultigraph;
import edu.uci.ics.jung.graph.Graph;
import edu.uci.ics.jung.io.PajekNetReader;
import edu.uci.ics.jung.io.PajekNetWriter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.util.HashMap;
import java.util.Map;

/**
 * User: jintrone
 * Date: 4/21/11
 * Time: 10:53 AM
 */
public class Loader {


    public static DirectedSparseMultigraph<NodeImpl, EdgeImpl> loadGraph(File file) throws IOException {
        Reader fileReader = new UnicodeReader(new FileInputStream(file), "UTF-8");
        DirectedSparseMultigraph<NodeImpl, EdgeImpl> graph = new DirectedSparseMultigraph<NodeImpl, EdgeImpl>();
        PajekNetReader<DirectedSparseMultigraph<NodeImpl, EdgeImpl>, NodeImpl, EdgeImpl> graphreader = new PajekNetReader<DirectedSparseMultigraph<NodeImpl, EdgeImpl>, NodeImpl, EdgeImpl>(NodeImpl.getFactory(), EdgeImpl.getFactory());
        graphreader.setEdgeWeightTransformer(new MapSettableTransformer<EdgeImpl, Number>(new HashMap<EdgeImpl, Number>()));
        graphreader.setVertexLabeller(new MapSettableTransformer<NodeImpl, String>(new HashMap<NodeImpl, String>()));
        graphreader.load(fileReader, graph);

        //TODO this is a problem
        for (NodeImpl node : graph.getVertices()) {
            node.setLabel(graphreader.getVertexLabeller().transform(node));
        }
        for (EdgeImpl edge : graph.getEdges()) {
            edge.setWeight(graphreader.getEdgeWeightTransformer().transform(edge).floatValue());
        }
        return graph;

    }

     public static void writeGraph(Graph<NodeImpl,EdgeImpl> graph, OutputStream stream, boolean labels, boolean weights) throws IOException {
        PajekNetWriter<NodeImpl, EdgeImpl> writer = new PajekNetWriter<NodeImpl, EdgeImpl>();
        OutputStreamWriter os = new OutputStreamWriter(stream);
         Map<NodeImpl,String> labelmap = new HashMap<NodeImpl, String>();
         Map<EdgeImpl,Number> weightmap = new HashMap<EdgeImpl,Number>();
         if (labels) {
             for (NodeImpl n:graph.getVertices()) {
                 labelmap.put(n,n.getLabel());
             }
         }

         if (weights) {
             for (EdgeImpl e:graph.getEdges()) {
                 weightmap.put(e,e.getWeight());
             }
         }

        writer.save(graph,os,new MapSettableTransformer<NodeImpl, String>(labelmap),new MapSettableTransformer<EdgeImpl, Number>(weightmap));
        //os.flush();
     }
}


<|start_filename|>src/edu/mit/cci/teva/engine/BinningMembershipEngine.java<|end_filename|>
package edu.mit.cci.teva.engine;


import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.TevaFactory;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.util.SimilarityBasedAssignment;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.Bin;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.TextToNetworkGenerator;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.IOException;
import java.util.*;

/**
 * User: jintrone
 * Date: 6/20/11
 * Time: 9:37 PM
 */
public class BinningMembershipEngine implements TopicMembershipEngine {


    Conversation info;


    CommunityModel communities;
    private TevaFactory factory;

    private CommunityMembershipStrategy strategy = new SimilarityBasedAssignment();

    private Map<Integer, Integer> activeThreads = new HashMap<Integer, Integer>();

    private static Logger log = Logger.getLogger(BinningMembershipEngine.class);

    public BinningMembershipEngine(CommunityModel communities, Conversation info, TevaFactory factory) {
        this.info = info;


        this.communities = communities;
        this.factory = factory;
        strategy = factory.getMembershipMatchingStrategy();

    }



    public void process() throws IOException {
        log.debug("Running topic membership assignment");
        BinningStrategy<Windowable> binningStrategy = factory.getTopicBinningStrategy(factory.getConversationData(), factory.getTopicWindowingFactory());
        TextToNetworkGenerator generator = factory.getNetworkCalculator();
        Date[][] dates = factory.getTopicWindowingFactory().getStrategy().getWindowBoundaries();
        for (int i = 0; i < binningStrategy.getNumWindows(); i++) {
            List<Bin<Windowable>> dataAtWin = binningStrategy.getDataAtWindow(i);

            for (Bin<Windowable> dataInBin : dataAtWin) {
                Network net = generator.calculateWeights(getTokens(0, dataInBin.getFirstItemIndex(), dataInBin), getTokens(dataInBin.getFirstItemIndex(), dataInBin.size(), dataInBin));
                int j =0;

                //only assign data at the after the end of the previous window
                if (i > 0) {
                    for (;j<dataInBin.size();j++) {
                        if (dataInBin.get(j).getStart().after(dates[i-1][1])) break;
                    }
                }
                if (j < dataInBin.size()) assignToCommunity(i, net, dataInBin.subList(j, dataInBin.size()));
            }

        }
        log.debug("Done topic membership assignment");
    }



    public void assignToCommunity(int window, Network net, List<Windowable> bin) {

        List<CommunityMembershipStrategy.Assignment> assignments = strategy.assignToCommunity(communities,window,net);
        for (CommunityMembershipStrategy.Assignment a: assignments) {
            a.community.addAssignment(new ConversationChunk(bin, window, a.coverage, a.similarity,a.edges));
        }
    }


    public List<String> getTokens(int start, int end, Bin<Windowable> bin) {
        List<String> result = new ArrayList<String>();
        for (int i = start; i < end; i++) {
            result.addAll(bin.get(i).getTokens());
        }
        return result;


    }


    public static void main(String[] args) {

    }


}


<|start_filename|>src/edu/mit/cci/text/preprocessing/PorterMunger.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

import edu.mit.cci.teva.util.PorterStemmer;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.List;


/**
 * Created by josh on 9/16/14.
 */
public class PorterMunger implements Munger {

    PorterStemmer stemmer = new PorterStemmer();
    List<String> buffer = new ArrayList<String>();
    private static Logger logger = Logger.getLogger(PorterMunger.class);

    @Override
    public boolean read(String from) {
        if (from==null || "".equals(from.trim())) {
            return false;
        }
        String result = stemmer.stem(from);

        if ("".equals(result)) {
            return false;
        }
        buffer.add(result);
        return true;



    }

    @Override
    public List<String> flush() {
        List<String> result =  new ArrayList<String>(buffer);
        buffer.clear();
        return result;
    }

    @Override
    public List<String> finish() {
        return flush();
    }
}


<|start_filename|>src/edu/mit/cci/sna/jung/DirectedJungNetwork.java<|end_filename|>
package edu.mit.cci.sna.jung;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.uci.ics.jung.graph.DirectedSparseMultigraph;

import java.util.Collection;

/**
 * User: jintrone
 * Date: 9/21/12
 * Time: 8:50 PM
 */
public class DirectedJungNetwork extends DirectedSparseMultigraph<Node,Edge> implements Network {

    public DirectedJungNetwork() {}

    public DirectedJungNetwork(DirectedJungNetwork copy) {
        for (Edge edge:copy.getEdges()){
            Node[] nodes = edge.getEndpoints();
            this.addEdge(new EdgeImpl(nodes[0],nodes[1],edge.getWeight(),true),copy.getEndpoints(edge));
        }
    }

    public Collection<Node> getNodes() {
        return getVertices();
    }

    public boolean isDirected() {
        return true;
    }

        public void remove(Edge e) {
       this.removeEdge(e);
    }

    public void remove(Node n) {
       this.removeVertex(n);
    }

    public void add(Node n) {
        this.addVertex(n);
    }

    public void add(Edge e) {
        if (!containsEdge(e)) {
            this.addEdge(e,e.getEndpoints()[0],e.getEndpoints()[1]);
        }
    }
}


<|start_filename|>src/edu/mit/cci/text/wordij/TextCorpusNetworkGenerator.java<|end_filename|>
package edu.mit.cci.text.wordij;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.util.NetWriterStrategy;
import edu.mit.cci.text.windowing.Bin;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.uci.ics.jung.graph.Graph;
import org.apache.log4j.Logger;

import java.io.File;
import java.util.ArrayList;
import java.util.List;

/**
 * User: jintrone
 * Date: 5/11/11
 * Time: 5:07 PM
 */

/**
 * Generates a list of graphs for a corpus, using a windowing strategy to
 * segment an underlying data stream.
 *
 * @param <T>
 */
public class TextCorpusNetworkGenerator<T extends Windowable> implements CorpusToNetworkGenerator<T> {

    private final JungUtils.MergePolicy merger;
    BinningStrategy<T> bmodel;
    TextToNetworkGenerator calculator;

    boolean overwrite = true;

    Logger log = Logger.getLogger(TextCorpusNetworkGenerator.class);


    public TextCorpusNetworkGenerator(BinningStrategy<T> bmodel,
                                      TextToNetworkGenerator calculator) {
        this(bmodel, calculator, true, JungUtils.MergePolicy.ADD_MAX_1);

    }

    public TextCorpusNetworkGenerator(BinningStrategy<T> bmodel,
                                      TextToNetworkGenerator calculator, boolean overwrite, JungUtils.MergePolicy merger) {
        this.bmodel = bmodel;
        this.calculator = calculator;
        this.overwrite = overwrite;
        this.merger = merger;

    }

    @Override
    public List<File> analyzeToFiles(NetWriterStrategy strategy) {
        List<File> results = new ArrayList<File>();
        log.info("Will process " + bmodel.getNumWindows() + " windows");
        for (int win = 0; win < bmodel.getNumWindows(); win++) {
            File f = strategy.getNetworkFilename(win);
            if (!overwrite && f.exists()) {
                log.info("File "+f+" exists, not generating");
                continue;
            }
            log.debug("Process window " + win);
            try {
                results.add(strategy.writeNetwork(analyzeWindow(win), win));
            } catch (Exception e) {
                throw new RuntimeException("Could not write network file",e);
            }

        }
        log.info("Done processing windows");
        return results;
    }

    @Override
    public List<Network> analyzeToMemory() {
        List<Network> results = new ArrayList<Network>();
        log.info("Will process " + bmodel.getNumWindows() + " windows");
        for (int win = 0; win < bmodel.getNumWindows(); win++) {
            log.info("Process window " + win);

            results.add(analyzeWindow(win));
        }
        log.info("Done processing windows");
        return results;
    }

    public Network analyzeWindow(int win) {
        UndirectedJungNetwork graph = new UndirectedJungNetwork();
        List<Bin<T>> bins = bmodel.getDataAtWindow(win);

        for (Bin<T> bin : bins) {
            if (!bin.isEmpty()) {
                JungUtils.merge(graph, (Graph<Node, Edge>) calculator.calculateWeights(getTokens(0, bin.getFirstItemIndex(), bin), getTokens(bin.getFirstItemIndex(), bin.size(), bin)), merger);
            }

        }
        if (merger == JungUtils.MergePolicy.ADD) {
            JungUtils.normalizeEdgeWeights(graph);
        }
        return graph;
    }


    public List<String> getTokens(int start, int end, Bin<T> bin) {
        List<String> result = new ArrayList<String>();
        for (int i = start; i < end; i++) {
            result.addAll(bin.get(i).getTokens());
        }
        return result;


    }
}


<|start_filename|>src/edu/mit/cci/teva/util/ResolvedConnection.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.engine.CommunityModel;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 5/22/13
 * Time: 3:39 PM
 * To change this template use File | Settings | File Templates.
 */
public class ResolvedConnection {

    private final CommunityModel.Connection connection;
    private final int from;

    public int getTo() {
        return to;
    }

    public CommunityModel.Connection getConnection() {
        return connection;
    }

    public int getFrom() {
        return from;
    }

    private final int to;

    public ResolvedConnection(int from, int to, CommunityModel.Connection connection) {
        this.connection = connection;
        this.from = from;
        this.to = to;

    }


}


<|start_filename|>src/edu/mit/cci/teva/util/Stringer.java<|end_filename|>
package edu.mit.cci.teva.util;

/**
* User: jintrone
* Date: 5/14/13
* Time: 3:48 PM
*/
public interface Stringer<T> {
    public String stringify(T object);
}


<|start_filename|>src/edu/mit/cci/text/preprocessing/DictionaryMunger.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

import org.apache.log4j.Logger;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * The dictionary munger also takes an input matrix of the form
 * <p/>
 * token1 token2 token3 replacement
 * token1 token2 replacement
 * token4 token5 token6 replacement
 * <p/>
 * It is an aggressive matcher, seeking to replace the greatest number of words
 * <p/>
 * User: jintrone
 * Date: 9/29/11
 * Time: 5:08 PM
 */


//TODO refactor this class into a module rather than subclass and finish implementing compositemunger
public class DictionaryMunger implements Munger {
    private static int pathid = 0;
    private static Logger log = Logger.getLogger(DictionaryMunger.class);

    protected Node top = new Node("TOP");
    private List<String> wordbuffer = new ArrayList<String>();
    private List<String> buffer = new ArrayList<String>();
    private Map<Node, Integer> rowmap = new HashMap<Node, Integer>();
    protected SimpleMatrix<Path> matrix;






    public DictionaryMunger(String[][] replacements) {

        for (String[] r : replacements) {
            top.add(r);
        }

        matrix = new SimpleMatrix<Path>(count(top, 0), 0);
    }

    public static DictionaryMunger read(InputStream i) throws IOException {
        List<String[]> dictionary = new ArrayList<String[]>();
        BufferedReader stream = new BufferedReader(new InputStreamReader(i));
        String line = null;
        while ((line= stream.readLine())!=null) {
            dictionary.add(line.trim().split("\\s+"));
        }
        return new DictionaryMunger(dictionary.toArray(new String[dictionary.size()][]));
    }


    protected int count(Node n, int depth) {
        rowmap.put(n, rowmap.size());
        if (depth == 1) n.isRoot = true;
        int result = 0;
        if (!n.children.isEmpty()) {
            for (Node child : n.children.values()) {
                result += count(child, depth + 1);
            }
        }
        return result + 1;
    }

    public boolean read(String input) {


        buffer.add(input);
        log.debug("Begin State: " + matrix.toString());
        log.debug("Adding: " + input);
        if (buffer.isEmpty()) {
            return false;
        } else {

            wordbuffer.addAll(buffer);
            matrix.addColumns(buffer.size());
            buffer.clear();

        }


        Node topmatch = top.match(input);
        if (topmatch != null) {
            matrix.set(rowmap.get(topmatch), matrix.cols() - 1, new Path(topmatch));
        }

        int i = 0;
        boolean result = false;
        while (!wordbuffer.isEmpty() && i < wordbuffer.size() - 1) {
            boolean empty = true;
            for (Node n : rowmap.keySet()) {

                Path root = matrix.get(rowmap.get(n), i);
                if (root == null) continue;
                empty = false;
                if (n.isRoot()) {
                    if (root.isFrozen() || !root.extend(input, i)) {
                        log.debug("Can't extend " + root + ":" + root.nodes);
                        buffer.addAll(resolve(root, 0));
                        result = true;
                        break;
                        //hmm
                    }
                } else if (!root.extend(input, i)) {
                    root.prune(i);
                    root.freeze();
                }
            }
            if (empty) {
                buffer.add(wordbuffer.get(i));
                wordbuffer.remove(i);
                matrix.removeColumn(i);
                result = true;
            }
            if (!result) i++;
        }


        log.debug("End State: " + matrix.toString());

        return !buffer.isEmpty();
    }

    public List<String> flush() {
        List<String> result = new ArrayList<String>();
        if (!buffer.isEmpty()) {
            result.addAll(buffer);
            buffer.clear();
        }
        return result;
    }

    public List<String> finish() {
        log.debug("Finishing with wordbuffer: "+wordbuffer);
        List<String> result = new ArrayList<String>();
        while (!wordbuffer.isEmpty()) {
            boolean found = false;
            for (Node n : rowmap.keySet()) {
                if (matrix.cols() == 0) {
                    log.warn("Not enough columns in matrix!");
                }
                Path p = matrix.column(0).get(rowmap.get(n));
                if (p != null && n.isRoot()) {
                    found = true;
                    result.addAll(resolve(p, 0));
                    break;

                }
            }
            if (!found) {
                result.add(wordbuffer.get(0));
                wordbuffer.remove(0);
                matrix.removeColumn(0);
            }
        }

        return result;

    }




    private void removeColumn(int i) {
        for (Path p : matrix.column(i)) {
            if (p != null) {
                p.clear(i);
            }
        }
        matrix.removeColumn(i);
    }

    private List<String> resolve(Path p, int idx) {
        List<String> result = new ArrayList<String>();
        p.prune(idx);
        if (p.nodes.isEmpty()) {
            result.add(wordbuffer.get(idx));
            wordbuffer.remove(0);
            removeColumn(0);
        } else {
            result.add(p.last().replacement);
            for (int i = 0; i < p.nodes.size(); i++) {
                wordbuffer.remove(0);
                removeColumn(0);
            }
        }

        return result;


    }


    protected class Path {

        private boolean frozen = false;

        private List<Node> nodes = new ArrayList<Node>();


        private int id = pathid++;

        public Path(Node... nodes) {
            add(nodes);
        }

        public void freeze() {
            frozen = true;
        }

        public boolean isFrozen() {
            return frozen;
        }


        public boolean extend(String text, int i) {
            if (last() == null) {
                log.debug("Path is empty!");
            }
            Node n = last().match(text);
            if (n != null) {
                nodes.add(n);
                matrix.set(rowmap.get(n), i + nodes.size() - 1, this);
                return true;
            }
            return false;
        }

        public Node last() {
            return nodes.isEmpty() ? null : nodes.get(nodes.size() - 1);
        }

        public void add(Node... nodes) {
            this.nodes.addAll(Arrays.asList(nodes));
        }

        public void prune(int start) {

            while (last() != null && !last().hasReplacement()) {
                log.debug("Pruning");
                matrix.clear(rowmap.get(last()), start + nodes.size() - 1);
                nodes.remove(nodes.size() - 1);


            }
        }

        public void clear(int i) {
            for (Node n : nodes) {
                matrix.clear(rowmap.get(n), i++);
            }

        }

        public String toString() {
            return "[" + id + "]";
        }


    }

    public class Node {

        public Map<String, Node> children = new HashMap<String, Node>();

        private String me;

        public boolean isRoot = false;

        public Node(String me, String... remainder) {
            this.me = me;
            if (remainder != null && remainder.length > 0) add(remainder);
        }

        public String replacement = null;

        public void add(String... others) {
            log.debug("Process rule:" + Arrays.toString(others));
            if (others.length > 1) {
                Node next = children.get(others[0]);
                if (next == null) {
                    next = new Node(others[0], Arrays.copyOfRange(others, 1, others.length));
                    children.put(others[0], next);
                } else {
                    next.add(Arrays.copyOfRange(others, 1, others.length));
                }
            } else {
                log.debug("Adding replacement " + others[0]);
                replacement = others[0];
            }
        }

        public boolean hasReplacement() {
            return replacement != null;
        }

        public boolean isTail() {
            return children.isEmpty();
        }

        public Node match(String text) {
            return children.get(text);
        }

        public String getReplacement() {
            return replacement;
        }

        public boolean isRoot() {
            return isRoot;
        }

        public String toString() {
            return me + (replacement != null ? "(" + replacement + ")" : "");
        }


    }

    protected class SimpleMatrix<T> {
        public List<List<T>> data = new ArrayList<List<T>>();

        private int rows = 0;

        public SimpleMatrix(int rows, int cols) {
            this.rows = rows;
            for (int i = 0; i < cols; i++) {

                data.add(emptyColumn());
            }
        }

        public int cols() {
            return data.size();
        }

        public int rows() {
            return rows;
        }

        public T get(int row, int col) {
            return data.get(col).get(row);
        }

        public void set(int row, int col, T val) {
            data.get(col).set(row, val);
        }

        public void clear(int row, int col) {
            set(row, col, null);
        }

        public void removeColumn(int col) {
            data.remove(col);
        }

        public void addColumns(int count) {
            for (int i = 0; i < count; i++) {
                data.add(emptyColumn());
            }
        }

        public List<T> column(int i) {
            return data.get(i);
        }

        private List<T> emptyColumn() {
            List<T> row = new ArrayList<T>();
            for (int j = 0; j < rows; j++) {
                row.add(null);
            }
            return row;
        }

        public String toString() {
            String header = "     ";
            for (String word : wordbuffer) {
                header += String.format("%-5s", word);
            }
            String row = "";
            for (Map.Entry<Node, Integer> ent : rowmap.entrySet()) {
                row += String.format("%-5s", ent.getKey());
                for (List<T> col : data) {
                    row += String.format("%-5s", col.get(ent.getValue()));
                }
                row += "\n";

            }
            return "\n" + header + "\n" + row;
        }


    }
}


<|start_filename|>src/edu/mit/cci/teva/DefaultTevaFactory.java<|end_filename|>
package edu.mit.cci.teva;

import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.teva.cpm.cfinder.CFinderCommunityFinder;
import edu.mit.cci.teva.cpm.cos.CosCommunityFinder;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.util.SimilarityBasedAssignment;
import edu.mit.cci.teva.util.WindowablePostAdapter;
import edu.mit.cci.text.preprocessing.AlphaNumericTokenizer;
import edu.mit.cci.text.preprocessing.DictionaryMunger;
import edu.mit.cci.text.preprocessing.Munger;
import edu.mit.cci.text.preprocessing.StopwordMunger;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.BasicBinningStrategy;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.TimeBasedSlidingWindowStrategy;
import edu.mit.cci.text.windowing.WindowStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.CorpusToNetworkGenerator;
import edu.mit.cci.text.wordij.LinearWeightNetworkGenerator;
import edu.mit.cci.text.wordij.TextCorpusNetworkGenerator;
import edu.mit.cci.text.wordij.TextToNetworkGenerator;
import org.apache.log4j.Logger;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Intended to allow pluggable implementations of modules.  Perhaps look to Spring instead, though?
 * <p/>
 * User: jintrone
 * Date: 10/8/12
 * Time: 1:05 PM
 */
public class DefaultTevaFactory implements TevaFactory {

    protected TevaParameters params;
    protected Conversation conversation;
    private static Logger log = Logger.getLogger(DefaultTevaFactory.class);
    List<List<Windowable>> conversationData  = null;

    public DefaultTevaFactory(TevaParameters params, Conversation conversation) {
        this.params = params;
        this.conversation = conversation;

    }

    public List<List<Windowable>> getConversationData() throws IOException {
        if (conversationData==null) {
            Tokenizer<String> tokenizer = getTokenizer();
            conversationData = new ArrayList<List<Windowable>>();
            for (DiscussionThread thread : conversation.getAllThreads()) {
                List<Windowable> threaddata = new ArrayList<Windowable>();
                for (Post p : thread.getPosts()) {
                    threaddata.add(new WindowablePostAdapter(p, tokenizer));
                }
                conversationData.add(threaddata);
            }
        }
        return conversationData;
    }

    public Munger[] getMungers() throws IOException {
         List<Munger> mungers = new ArrayList<Munger>();
        if (params.getReplacementDictionary() != null && !params.getReplacementDictionary().isEmpty()) {
            if (params.getReplacementDictionary().startsWith("/") || params.getReplacementDictionary().startsWith(".")) {
                mungers.add(DictionaryMunger.read(new FileInputStream(params.getReplacementDictionary())));
                log.info("Loaded replacement list from file: "+params.getReplacementDictionary());
            } else {
                mungers.add(DictionaryMunger.read(getClass().getResourceAsStream("/" + params.getReplacementDictionary())));
                log.info("Loaded replacement list from resource: "+params.getReplacementDictionary());
            }
        }
        if (params.getStopwordList() != null && !params.getStopwordList().isEmpty()) {
            if (params.getReplacementDictionary().startsWith("/") || params.getReplacementDictionary().startsWith(".")) {
                mungers.add(StopwordMunger.readAsNew(new FileInputStream(params.getStopwordList())));
                log.info("Loaded stopword list from file: "+params.getStopwordList());
            } else {
                mungers.add(StopwordMunger.readAsNew(getClass().getResourceAsStream(("/" + params.getStopwordList()))));
                log.info("Loaded stopword list from resource: "+params.getStopwordList());
            }
        }
        return mungers.toArray(new Munger[mungers.size()]);
    }

    public Tokenizer<String> getTokenizer() throws IOException {
        return new AlphaNumericTokenizer(getMungers());
    }

    public MergeStrategy getMerger() {
        return new SlowMergeStrategy(getFinder(),params);
    }

    public EvolutionStepStrategy getStepper(CommunityModel model) {
        return new BasicStepStrategy(model, params);
    }


    public WindowStrategy.Factory<Windowable> getTopicWindowingFactory() {
        return new WindowStrategy.Factory<Windowable>() {

            public WindowStrategy<Windowable> getStrategy() {
                return new TimeBasedSlidingWindowStrategy(conversation.getFirstPost().getTime(), conversation.getLastPost().getTime(), params.getWindowSize(), params.getWindowDelta());
            }
        };
    }

    public WindowStrategy.Factory<Windowable> getPostAssignmentWindowingFactory() {
        return new WindowStrategy.Factory<Windowable>() {

            public WindowStrategy<Windowable> getStrategy() {
                return new TimeBasedSlidingWindowStrategy(conversation.getFirstPost().getTime(), conversation.getLastPost().getTime(), params.getMembershipWindowSize(), params.getMembershipWindowDelta());
            }
        };
    }

    public TextToNetworkGenerator getNetworkCalculator() {
        return new LinearWeightNetworkGenerator(params.getWordijIndirection(), params.getWordijTupleSize(), params.getWordijMaxWeight());
    }


    public CorpusToNetworkGenerator<Windowable> getNetworkGenerator(BinningStrategy<Windowable> binningStrategy) {
        return new TextCorpusNetworkGenerator<>(binningStrategy,this.getNetworkCalculator(),params.getOverwriteNetworks(), JungUtils.MergePolicy.valueOf(params.getParallelNetworkMergePolicy()));
    }

    @Override
    public CommunityMembershipStrategy getMembershipMatchingStrategy() {
        return new SimilarityBasedAssignment();
    }

    public CommunityFinder getFinder() {
        return new CosCommunityFinder(params);
    }

    public BinningStrategy<Windowable> getTopicBinningStrategy(List<List<Windowable>> data, WindowStrategy.Factory<Windowable> windows) {
        return new BasicBinningStrategy<>(data, windows);

    }

    public BinningStrategy<Windowable> getMembershipBinningStrategy(List<List<Windowable>> data, WindowStrategy.Factory<Windowable>windows) {
        return new BasicBinningStrategy<>(data, windows);
    }

    @Override
    public TopicMembershipEngine getMembershipEngine(CommunityModel model, Conversation conversation) {
        return new BinningMembershipEngine(model,conversation,this);
    }



}


<|start_filename|>src/edu/mit/cci/teva/util/LeafPruningDiscussionThreadAdapter.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * User: jintrone
 * Date: 5/14/13
 * Time: 1:25 PM
 */
public class LeafPruningDiscussionThreadAdapter implements DiscussionThread {

    private final DiscussionThread thread;

    private List<Post> result = null;

    public LeafPruningDiscussionThreadAdapter(DiscussionThread t) {
        this.thread = t;
    }

    public List<Post> getPosts() {


        if (result == null) {
            result = new ArrayList<Post>();
            Map<String, Post> postmap = new HashMap<String, Post>();
            Map<Post, Integer> map = new LinkedHashMap<Post, Integer>();
            for (Post p : thread.getPosts()) {
                map.put(p, 0);
                postmap.put(p.getPostid(), p);
                if (p.getReplyToId() != null) {
                    map.put(postmap.get(p.getReplyToId()), map.get(postmap.get(p.getReplyToId())) + 1);
                }
            }
            for (Map.Entry<Post, Integer> ent : map.entrySet()) {
                if (ent.getValue() == 0) continue;
                result.add(ent.getKey());
            }

        }
        return result;
    }

    public List<String> getSentiment(Post p) {
        return thread.getSentiment(p);
    }

    public String getThreadId() {
        return thread.getThreadId();
    }
}


<|start_filename|>src/edu/mit/cci/text/preprocessing/Munger.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

/**
 * User: jintrone
 * Date: 5/11/11
 * Time: 5:11 PM
 */


import java.util.List;

/**
 * Generic interface for manipulating words from a data set - for handling things like stemming and stop lists
 * Not appropriate for block level operations (e.g. removing quoted sections)
 */
public interface Munger {




    /**
     *
     * @param from
     * @return true if there is new data in the buffer, false otherwise
     */
    public boolean read(String from);

    /**
     *
     * @return Returns all data and clears the buffer
     */
    public List<String> flush();

    /**
     * Finish processing any remaining state and return result
     * Clients should be sure to call this after processing is finished
     */
    public List<String> finish();






}


<|start_filename|>src/edu/mit/cci/teva/MemoryBasedRunner.java<|end_filename|>
package edu.mit.cci.teva;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.cpm.cfinder.CFinderCommunityFinder;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.util.TevaUtils;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.WindowStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.CorpusToNetworkGenerator;
import edu.mit.cci.text.wordij.TextCorpusNetworkGenerator;
import org.apache.log4j.Logger;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/3/12
 * Time: 9:39 PM
 */
public class MemoryBasedRunner implements TevaRunner {


    Logger log = Logger.getLogger(MemoryBasedRunner.class);
    TevaParameters params;
    Conversation conversation;
    private WindowStrategy.Factory<Windowable> windowStrategyFactory;
    private TevaFactory factory;


    public MemoryBasedRunner(Conversation c) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters();
        this.factory = new DefaultTevaFactory(params,conversation);
        //process();
    }

    public MemoryBasedRunner(Conversation c, InputStream parameters) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters(parameters);
        this.factory = new DefaultTevaFactory(params,conversation);
        //process();
    }

     public MemoryBasedRunner(Conversation c, InputStream parameters, TevaFactory factory) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = new TevaParameters(parameters);
        this.factory =factory;
        //process();
    }

    public MemoryBasedRunner(Conversation c, TevaParameters parameters, TevaFactory factory) throws IOException, CommunityFinderException, JAXBException {
        this.conversation = c;
        this.params = parameters;
        this.factory =factory;

        //process();
    }





    public NetworkProvider getNetworkProvider() throws IOException {

        NetworkProvider provider;
        File f = new File(params.getWorkingDirectory());
        if (!f.exists()) {
            f.mkdirs();
        }

        if (params.getSkipNetworkGeneration()) {
            WindowStrategy.Factory<Windowable> winFactory = factory.getTopicWindowingFactory();
            final int i = winFactory.getStrategy().getNumberWindows();
            provider = new NetworkProvider() {
                public int getNumberWindows() {
                    return i;
                }

                public Network getNetworkAt(int i) {
                    File f = CFinderCommunityFinder.getNetworkFile(new File(params.getWorkingDirectory()),i,params.getFilenameIdentifier());
                    try {
                        return NetworkUtils.readNetworkFile(f);
                    } catch (IOException e) {
                        log.error("Error reading network file "+f.getAbsolutePath()+", cannot proceed");
                        throw new RuntimeException(e);
                    }
                }
            };
        } else {
            BinningStrategy<Windowable> binningStrategy =  factory.getTopicBinningStrategy(factory.getConversationData(), factory.getTopicWindowingFactory());

            CorpusToNetworkGenerator<Windowable> networkGenerator = new TextCorpusNetworkGenerator<Windowable>(binningStrategy, factory.getNetworkCalculator());
            final List<Network> result = networkGenerator.analyzeToMemory();
            log.info("Done creating networks.");
            provider = new NetworkProvider() {
                public int getNumberWindows() {
                    return result.size();
                }

                public Network getNetworkAt(int i) {
                    return result.get(i);
                }
            };
        }

        return provider;
    }

    public CommunityModel process() throws IOException, CommunityFinderException {
        log.info("Begin process.");
        Community.reset();
        NetworkProvider provider = getNetworkProvider();
        CommunityModel model = new CommunityModel(params,factory.getTopicWindowingFactory().getStrategy().getWindowBoundaries(),conversation.getName());
        EvolutionEngine engine = new EvolutionEngine(model,params, provider, factory.getFinder(), factory.getStepper(model), factory.getMerger());
        engine.process();
        TopicMembershipEngine membershipEngine = factory.getMembershipEngine(model,conversation);
        membershipEngine.process();
        return model;
    }

    public void processAndSave() throws IOException, CommunityFinderException, JAXBException {

        TevaUtils.serialize(new File(params.getWorkingDirectory()+"/CommunityOutput."+conversation.getName()+"."+params.getFilenameIdentifier()+".xml"), process(), CommunityModel.class);

    }


}


<|start_filename|>src/edu/mit/cci/text/windowing/BasicBinningStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;


import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
 * Basic implementation of a binning strategy.  Presumes N threads, evolving in parallel.  Ignores reply structure if it exists.
 *
 * User: jintrone
 * Date: 5/23/11
 * Time: 5:33 PM
 */
public class BasicBinningStrategy<T extends Windowable> implements BinningStrategy<Windowable> {


    public List<WindowStrategy<Windowable>> binmodel = new ArrayList<WindowStrategy<Windowable>>();
    private static Logger log = Logger.getLogger(BasicBinningStrategy.class);
    private int maxwin = 0;

    public BasicBinningStrategy(List<List<Windowable>> info, WindowStrategy.Factory<Windowable> factory) {

        for (List<Windowable> bin : info) {
            Collections.sort(bin, new Comparator<Windowable>() {
                public int compare(Windowable argument, Windowable argument1) {
                    return argument.getStart().compareTo(argument1.getStart());
                }
            });
        }

        Collections.sort(info, new Comparator<List<Windowable>>() {

            public int compare(List<Windowable> arguments, List<Windowable> arguments1) {
                Windowable arg0 = arguments.get(0);
                Windowable arg1 = arguments1.get(0);
                return arg0.getStart().compareTo(arg1.getStart());
            }
        });
        int i = 0;
        for (List<Windowable> args : info) {
            WindowStrategy<Windowable> strategy = factory.getStrategy();
            strategy.setData(args);
            maxwin = Math.max(maxwin,strategy.getNumberWindows());
            binmodel.add(strategy);
        }
    }


    public List<Bin<Windowable>> getDataAtWindow(int window) {
        List<Bin<Windowable>> result = new ArrayList<Bin<Windowable>>();
        for (WindowStrategy<Windowable> strategy:binmodel) {
            result.add(new BasicBin<Windowable>(strategy.getWindow(window)));
        }
        return result;
    }





    public int getNumBins() {
        return binmodel.size();
    }

    public int getNumWindows() {
        return maxwin;
    }



}


<|start_filename|>src/edu/mit/cci/teva/util/CompositeAnalysis.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.TevaFactory;
import edu.mit.cci.teva.engine.CommunityMembershipStrategy;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.TextToNetworkGenerator;

import java.io.IOException;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 8/30/13
 * Time: 11:17 AM
 * To change this template use File | Settings | File Templates.
 */
public class CompositeAnalysis {

    private final CommunityModel topic;
    private final CommunityModel social;
    private TevaFactory factory;
    private CommunityMembershipStrategy assignmentStrategy = new ExhaustiveAssignment();

    public CompositeAnalysis(TevaFactory topicFactory,CommunityModel social, CommunityModel topic) {
        this.social = social;
        this.topic = topic;
        this.factory = topicFactory;
        if (social.getWindows().length != topic.getWindows().length) {
            throw new RuntimeException("Window lengths must be identical");
        }
    }

    public void analyze() throws IOException {
        List<List<Windowable>> data = factory.getConversationData();
        BinningStrategy<Windowable> binningStrategy = factory.getTopicBinningStrategy(factory.getConversationData(), factory.getTopicWindowingFactory());
        Tokenizer<String> tokenizer = factory.getTokenizer();
        TextToNetworkGenerator generator = factory.getNetworkCalculator();

    }
}


<|start_filename|>src/edu/mit/cci/sna/Clique.java<|end_filename|>
package edu.mit.cci.sna;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * User: jintrone
 * Date: 6/19/12
 * Time: 6:54 AM
 */
@XmlRootElement(name = "Clique")
@XmlAccessorType(XmlAccessType.NONE)
public class Clique {

    private static int MAX_ID = -1;

    private int id;
    private Set<String> nodes = new HashSet<String>();


    public Clique() {
        setId(MAX_ID + 1);
    }

//    public Clique(int capacity) {
//        setId(MAX_ID + 1);
//    }

    private void setId(int id) {
        this.id = id;
        MAX_ID = Math.max(id, MAX_ID);
    }

    public Clique(int localId) {
        setId(localId);
    }


    public Clique(int localId, Collection<String> nodes) {
        this.nodes.addAll(nodes);
        setId(localId);
    }

    @XmlAttribute(name = "id")
    public int getId() {
        return id;
    }

    public void addNode(String node) {
        this.nodes.add(node);
    }

    public void setNodes(Collection<String> nodes) {
        this.nodes.addAll(nodes);
    }

    public void setUsingNodes(Collection<Node> nodes) {
        for (Node n:nodes) {
            this.nodes.add(n.getLabel());
        }
    }

    @XmlElement(name = "node")
    public Set<String> getNodes() {
        return nodes;
    }

    public boolean equals(Object o) {
        return (o instanceof Clique && ((Clique) o).getNodes().equals(nodes));
    }

    public int hashCode() {
        return (nodes.hashCode() + Clique.class.hashCode()) * 7 - 13;
    }

    public int overlap(Clique c) {
        Set<String> nodes = new HashSet<String>(c.getNodes());
        nodes.retainAll(this.nodes);
        return nodes.size();
    }

    public Collection<Set<String>> getEdgeSet() {
        List<Set<String>> result = new ArrayList<Set<String>>();
        String[] s = nodes.toArray(new String[nodes.size()]);
        for (int i = 0; i < s.length; i++) {
            for (int j = i + 1; j < s.length; j++) {
                Set<String> pair = new HashSet<String>();
                result.add(pair);
                pair.add(s[i]);
                pair.add(s[j]);
            }
        }
        return result;
    }

    public String toString() {
        return "Clique " + id + ": " + getNodes();

    }

}


<|start_filename|>src/edu/mit/cci/teva/cpm/cos/CosFileReader.java<|end_filename|>
package edu.mit.cci.teva.cpm.cos;

import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.CliqueDecoratedNetwork;
import edu.mit.cci.sna.CliqueDecorater;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.util.F;
import edu.mit.cci.util.U;
import edu.uci.ics.jung.algorithms.filters.VertexPredicateFilter;
import org.apache.commons.collections15.Predicate;
import org.apache.log4j.Logger;

import java.io.*;
import java.nio.ByteOrder;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.*;

/**
 * User: jintrone
 * Date: 11/13/12
 * Time: 4:17 PM
 */
public class CosFileReader {

    private static Logger logger = Logger.getLogger(CosFileReader.class);

    public static List<CliqueDecoratedNetwork> readCommunities(String networkname, int cliquesize, File basedir) throws CommunityFinderException {
        File communities = new File(basedir, cliquesize + "_communities.txt");
        File mapping_file = new File(basedir, networkname + ".map");
        File max_cliques = new File(basedir, networkname + ".mcliques");
        File networkfile = new File(basedir, networkname);

        logger.info("Communities file: " + communities.getAbsolutePath());
        logger.info("Mapping file: " + mapping_file.getAbsolutePath());
        logger.info("Max cliques file: " + max_cliques.getAbsolutePath());
        logger.info("Network file: " + networkfile.getAbsolutePath());


        if (!communities.exists() || !mapping_file.exists() || !max_cliques.exists()) {
            throw new CommunityFinderException("Missing required files");
        }

        logger.debug("Begin read map");
        Map<Integer, Node> idNodemap = readMap(mapping_file);
        logger.debug("Done");
        final Map<String, Node> namedNodeMap = new HashMap<String, Node>();
        for (Node n : idNodemap.values()) {
            namedNodeMap.put(n.getLabel(), n);
        }

        // UndirectedJungNetwork network = readNetwork(networkfile, namedNodeMap);

        final Map<String, Set<Clique>> cliques = new HashMap<String, Set<Clique>>();

        logger.debug("Begin read communities");
        try {
            InputStreamReader isr = new InputStreamReader(new FileInputStream(communities));
            BufferedReader reader = new BufferedReader(isr, 32768);

            int i = 0;
            StringBuilder buffer = new StringBuilder(16);
            String head = null;
            BitSet b = new BitSet(cliques.size());
            int line = 0;
            while ((i = reader.read()) > -1) {
                switch ((char) i) {

                    case ':':
                        head = buffer.toString();
                        buffer = new StringBuilder(16);
                        if (!cliques.containsKey(head)) {
                            cliques.put(head,new HashSet<Clique>());
                        }
                        break;

                    case ' ':
                        b.set(Integer.parseInt(buffer.toString()));
                        buffer = new StringBuilder(16);
                        break;

                    case '\n':
                        if (buffer.length() > 0) {
                            b.set(Integer.parseInt(buffer.toString()));
                            buffer = new StringBuilder(16);
                        }
                        Clique c = new Clique();
                        addNodes(c,b,idNodemap);
                        cliques.get(head).add(c);
                        if (line++ % 1000 == 0) {
                            logger.debug("Line " + line);
                        }
                        break;

                    default:
                        buffer.append((char) i);
                }

            }

        } catch (IOException e) {
            throw new CommunityFinderException("Error reading mapping file");
        }
        logger.debug("Done");


        //to assist with interpretation:
        //loop through all set of cliques
        //add edges from each set to a network
        //return list of networks
        final List<CliqueDecoratedNetwork> result = new ArrayList<CliqueDecoratedNetwork>();
        U.map(cliques.entrySet()).forEach(new F<Map.Entry<String, Set<Clique>>>() {
            public void apply(Map.Entry<String, Set<Clique>> stringSetEntry) {
                final UndirectedJungNetwork net = new UndirectedJungNetwork();
                U.map(stringSetEntry.getValue()).forEach(new F<Clique>() {
                    public void apply(Clique clique) {
                        U.map(clique.getEdgeSet()).forEach(new F<Set<String>>() {
                            public void apply(Set<String> strings) {
                                String[] toArray = strings.toArray(new String[2]);
                                net.add(new EdgeImpl(namedNodeMap.get(toArray[0]), namedNodeMap.get(toArray[1])));
                            }
                        });
                    }
                });
                CliqueDecorater d = new CliqueDecorater(net);
                d.addCliques(cliques.get(stringSetEntry.getKey()));
                result.add(d);
            }
        });

//        for (final String s : nodes.keySet()) {
//            UndirectedJungNetwork community = (UndirectedJungNetwork) new VertexPredicateFilter<Node, Edge>(new Predicate<Node>() {
//                public boolean evaluate(Node node) {
//                    return nodes.get(s).contains(node);
//                }
//            }).transform(network);
//            CliqueDecorater d = new CliqueDecorater(community);
//            d.addCliques(cliques.get(s));
//            result.add(d);
//        }
        return result;


    }

    private static void addNodes(Clique c, BitSet set, Map<Integer,Node> idmap) {
      for (int i=0;i<set.length();i++) {
          if (set.get(i)) {
              c.addNode(idmap.get(i).getLabel());
              set.clear(i);
          }
      }
    }

    private static UndirectedJungNetwork readNetwork(File networkfile, Map<String, Node> namedNodeMap) throws CommunityFinderException {
        try {
            BufferedReader reader = new BufferedReader(new FileReader(networkfile));
            UndirectedJungNetwork network = new UndirectedJungNetwork();
            String line;
            while ((line = reader.readLine()) != null) {
                String[] tokens = line.split(" ");
                if (tokens.length < 2) {
                    throw new CommunityFinderException("Malformed network file");
                }
                network.addEdge(namedNodeMap.get(tokens[0]), namedNodeMap.get(tokens[1]), 1.0f);

            }
            return network;
        } catch (IOException e) {
            throw new CommunityFinderException("Error reading network", e);
        }
    }


    /**
     * Create a map of ID->Node(String label)
     *
     * @param mapping_file
     * @return
     * @throws CommunityFinderException
     */
    public static Map<Integer, Node> readMap(File mapping_file) throws CommunityFinderException {
        try {
            BufferedReader reader = new BufferedReader(new FileReader(mapping_file));
            Map<Integer, Node> result = new HashMap<>();
            String line;

            while ((line = reader.readLine()) != null) {
                String[] tokens = line.split(" ");
                if (tokens.length < 2) {
                    throw new CommunityFinderException("Malformed node map file");
                }
                result.put((Integer.parseInt(tokens[1])), new NodeImpl(tokens[0]));


            }
            return result;
        } catch (IOException e) {
            throw new CommunityFinderException("Error reading node map", e);
        }

    }


}



<|start_filename|>src/edu/mit/cci/text/wordij/LinearWeightNetworkGenerator.java<|end_filename|>
package edu.mit.cci.text.wordij;


import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.uci.ics.jung.algorithms.shortestpath.UnweightedShortestPath;
import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


/**
 * Generates a network in which nodes with a degree of indirection of 0 are connected by an
 * edge with a maximum weight of l. Links are created between nodes increasing degree linearly
 * reducing the weight on the edge until degree = max.  The weakest link in the constructed
 * graph will have a weight of 1 - (max-1)/max.
 * <p/>
 * <p/>
 * User: jintrone
 * Date: 5/23/11
 * Time: 11:42 AM
 */
public class LinearWeightNetworkGenerator implements TextToNetworkGenerator {


    int maxIndirection = 2;
    int ngramsize = 2;
    float maxweight = 1.0f;


    Logger log = Logger.getLogger(LinearWeightNetworkGenerator.class);

    public LinearWeightNetworkGenerator(int maxindirection, int ngramsize,float maxWeight) {
        this.maxIndirection = maxindirection;
        this.ngramsize = ngramsize;
        this.maxweight = maxWeight;
    }


    public Network calculateWeights(List<String> preceding, List<String> sample) {

        if (sample == null || sample.size() == 0) return null;
        log.debug("Calculating network on sample (" + sample.size() + " tokens)");
        UndirectedJungNetwork result = new UndirectedJungNetwork();
        Map<String, Node> vertices = new HashMap<String, Node>();
        List<Node> prebuffer = new ArrayList<Node>();
        if (preceding != null && !preceding.isEmpty()) {
            for (String word : preceding.subList(Math.max(0, preceding.size() - ngramsize + 1), preceding.size())) {
                Node vertex = vertices.get(word);
                if (vertex == null) {
                    vertex = new NodeImpl(word);
                    vertices.put(word, vertex);
                }
                prebuffer.add(vertex);
            }

        }
        List<Node> buffer = new ArrayList<Node>();


        for (String word : sample) {
            Node vertex = vertices.get(word);
            if (vertex == null) {
                vertex = new NodeImpl(word);
                vertices.put(word, vertex);
            }
            buffer.add(vertex);
            if (!prebuffer.isEmpty()) {
                for (Node from:prebuffer) {
                    Edge edge = result.findEdge(from, vertex);
                     if (edge == null) {
                         edge = new EdgeImpl(from, vertex, maxweight, false);
                         result.addEdge(edge, from, vertex);
                         log.debug("Adding prebuffered edge: " + edge);

                     }
                }
                prebuffer.remove(0);
            }


            if (buffer.size() < ngramsize) {
                continue;
            } else if (buffer.size() > ngramsize) {
                buffer.remove(0);
            }

            for (int i = buffer.size() - 1; i >= 1; i--) {
                for (int j = buffer.size() - 2; j >= 0; j--) {
                    Node from = buffer.get(i);
                    Node to = buffer.get(j);
                    Edge edge = result.findEdge(from, to);
                    if (edge != null) {
                        //this shouldn't really happen.
                        if (edge.getWeight() < maxweight) {
                            edge.setWeight(maxweight);
                        }
                    } else {
                        edge = new EdgeImpl(from, to, maxweight, false);
                        result.addEdge(edge, from, to);
                        log.debug("Adding edge: " + edge);

                    }
                }
            }
        }

        if (maxIndirection > 1) updateIndirection(result);
        return result;
    }


    private void updateIndirection(UndirectedJungNetwork graph) {
        float delta = maxweight / (float) maxIndirection;
        UnweightedShortestPath<Node, Edge> paths = new UnweightedShortestPath<Node, Edge>(graph);
        List<Edge> edges = new ArrayList<Edge>();
        for (Node node : graph.getVertices()) {
            Map<Node, Number> p = paths.getDistanceMap(node);
            for (Map.Entry<Node, Number> ent : p.entrySet()) {
                if (ent.getValue().floatValue() > maxweight && ent.getValue().floatValue() <= maxIndirection) {
                    Edge nedge = new EdgeImpl(node, ent.getKey(), maxweight - (ent.getValue().intValue() - 1) * delta, false);
                    edges.add(nedge);
                }
            }
        }

        for (Edge edge : edges) {
            Edge existing = graph.findEdge(edge.getEndpoints()[0], edge.getEndpoints()[1]);
            if (existing == null) {
                graph.addEdge(edge, edge.getEndpoints()[0], edge.getEndpoints()[1]);
            } else {
                //TODO this should be a policy of some kind
                edge.setWeight(Math.min(1.0f,edge.getWeight()+existing.getWeight()));
                if (edge.getWeight() > existing.getWeight()) {
                    existing.setWeight(edge.getWeight());
                }
            }
        }


    }


}


<|start_filename|>src/edu/mit/cci/text/preprocessing/StringPreprocessor.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

/**
 * User: jintrone
 * Date: 10/20/11
 * Time: 12:46 PM
 */
public interface StringPreprocessor {

    public String process(String s);


    public static class Composite implements StringPreprocessor {

        StringPreprocessor[] preprocessors;


        public Composite(StringPreprocessor... p) {
           preprocessors = p;
        }

        public String process(String s) {
            String result = s;
            for (StringPreprocessor p:preprocessors) {
                result = p.process(result);
            }
            return result;
        }
    }

}


<|start_filename|>src/edu/mit/cci/sna/CliqueDecoratedNetwork.java<|end_filename|>
package edu.mit.cci.sna;

import java.util.Collection;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/3/12
 * Time: 10:57 PM
 */
public interface CliqueDecoratedNetwork extends Network {

    public Set<Clique> getCliques();


    public void addCliques(Collection<Clique> cliques);
}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityFrame.java<|end_filename|>
package edu.mit.cci.teva.engine;


import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.CliqueDecoratedNetwork;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.serialization.CommunityFrameJaxbAdapter;

import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;

/**
 * User: jintrone
 * Date: 6/15/11
 * Time: 1:27 PM
 */

@XmlJavaTypeAdapter(CommunityFrameJaxbAdapter.class)
public class CommunityFrame extends UndirectedJungNetwork {

    public static int id = -1;

    public static CommunityFrame EMPTY = new CommunityFrame(-1) {
        public String toString() {
            return "CF: Empty Frame";
        }
    };

    public Set<Clique> cliques = new HashSet<Clique>();


    private Community community = null;
    private int window = -1;



    public CommunityFrame(int bin) {
        this.window = bin;
    }

    public CommunityFrame(int window, CliqueDecoratedNetwork network) {

        this.window = window;
        for (Edge e:network.getEdges()) {
            this.add(e);
        }
        for (Clique c:network.getCliques()) {
            this.addClique(c);
        }
    }


    public int getWindow() {
        return window;
    }

    public String toString() {

        return "CF:"+ ((community==null?"(no community)":"("+community.getId()+")")+getWindow());
    }


    public Community getCommunity() {
        return community;
    }



    public void setCommunity(Community community) {
        this.community = community;
    }

    public void addClique(Clique c) {
        cliques.add(c);
    }


    public Set<Clique> getCliques() {
        return cliques;
    }



    public int hashCode() {
        return (this.getEdges().hashCode() + window)*13 + 7;
    }

    public boolean equals(CommunityFrame cf) {
        return cf.window == this.window &&
                this.getEdges().equals(cf.getEdges());
    }


}


<|start_filename|>src/edu/mit/cci/teva/util/WindowablePostAdapter.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.model.Post;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.Windowable;

import java.util.Date;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/4/12
 * Time: 4:30 PM
 */

public class WindowablePostAdapter implements Windowable {

    private Post post;
    private Tokenizer<String> tokenizer;

    public WindowablePostAdapter(Post p, Tokenizer<String> tokenizer) {
        this.post = p;
        this.tokenizer = tokenizer;
    }

    public Date getStart() {
        return post.getTime();
    }

    public List<String> getTokens() {
       return tokenizer.tokenize(post.getContent());
    }

    public String getRawData() {
        return post.getContent();
    }

    public String getId() {
        return post.getPostid();
    }

    public Post getPost() {
        return post;
    }

    public String toString() {
        return post.toString();

    }
}


<|start_filename|>src/edu/mit/cci/text/wordij/CorpusToNetworkGenerator.java<|end_filename|>
package edu.mit.cci.text.wordij;

import edu.mit.cci.sna.Network;
import edu.mit.cci.teva.util.NetWriterStrategy;
import edu.mit.cci.teva.util.WindowablePostAdapter;
import edu.mit.cci.text.windowing.Windowable;

import java.io.File;
import java.util.List;

/**
 * Created by josh on 7/26/14.
 */
public interface CorpusToNetworkGenerator<T extends Windowable>  {
    List<File> analyzeToFiles(NetWriterStrategy strategy);

    List<Network> analyzeToMemory();
}


<|start_filename|>src/edu/mit/cci/sna/NetworkUtils.java<|end_filename|>
package edu.mit.cci.sna;

import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 10:15 PM
 */
public class NetworkUtils {

    /**
     * Ratio of one's edges contained by two
     *
     * @param one
     * @param two
     * @return
     */
    public static float coverage(Network one, Network two) {
        if (one == null || two == null || one.getNodes().isEmpty() || two.getNodes().isEmpty()) {
            return 0f;
        } else {
            Set<Edge> fromedges = new HashSet<Edge>(one.getEdges());
            float denom = fromedges.size();
            fromedges.retainAll(two.getEdges());

            return (float) fromedges.size() / denom;
        }
    }

    public static float similarity(Network one, Network two) {
        if (one == null || two == null || one.getNodes().isEmpty() || two.getNodes().isEmpty()) return 0f;
        Set<Edge> intersection = new HashSet<Edge>(one.getEdges());
        intersection.retainAll(two.getEdges());
        Set<Edge> union = new HashSet<Edge>(one.getEdges());
        union.addAll(two.getEdges());
        return intersection.size() / (float) union.size();
    }


    public static float centralityWeightedScore(Network one, Network two) {
        if (one == null || two == null || one.getNodes().isEmpty() || two.getNodes().isEmpty()) return 0f;
        Set<Edge> intersection = new HashSet<Edge>(one.getEdges());
        intersection.retainAll(two.getEdges());
        Set<Edge> union = new HashSet<Edge>(one.getEdges());
        union.addAll(two.getEdges());
        return intersection.size() / (float) union.size();
    }

    public static List<Clique> findCliquesContaining(UndirectedJungNetwork graph, Set<Node> nodes, int k) {
        if (nodes.size() >= k) throw new RuntimeException("Clique size must be greater than incoming clique");
        Node[] nary = nodes.toArray(new Node[nodes.size()]);

        for (int i = 0; i < nary.length; i++) {
            for (int j = i + 1; j < nary.length; j++) {
                if (graph.findEdge(nary[i], nary[j]) == null) {
                    throw new RuntimeException("Incoming nodes must form a clique");
                }
            }
        }

        Set<Node> candidates = null;
        for (Node n : nodes) {
            if (candidates == null) {
                candidates = new HashSet<Node>();
                candidates.addAll(graph.getNeighbors(n));
            } else {
                candidates.retainAll(graph.getNeighbors(n));
            }
        }
        candidates.removeAll(nodes);
        return cliquex(graph, new ArrayList<Node>(nodes), new ArrayList<Node>(candidates), k);

    }

    public static List<Clique> cliquex(UndirectedJungNetwork graph, List<Node> included, List<Node> candidates, int cliquesize) {
        List<Clique> result = new ArrayList<Clique>();

        while ((candidates.size() + included.size()) >= cliquesize) {

            // could optimize this
            List<Node> included_copy = new ArrayList<Node>(included);
            List<Node> candidates_copy = new ArrayList<Node>(candidates.subList(1, candidates.size()));


            Node target = candidates.remove(0);
            for (Iterator<Node> i = candidates_copy.iterator(); i.hasNext(); ) {
                if (graph.findEdge(target, i.next()) == null) {
                    i.remove();
                }
            }


            included_copy.add(target);
            if (included_copy.size() == cliquesize) {
                result.add(cliquify(included_copy));
            } else {
                result.addAll(cliquex(graph, included_copy, candidates_copy, cliquesize));
            }


        }
        return result;
    }

    public static Clique cliquify(Collection<Node> nodes) {
        Clique c = new Clique();
        Set<String> nnames = new HashSet<String>();
        for (Node n : nodes) {
            nnames.add(n.getLabel());
        }
        c.setNodes(nnames);
        return c;
    }

    public static void filterEdges(Network currentGraph, float minimumLinkWeight) {
        for (Edge e : new ArrayList<Edge>(currentGraph.getEdges())) {
            if (e.getWeight() < minimumLinkWeight) currentGraph.remove(e);

        }
    }

    public static Network combine(Network one, Network two) {
        UndirectedJungNetwork result = new UndirectedJungNetwork();
        for (Edge e : one.getEdges()) {
            result.add(e);
        }
        for (Edge e : two.getEdges()) {
            Node a = e.getEndpoints()[0];
            Node b = e.getEndpoints()[1];
            Edge c = result.findEdge(a, b);
            if (c != null) {
                if (e.getWeight() > c.getWeight()) {
                    c.setWeight(e.getWeight());
                }
            } else {
                result.add(e);
            }


        }
        return result;
    }



    public static void createNetworkFile(Network adapter, File f, boolean edgeWeights) throws IOException {
        //String filename = prefix + "." + System.currentTimeMillis() + Math.random() + ".net";

        PrintWriter output = new PrintWriter(new FileWriter(f));
        //output.println("#" + new Date());
        for (Edge e : adapter.getEdges()) {
            output.println(e.getEndpoints()[0].getLabel() + " " + e.getEndpoints()[1].getLabel() + (edgeWeights ? " " + e.getWeight() : ""));
        }
        output.flush();
        output.close();
    }

    public static String simpleString(Network n) {
        StringBuilder builder = new StringBuilder();
        builder.append("Net[");
        for (Node node : n.getNodes()) {
            builder.append(node.getLabel()).append(",");
        }
        builder.append("]");
        return builder.toString();
    }


    public static Network readNetworkFile(File f) throws IOException {
        //String filename = prefix + "." + System.currentTimeMillis() + Math.random() + ".net";
        UndirectedJungNetwork network = new UndirectedJungNetwork();
        BufferedReader reader = new BufferedReader(new FileReader(f));
        String line = null;
        while ((line = reader.readLine()) != null) {
            if (line.isEmpty() || line.startsWith("#")) continue;


            String[] tokens = line.split("\\s+");
            if (tokens.length == 3) {
                network.add(new EdgeImpl(new NodeImpl(tokens[0]), new NodeImpl(tokens[1]), Float.parseFloat(tokens[2]), false));

            } else {
                network.add(new EdgeImpl(new NodeImpl(tokens[0]), new NodeImpl(tokens[1]), 1.0f, false));
            }
        }
        return network;
    }
}


<|start_filename|>src/edu/mit/cci/teva/analysis/DefaultAnalysisConsole.java<|end_filename|>
package edu.mit.cci.teva.analysis;

import edu.mit.cci.teva.model.Conversation;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;

import java.io.IOException;
import java.text.ParseException;

/**
 * Created by josh on 10/16/16.
 */
public class DefaultAnalysisConsole extends AnalysisConsole {
    public DefaultAnalysisConsole() throws IOException {
        super();
    }

    public DefaultAnalysisConsole(String filename) throws IOException {
        super(filename);
    }

    @Override
    public Conversation getConversation() throws IOException, ParseException {
        throw new RuntimeException("Get conversation is not implemented");
    }

    public static void main(String[] args) throws IOException {
        new DefaultAnalysisConsole().start();
    }
}


<|start_filename|>src/edu/mit/cci/teva/cpm/cfinder/CFinderRunner.java<|end_filename|>
package edu.mit.cci.teva.cpm.cfinder;

import edu.mit.cci.util.StreamGobbler;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Map;

/**
 * User: jintrone
 * Date: 6/15/11
 * Time: 11:11 AM
 */
public class CFinderRunner {


    private static Logger log = Logger.getLogger(CFinderRunner.class);
    public static String DEFAULT_CFINDER_EXECUTABLE = "/usr/local/bin/cfinder";
    public static String DEFAULT_CFINDER_LICENSE = "/Users/josh/Applications/CFinder-v2.0.5/licence.txt";


    public Map<CommandLineParams, String> params = new HashMap<CommandLineParams, String>();

    private String directory_suffix;
    private String info_prefix;
    private File cfinderExecutablePath;
    private File cfinderLicensePath;


    public enum CommandLineParams {
        MIN_WEIGHT("w"), MAX_WEIGHT("W"), CLIQUE_SIZE("k"), INTENSITY("I"), MAX_TIME("t"), DIRECTED("D"), OUTPUT("o");


        String param;

        CommandLineParams(String param) {
            this.param = param;
        }

        public String format(String s, String val) {
            return s + " -" + param + " " + val;
        }

        public String dirSuffix(String val) {
            return val;
        }

        public String info_prefix() {
            return "";
        }

        public static CommandLineParams resolve(String s) {
            CommandLineParams result = null;
            try {
                result = CommandLineParams.valueOf(s);

            } catch (IllegalArgumentException e) {
                return null;
            }
            return result;
        }


    }

    public CFinderRunner(String pathToExectutable, String pathToLicense, String... params) {

        this.cfinderExecutablePath = new File(pathToExectutable);
        this.cfinderLicensePath = new File(pathToLicense);
        setParams(params);
    }


    public void setParams(String[] pmap) {
        params.clear();
        for (int i = 0; i < pmap.length; i += 2) {

            CommandLineParams p = CommandLineParams.resolve(pmap[i]);
            if (p == null) {
                log.warn("Do not recognize " + pmap[i] + "; ignoring");
            } else {
                setParam(p, pmap[i + 1]);
            }
        }
    }

    public void setParam(CommandLineParams p, String value) {
        params.put(p, value);
    }

    public String buildCommandLine(File inputFile) {
        String cline = cfinderExecutablePath.getAbsolutePath();
        for (Map.Entry<CommandLineParams, String> ent : params.entrySet()) {
            cline = ent.getKey().format(cline, ent.getValue());
        }
        cline += " -l " + cfinderLicensePath;
        return cline + " -i " + inputFile.getAbsolutePath();

    }

    public boolean process(File inputFile, boolean overwrite) throws IOException, InterruptedException {
        log.debug("Process:" + inputFile + " with params " + params);
        if (getOutputDir(inputFile).exists()) {
            if (overwrite) {
                U.delete(getOutputDir(inputFile));
            } else {
                log.warn("Output directory exists, skipping community generation");
                return false;
            }
        }

        log.info("Generating communities for " + inputFile.getName());
        long current = System.currentTimeMillis();
        String cmdline = buildCommandLine(inputFile);
        log.debug("Will execute: "+cmdline);
        Process p = Runtime.getRuntime().exec(cmdline);
        StreamGobbler errorGobbler = new
                StreamGobbler(p.getErrorStream(), "ERROR");

        // any output?
        StreamGobbler outputGobbler = new
                StreamGobbler(p.getInputStream(), "OUTPUT");

        // kick them off
        errorGobbler.start();
        outputGobbler.start();

        // any error???
        int exitVal = p.waitFor();
        log.debug("ExitValue: " + exitVal);

        if (log.isDebugEnabled()) {
            BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
            String s = null;
            while ((s = r.readLine()) != null) {
                System.err.println(s);
            }
        }

        log.info("Finished in " + (System.currentTimeMillis() - current) / 1000l + " seconds.");
        return true;
    }


    public File getOutputDir(File inputfile) {
        if (params.containsKey(CommandLineParams.OUTPUT)) {
            return new File(params.get(CommandLineParams.OUTPUT));
        } else return new File(inputfile.getAbsolutePath() + "_files");

    }


    /**
     * returns true if cliquedir is avaiable, optionally deleting if it existis
     *
     * @param f
     * @param delete
     * @return
     * @throws IOException
     */
    public boolean checkCliqueDir(final File f, boolean delete) throws IOException {
        File[] fs = f.getParentFile().listFiles(new FileFilter() {
            public boolean accept(File file) {
                return (file.getName().startsWith(f.getName()) && !file.equals(f));
            }
        });

        if (fs.length > 0) {
            if (delete) {
                for (File td : fs) {
                    U.delete(td);
                }
            } else {
                return false;
            }

        }


        return true;
    }


}


<|start_filename|>src/edu/mit/cci/util/Filter.java<|end_filename|>
package edu.mit.cci.util;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 5/23/13
 * Time: 12:47 PM
 * To change this template use File | Settings | File Templates.
 */
public interface Filter<T> {

    public boolean accept(T obj);
}


<|start_filename|>src/edu/mit/cci/teva/util/CommunityWindow.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.teva.engine.Community;

/**
* User: jintrone
* Date: 5/14/13
* Time: 6:18 AM
*/
public class CommunityWindow {
    public int window = -1;
    public String community = null;

    public CommunityWindow(String community, int window) {
        this.community = community;
        this.window = window;
    }

    public String toString() {
        return "C("+community+")W("+window+")";
    }

    public boolean equals(Object o) {
        if (o instanceof CommunityWindow) {
            CommunityWindow other = ((CommunityWindow)o);
            return other.window == window && other.community.equals(community);
        }
        return false;
    }

    public int hashCode() {
        return CommunityWindow.class.hashCode()+window*community.hashCode()*13+7;
    }



}


<|start_filename|>src/edu/mit/cci/text/windowing/WindowStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.Date;
import java.util.List;

/**
 * A window delineates a contiguous area in a temporal data stream. WindowingStrategy provides
 * random access to an underlying windowed data stream
 *
 * User: jintrone
 * Date: 5/11/11
 * Time: 5:26 PM
 */
public interface WindowStrategy<T> {

    public int getNumberWindows();
    public List<T> getWindow(int i);
    public int[] getWindowIndices(int i);
    public Date[][] getWindowBoundaries();

    /**
     * Sets the underlying data stream to be indexed.  Expects data to be sorted.
     * @param data
     */
    public void setData(List<? extends T> data);



    public static interface Factory<T> {

        public WindowStrategy<T> getStrategy();

    }

}


<|start_filename|>src/edu/mit/cci/teva/serialization/ParameterAdapter.java<|end_filename|>
package edu.mit.cci.teva.serialization;

import edu.mit.cci.teva.engine.TevaParameters;

import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;

/**
 * User: jintrone
 * Date: 10/12/12
 * Time: 12:31 PM
 */


@XmlRootElement(name = "parameters")
public class ParameterAdapter extends XmlAdapter<ParameterAdapter,TevaParameters> {

    public ParameterAdapter(){}

    @XmlAttribute
    public float minimumLinkWeight = .5f;
    @XmlAttribute
    public int fixedCliqueSize = 4;
    @XmlAttribute
    public boolean expireConsumedCommunities = false;
    @XmlAttribute
    public String scoringMethod = "SIMILARITY";
    @XmlAttribute
    public String cfinderExecutable = "";
    @XmlAttribute
    public String filenameIdentifier = "";
    @XmlAttribute
    public String replacementDictionary = "";
    @XmlAttribute
    public String stopwordList = "";
    @XmlAttribute
    public long windowSize = -1;
    @XmlAttribute
    public long windowDelta = -1;
    @XmlAttribute
    public int wordijIndirection = -1;
    @XmlAttribute
    public int wordijTuple = -1;
    @XmlAttribute
    public long membershipWIndowSize=-1l;

    @XmlAttribute
    public long membershipWindowDelta = -1;
    @XmlAttribute
    public String workingDirectory = "";








    @Override
    public TevaParameters unmarshal(ParameterAdapter pa) throws Exception {
        TevaParameters params = new TevaParameters();
        params.setMinimumLinkWeight(pa.minimumLinkWeight);
        params.setFixedCliqueSize(pa.fixedCliqueSize);
        params.setExpireConsumedCommunities(pa.expireConsumedCommunities);
        params.setScoringMethod(pa.scoringMethod);
        params.setCFinderExecutable(pa.cfinderExecutable);
        params.setFilenameIdentifier(pa.filenameIdentifier);
        params.setReplacementDictionary(pa.replacementDictionary);
        params.setStopwordList(pa.stopwordList);
        params.setWindowSize(pa.windowSize);
        params.setWindowDelta(pa.windowDelta);
        params.setWordijIndirection(pa.wordijIndirection);
        params.setWordijTupleSize(pa.wordijTuple);
        params.setMembershipWindowSize(pa.membershipWIndowSize);
        params.setMembershipWindowDelta(pa.membershipWindowDelta);
        return params;
    }

    @Override
    public ParameterAdapter marshal(TevaParameters tp) throws Exception {
        ParameterAdapter result = new ParameterAdapter();
        result.minimumLinkWeight =tp.getMinimumLinkWeight();
        result.fixedCliqueSize = tp.getFixedCliqueSize();
        result.expireConsumedCommunities = tp.expireConsumedCommunities();
        result.scoringMethod = tp.getScoringMethod().toString();
        result.cfinderExecutable = tp.getCFinderExecutable();
        result.filenameIdentifier = tp.getFilenameIdentifier();
        result.replacementDictionary = tp.getReplacementDictionary();
        result.stopwordList = tp.getStopwordList();
        result.windowSize = tp.getWindowSize();
        result.windowDelta = tp.getWindowDelta();
        result.wordijIndirection = tp.getWordijIndirection();
        result.wordijTuple = tp.getWordijTupleSize();
        result.membershipWIndowSize = tp.getMembershipWindowSize();
        result.membershipWindowDelta = tp.getMembershipWindowDelta();
        return result;

    }
}


<|start_filename|>src/edu/mit/cci/teva/serialization/CommunityFrameJaxbAdapter.java<|end_filename|>
package edu.mit.cci.teva.serialization;

import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.teva.engine.CommunityFrame;

import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.Collection;
import java.util.HashSet;

/**
* User: jintrone
* Date: 10/8/12
* Time: 4:40 PM
*/
@XmlRootElement(name = "Frame")
public class CommunityFrameJaxbAdapter extends XmlAdapter<CommunityFrameJaxbAdapter,CommunityFrame> {

    Collection<Edge> edges;
    Collection<Clique> cliques;
    int window;

    @Override
    public CommunityFrame unmarshal(CommunityFrameJaxbAdapter jaxbAdapter) throws Exception {

        CommunityFrame result = new CommunityFrame(jaxbAdapter.getWindow());
       for (Edge e:jaxbAdapter.getEdges()) {
           result.add(e);
       }
//        for (Clique c:jaxbAdapter.getCliques()) {
//            result.addClique(c);
//        }
        return result;
    }

    @Override
    public CommunityFrameJaxbAdapter marshal(CommunityFrame communityFrame) throws Exception {
        return new CommunityFrameJaxbAdapter(communityFrame);
    }

    public CommunityFrameJaxbAdapter() {}

    public CommunityFrameJaxbAdapter(CommunityFrame frame) {
        this.edges = new HashSet<Edge>(frame.getEdges());
        this.cliques = new HashSet<Clique>(frame.getCliques());
        this.window = frame.getWindow();
    }

    public void setEdges(Collection<Edge> edges) {
        this.edges = edges;
    }
    @XmlElementWrapper(name = "edges")
    @XmlElement(name = "edge")
    public Collection<Edge> getEdges() {
        return edges;
    }

    public Collection<Clique> getCliques() {
        return cliques;
    }

    public void setCliques(Collection<Clique> cliques) {
        this.cliques = cliques;
    }

    @XmlAttribute
    public int getWindow() {
        return window;
    }

    public void setWindow(int window) {
        this.window = window;
    }
}


<|start_filename|>src/edu/mit/cci/sna/impl/EdgeImpl.java<|end_filename|>
package edu.mit.cci.sna.impl;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Node;
import org.apache.commons.collections15.Factory;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.HashMap;
import java.util.Map;

/**
 * User: jintrone
 * Date: 4/20/11
 * Time: 2:26 PM
 */

@XmlRootElement(name="Node")
@XmlAccessorType(XmlAccessType.NONE)
public class EdgeImpl implements Edge {

    private static int idgen = 0;

    private float weight = 0;

    private Map<String, Object> attributes = new HashMap<String, Object>();

    private boolean directed = false;

    private Node[] endpoints;




    private EdgeImpl() {
        //throw new RuntimeException("Fix any clases that depend upon the no-arg constructor in EdgeImpl");
    }

    public EdgeImpl(Node from, Node to) {
        this(from, to, 1.0f, false);
    }

    public EdgeImpl(Node from, Node to, float weight, boolean directed) {

        this.directed = directed;
        this.setWeight(weight);
        this.endpoints = new Node[]{from, to};
    }

    @XmlAttribute(name = "weight")
    public void setWeight(float weight) {
        this.weight = weight;
    }

    public float getWeight() {
        return weight;
    }

    public boolean equals(Object o) {
        if (o instanceof Edge) {
            Edge e = (Edge) o;
            return ((getEndpoints()[0].equals(e.getEndpoints()[0]) && getEndpoints()[1].equals(e.getEndpoints()[1])) ||
                    (!e.isDirected() && !isDirected() && (getEndpoints()[1].equals(e.getEndpoints()[0]) && getEndpoints()[0].equals(e.getEndpoints()[1]))));
        }
        return false;
    }

    public int hashCode() {
        return (getEndpoints()[0].hashCode() + getEndpoints()[1].hashCode()) * 7 + 13;
    }


    public boolean isDirected() {
        return directed;
    }


    public void setProperty(String key, Object val) {
        attributes.put(key, val);
    }

    @XmlElementWrapper(name = "endpoints")
    @XmlElement(name = "node")
    public Node[] getEndpoints() {
        return endpoints;
    }

    public void setEndpoints(Node[] endpoints) {
        this.endpoints = endpoints;
    }



    public Map<String, Object> getProperties() {
        return attributes;
    }

    public Object getProperty(String key) {
        return attributes.get(key);
    }

    public static Factory<EdgeImpl> getFactory() {
        return new Factory<EdgeImpl>() {

            public EdgeImpl create() {
                return new EdgeImpl();
            }
        };
    }


    public String toString() {
        return getEndpoints()[0] + "->" + getEndpoints()[1] + "(" + getWeight() + ")";
    }


    public static class JaxbAdapter extends XmlAdapter<EdgeImpl, Edge> {

        @Override
        public Edge unmarshal(EdgeImpl edge) throws Exception {
            return edge;
        }

        @Override
        public EdgeImpl marshal(Edge edge) throws Exception {
            //TODO generalize this
            return (EdgeImpl) edge;
        }
    }
}


<|start_filename|>src/edu/mit/cci/teva/util/SimilarityBasedAssignment.java<|end_filename|>
package edu.mit.cci.teva.util;

import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.teva.engine.*;
import edu.mit.cci.text.windowing.Bin;
import edu.mit.cci.text.windowing.Windowable;
import org.apache.log4j.Logger;

import java.util.*;

/**
 * Created with IntelliJ IDEA.
 * User: josh
 * Date: 8/30/13
 * Time: 1:58 PM
 * To change this template use File | Settings | File Templates.
 */
public class SimilarityBasedAssignment implements CommunityMembershipStrategy {

    Logger log = Logger.getLogger(SimilarityBasedAssignment.class);

    @Override
    public List<Assignment> assignToCommunity(CommunityModel communities, int window, Network net) {
        Assignment best = null;
        for (Community c : communities.getCommunities()) {
            float coverage = NetworkUtils.coverage(net, c.getCommunityAtBin(window));
            float similarity = NetworkUtils.similarity(net, c.getCommunityAtBin(window));

            //TODO handle the unlikely possibility there could be more than one match here.
            if (coverage > 0) {

                if (best == null) {
                    best = new Assignment(c, coverage, similarity);
                } else if (best.coverage < coverage || best.coverage == coverage && best.similarity < similarity) {
                    best.community = c;
                    best.coverage = coverage;
                    best.similarity = similarity;
                }
            }


        }
        if (best == null) {
            log.info("Could not identify topic for posts");
            return Collections.emptyList();
        } else {
            return Collections.singletonList(best);


        }

    }
}


<|start_filename|>src/edu/mit/cci/text/windowing/TimeBasedSlidingWindowStrategy.java<|end_filename|>
package edu.mit.cci.text.windowing;

import org.apache.log4j.Logger;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;

/**
 * Slides a window of some width at a given delta across a sample. Guarantees that data at
 * the max time will be included in a window.  Windows are inclusive of the start,
 * exclusive of their end
 *
 * User: jintrone
 * Date: 5/23/11
 * Time: 6:18 PM
 */
public class TimeBasedSlidingWindowStrategy implements WindowStrategy<Windowable> {

    int numWindows = 1;
    List<? extends Windowable> current;
    Date min, max;
    long size;
    long delta;

    Logger log = Logger.getLogger(TimeBasedSlidingWindowStrategy.class);


    public TimeBasedSlidingWindowStrategy(Date min, Date max, long size, long delta) {
        //this is so embarrassing
        long x = min.getTime();
        while (x+size <= max.getTime()) {
            numWindows++;
            x+=delta;
        }
        this.min = min;
        this.max = max;
        this.size = size;
        this.delta = delta;
    }


    public int getNumberWindows() {
        return numWindows;
    }

    public void setData(List<? extends Windowable> args) {
        this.current = args;
    }

    public Date[] getWindowBounds(int idx) {
       Date start = new Date(min.getTime() + (idx * delta));
       return new Date[] {start,new Date(start.getTime()+size)};

    }

    public List<Windowable> getWindow(int idx) {


        int[] indices = getWindowIndices(idx);
        return indices==null? Collections.<Windowable>emptyList() :new ArrayList<>(current.subList(indices[0], indices[1]));

    }

    @Override
    public int[] getWindowIndices(int idx) {
        Date beginning = getWindowBounds(idx)[0];
        Date end = getWindowBounds(idx)[1];


        int first = -1;
        int last = -1;
        if (current.get(0).getStart().after(end) || current.get(current.size() - 1).getStart().before(beginning))
            return null;
        for (int i = 0; i < current.size(); i++) {
            if (current.get(i).getStart().before(beginning)) continue;
            else if (first < 0) {
                first = i;
            }
            if (current.get(i).getStart().equals(end) || current.get(i).getStart().after(end)) {
                last = i;
                break;
            }


        }
        if (first < 0) return null;
        if (last < 0) {
            last = current.size();
        }
        return new int[]{first,last};
    }

    public Date[][] getWindowBoundaries() {
       List<Date[]> dates = new ArrayList<Date[]>();
        for (int i = 0;i<getNumberWindows();i++) {
            dates.add(getWindowBounds(i));
        }
        return dates.toArray(new Date[dates.size()][]);

    }




}


<|start_filename|>src/edu/mit/cci/text/windowing/Windowable.java<|end_filename|>
package edu.mit.cci.text.windowing;

import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.Date;
import java.util.List;

/**
* User: jintrone
* Date: 9/24/12
* Time: 11:42 PM
*/

@XmlJavaTypeAdapter(BasicWindowable.JaxbAdapter.class)
public interface Windowable {
    public Date getStart();
    public List<String> getTokens();
    public String getRawData();
    public String getId();
}


<|start_filename|>src/edu/mit/cci/text/windowing/BasicBin.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.ArrayList;
import java.util.Collection;

/**
 * User: jintrone
 * Date: 4/18/13
 * Time: 1:00 PM
 */
public class BasicBin<T> extends ArrayList<T> implements Bin<T> {

    int firstIndex = 0;

    public BasicBin() {
    }

    public BasicBin(Collection<? extends T> ts) {
        super(ts);
    }

    public BasicBin(int idx, Collection<? extends T> ts) {
        super(ts);
        firstIndex = idx;
    }

    public BasicBin(int i) {
        super(i);
    }

    public void setFirstItemIndex(int idx) {
        this.firstIndex = idx;
    }

    public int getFirstItemIndex() {
        return firstIndex;
    }

    public String toString() {
        StringBuilder builder = new StringBuilder();
        builder.append("[");
        String sep = "";
        for (int i=0;i<this.size();i++) {

            builder.append(sep);

            if (i == getFirstItemIndex()) {
                builder.append("*");
            }
            builder.append(this.get(i));
            sep = ", ";

        }
        builder.append("]");
        return builder.toString();
    }

}


<|start_filename|>src/edu/mit/cci/teva/engine/ConversationChunk.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Edge;
import edu.mit.cci.text.windowing.Windowable;

import javax.xml.bind.annotation.*;
import java.util.Collection;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/14/12
 * Time: 11:02 PM
 */

@XmlRootElement(name = "chunk")
@XmlAccessorType(XmlAccessType.NONE)
public class ConversationChunk {

    @XmlElement(name = "post")
    public List<Windowable> messages;

    @XmlAttribute
    public float coverage;
    @XmlAttribute
    public float similarity;

    public Collection<Edge> edges;


    @XmlElementWrapper(name = "edges")
    @XmlElement(name = "edge")
    public Collection<Edge> getEdges() {
        return edges;
    }

    @XmlAttribute
    public int window;

    public ConversationChunk() {}

    public ConversationChunk(List<Windowable> messages, int window, float coverage, float similarity) {
        this.coverage = coverage;
        this.window = window;
        this.similarity = similarity;
        this.messages = messages;
    }

    public ConversationChunk(List<Windowable> messages, int window, float coverage, float similarity,Collection<Edge> edges) {
        this(messages,window,coverage,similarity);
        this.edges = edges;
    }

    public void setEdges(Collection<Edge> edges) {
        this.edges = edges;
    }





}


<|start_filename|>src/edu/mit/cci/util/F.java<|end_filename|>
package edu.mit.cci.util;

/**
 * User: jintrone
 * Date: 5/17/13
 * Time: 9:38 AM
 */
public interface F<T> {

    public void apply(T t);
}


<|start_filename|>src/edu/mit/cci/teva/cpm/cfinder/CFinderReader.java<|end_filename|>
package edu.mit.cci.teva.cpm.cfinder;

import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.CliqueDecoratedNetwork;
import edu.mit.cci.sna.CliqueDecorater;
import edu.mit.cci.sna.impl.EdgeImpl;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.engine.CommunityFinderException;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * User: jintrone
 * Date: 4/19/12
 * Time: 3:45 AM
 */
public class CFinderReader {

    private static Logger log = Logger.getLogger(CFinderReader.class);


    public static List<CliqueDecoratedNetwork> readCommunities(int cliquesize, File basedir) throws IOException, CommunityFinderException {
        File target = new File(basedir, "k=" + cliquesize);
        Map<Integer, Clique> cliques = processGlobalCliques(basedir);
        File communitiesFile = new File(target, "communities_links");
        boolean intensity = false;
        if (!communitiesFile.exists()) {
            communitiesFile = new File(target, "intensity_communities_links");
            if (!communitiesFile.exists()) {
                throw new CommunityFinderException("Could not identify communities file");

            } else {
                intensity = true;
            }
        }

        List<CliqueDecoratedNetwork> infos = new ArrayList<CliqueDecoratedNetwork>();

        BufferedReader communityReader = new BufferedReader(new FileReader(communitiesFile));
        BufferedReader cliqueReader = new BufferedReader(new FileReader(new File(target, intensity ? "intensity_communities_cliques" : "communities_cliques")));

        String line;
        List<String> links = new ArrayList<String>();
        Pattern commPattern = Pattern.compile("^-?(\\d+):$");
        CliqueDecoratedNetwork cinfo = null;
        String communityId = "";

        while ((line = communityReader.readLine()) != null) {
            if (line.startsWith("#") || line.isEmpty()) continue;
            Matcher m = commPattern.matcher(line);
            if (m.matches()) {
                log.debug("Match line: "+line);
                if (cinfo != null) {
                    cinfo.addCliques(readCliques(communityId, cliqueReader, cliques));
                    log.debug("Adding info for community: "+communityId);
                    infos.add(cinfo);
                }
                cinfo = new CliqueDecorater(new UndirectedJungNetwork());
                communityId = m.group(1);


            } else {
                String[] n = line.trim().split(" ");
                if (cinfo == null) {
                    throw new CommunityFinderException("Error processing community file:" + communitiesFile.getAbsolutePath());
                }
                cinfo.add(new EdgeImpl(new NodeImpl(n[0]), new NodeImpl(n[1]), 1.0f, false));
            }
        }


        if (cinfo != null) {
            log.debug("Adding info for community: "+communityId);
            infos.add(cinfo);
            cinfo.addCliques(readCliques(communityId, cliqueReader, cliques));
        }
        communityReader.close();
        cliqueReader.close();
        return infos;
    }

    public static Map<Integer, Clique> processGlobalCliques(File outputdir) throws IOException {
        Map<Integer, Clique> result = new HashMap<Integer, Clique>();
        File f = new File(outputdir, "cliques");
        if (!f.exists()) {
            log.warn("Can't identify clique file");
            return null;
        }

        Pattern p = Pattern.compile("^(\\d+):(.+)$");

        BufferedReader reader = new BufferedReader(new FileReader(f));
        String line;

        while ((line = reader.readLine()) != null) {
            if (line.startsWith("#") || line.isEmpty()) continue;
            Matcher m = p.matcher(line);
            if (m.matches()) {
                Clique c = new Clique(Integer.parseInt(m.group(1)));

                c.setNodes(Arrays.asList(U.mysplit(m.group(2), "\\s+")));

                result.put(c.getId(), c);
            }
        }
        return result;
    }

    public static List<Clique> readCliques(String id, BufferedReader reader, Map<Integer, Clique> gcliques) throws IOException {
        String line = null;
        Pattern p = Pattern.compile("^(\\d+):(.+)$");
        List<Clique> result = new ArrayList<Clique>();
        while ((line = reader.readLine()) != null) {
            if (line.startsWith("#") || line.isEmpty()) continue;
            Matcher m = p.matcher(line);
            if (m.matches()) {
                if (!m.group(1).equals(id)) {
                    log.warn("Could not identify community " + id + " in clique cliquelist; continuing");
                } else {
                    for (String cid : m.group(2).split("\\s+")) {
                        if (cid.trim().isEmpty()) continue;
                        result.add(gcliques.get(Integer.parseInt(cid)));
                    }
                    break;
                }


            }

        }
        return result;
    }


    public static void main(String[] args) throws IOException, CommunityFinderException {
        readCommunities(4, new File("/Users/jintrone/Documents/DEV/REASONTEVA/work/CFinderNetwork.TEvA.0.net_files"));
    }


}


<|start_filename|>src/edu/mit/cci/text/preprocessing/TextTransformable.java<|end_filename|>
package edu.mit.cci.text.preprocessing;

/**
 * User: jintrone
 * Date: 9/27/12
 * Time: 3:56 PM
 */
public interface TextTransformable {

    public String transform();
}


<|start_filename|>src/edu/mit/cci/teva/engine/NumTokensMembershipEngine.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;
import edu.mit.cci.teva.TevaFactory;
import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.DiscussionThread;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.util.WindowablePostAdapter;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.text.wordij.TextToNetworkGenerator;

import java.io.IOException;
import java.util.*;

/**
 * Created by josh on 1/14/14.
 */
public class NumTokensMembershipEngine implements TopicMembershipEngine {


    private final TevaFactory factory;
    private final Conversation conversation;
    private final CommunityModel model;
    private Map<String, PostTokens> postmap;
    private Tokenizer<String> tokenizer;
    private TextToNetworkGenerator generator;

    private Map<String, String> communityAssignmentMap;

    private CommunityMembershipStrategy strategy;

    private int numTokens = 1;

    public NumTokensMembershipEngine(CommunityModel model, Conversation conversation,  TevaFactory factory, int numTokens) {
        this.factory = factory;
        this.conversation = conversation;
        this.model = model;
        try {
            tokenizer = factory.getTokenizer();
        } catch (IOException e) {
            e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            throw new RuntimeException(e);
        }
        generator = factory.getNetworkCalculator();
        this.numTokens = numTokens;
        strategy = factory.getMembershipMatchingStrategy();
    }



    public void process() throws IOException {
        communityAssignmentMap = new HashMap<String, String>();
        Date[][] windows = model.getWindows();
        List<Date> ends = new ArrayList<Date>();
        postmap = new HashMap<String, PostTokens>();
        for (Date[] w : windows) {
            ends.add(w[1]);
        }

        for (DiscussionThread t : conversation.getAllThreads()) {
            for (Post p : t.getPosts()) {
                postmap.put(p.getPostid(), new PostTokens(p,tokenizer.tokenize(p.getContent())));
                int win = Collections.binarySearch(ends, p.getTime());
                if (win < 0) {
                    win = -(win + 1);
                }

                findCommunityInWindows(p, win);
            }
        }

    }

    public List<String> getTokens(Post p) {
        List<String> tokens = new ArrayList<String>();
        PostTokens pt = postmap.get(p.getPostid());
        while (tokens.size()<numTokens && pt!=null) {
            if (pt.tokens!=null) {tokens.addAll(pt.tokens);}
            pt = postmap.get(pt.post.getReplyToId());
        }
        return tokens;
    }

    public void findCommunityInWindows(Post p, int win) {

        List<String> tokens =  getTokens(p);
        Network net = generator.calculateWeights(null, tokens);
        List<CommunityMembershipStrategy.Assignment> assignments = strategy.assignToCommunity(model,win,net);
        for (CommunityMembershipStrategy.Assignment a: assignments) {
            a.community.addAssignment(new ConversationChunk(Collections.singletonList((Windowable)new WindowablePostAdapter(p,tokenizer)), win, a.coverage, a.similarity,a.edges));
        }

    }

    private class PostTokens {
        Post post;
        List<String> tokens;

        PostTokens(Post p, List<String> tokens) {
            this.post = p;
            this.tokens = tokens;
        }


    }
}


<|start_filename|>src/edu/mit/cci/text/wordij/ParallelCorpusToNetworkGenerator.java<|end_filename|>
package edu.mit.cci.text.wordij;

import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.text.windowing.BinningStrategy;
import edu.mit.cci.text.windowing.Windowable;
import edu.mit.cci.util.ThreadPoolRunner;

/**
 * Created by josh on 7/16/14.
 */
public class ParallelCorpusToNetworkGenerator<T extends Windowable> extends TextCorpusNetworkGenerator<T> {


    private ThreadPoolRunner pool = new ThreadPoolRunner(8);

    public ParallelCorpusToNetworkGenerator(BinningStrategy bmodel, TextToNetworkGenerator calculator) {
        super(bmodel, calculator);
    }

    public ParallelCorpusToNetworkGenerator(BinningStrategy<T> bmodel,
                                    TextToNetworkGenerator calculator, boolean overwrite, JungUtils.MergePolicy merger) {
       super(bmodel,calculator,overwrite,merger);

    }












}


<|start_filename|>src/edu/mit/cci/sna/Node.java<|end_filename|>
package edu.mit.cci.sna;

import edu.mit.cci.sna.impl.NodeImpl;

import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.Map;

/**
 * User: jintrone
 * Date: 9/21/12
 * Time: 9:06 PM
 */

@XmlJavaTypeAdapter(NodeImpl.JaxbAdapter.class)
public interface Node {

    public String getLabel();
    public void setLabel(String s);

    public String getId();

    public void setProperty(String property, Object value);

    public Object getProperty(String property);

    public Map<String,Object> getProperties();


}


<|start_filename|>src/edu/mit/cci/teva/engine/EvolutionStepStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;

import java.util.List;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 9:33 PM
 */
public interface EvolutionStepStrategy {
    void processStep(int i, List<CommunityFrame> fromcliques, List<Network> mergedcliques, List<CommunityFrame> tocliques);

}


<|start_filename|>src/edu/mit/cci/teva/model/Post.java<|end_filename|>
package edu.mit.cci.teva.model;

import java.util.Date;
import java.util.List;

/**
 * User: jintrone
 * Date: 8/10/11
 * Time: 5:20 PM
 */
public interface Post  {
    String getThreadid();

    String getPostid();

    String getReplyToId();

    String getUserid();

    Date getTime();

    List<String> getSentiment();

    String getContent();


}


<|start_filename|>src/edu/mit/cci/text/windowing/Bin.java<|end_filename|>
package edu.mit.cci.text.windowing;

import java.util.Collection;
import java.util.List;

/**
 * A bin is a simple wrapper around a list that provides a hint (the "first item index") to the network
 * generator to indicate which element is the first underlying item to be processed.  The preceding items
 * will be used to generate initial connections.
 *
 * User: jintrone
 * Date: 4/18/13
 * Time: 9:34 AM
 */
public interface Bin<T> extends List<T> {

    public int getFirstItemIndex();
    public void setFirstItemIndex(int i);

}


<|start_filename|>src/edu/mit/cci/teva/analysis/DumpTopicsToCsvLong.java<|end_filename|>
package edu.mit.cci.teva.analysis;

import edu.mit.cci.sna.Node;
import edu.mit.cci.teva.engine.Community;
import edu.mit.cci.teva.engine.CommunityModel;
import edu.mit.cci.teva.util.Stringer;

import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/12/12
 * Time: 2:02 PM
 */
public class DumpTopicsToCsvLong extends DumpTopicsToCsv {

    public List<String> headers = new ArrayList<String>(Arrays.asList("From", "To","Spawns","Consumes"));

    public DumpTopicsToCsvLong(CommunityModel model, OutputStream stream) {
        super(model, stream);

    }

    public void write() {
        for (Community c : model.getCommunities()) {
            headers.add("Topic" + c.getId());
        }
        Date[][] wins = model.getWindows();
        printLine(headers.toArray());
        List<Object> line = new ArrayList<Object>();

        for (int i = 0; i < wins.length; i++) {
            line.clear();
            line.add(wins[i][0]);
            line.add(wins[i][1]);
            line.add(model.getConnection(i, CommunityModel.ConnectionType.SPAWNS));
            line.add(model.getConnection(i, CommunityModel.ConnectionType.CONSUMES));


            for (Community c : model.getCommunities()) {
                //Collection<String> s = c.getCommunityAtBin(i) == null ? Collections.<String>emptyList() : TevaUtils.getCommunityRepresentation(c.getCommunityAtBin(i), 5);
                String l = flatten(c.getCommunityAtBin(i) == null ? Collections.<Node>emptyList() : c.getCommunityAtBin(i).getNodes(), new Stringer<Node>() {

                    public String stringify(Node object) {
                        return object.getLabel();
                    }
                });
                line.add(l);

            }
            printLine(line.toArray());
        }
        writer.flush();
        writer.close();

    }

}


<|start_filename|>src/edu/mit/cci/teva/model/DiscussionThread.java<|end_filename|>
package edu.mit.cci.teva.model;

import java.util.List;

/**
 * User: jintrone
 * Date: 8/10/11
 * Time: 5:28 PM
 */
public interface DiscussionThread {

    public String getThreadId();

    /**
     * Returns a sorted list of posts
     *
     * @return
     */
    public List<Post> getPosts();

    public List<String> getSentiment(Post p);
}


<|start_filename|>src/edu/mit/cci/teva/engine/CommunityFinder.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Network;

import java.io.File;
import java.util.List;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 11:41 AM
 */
public interface CommunityFinder {

    List<CommunityFrame> findCommunities(File currentGraph, int cliqueSizeAtWindow, int window) throws CommunityFinderException;
    List<CommunityFrame> findCommunities(Network currentGraph, int cliqueSizeAtWindow, int window, String id) throws CommunityFinderException;
}


<|start_filename|>src/edu/mit/cci/teva/engine/FastMergeStrategy.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.Clique;
import edu.mit.cci.sna.Edge;
import edu.mit.cci.sna.Network;
import edu.mit.cci.sna.NetworkUtils;
import edu.mit.cci.sna.Node;
import edu.mit.cci.sna.impl.NodeImpl;
import edu.mit.cci.sna.jung.UndirectedJungNetwork;
import edu.mit.cci.teva.cpm.cfinder.CFinderCommunityFinder;
import edu.mit.cci.util.U;
import edu.uci.ics.jung.algorithms.cluster.WeakComponentClusterer;
import org.apache.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 6/20/12
 * Time: 6:55 AM
 */
public class FastMergeStrategy implements MergeStrategy {

    private static Logger log = Logger.getLogger(FastMergeStrategy.class);

    int k = -1;

    private Map<Clique, Node> cliqueAdjacencyNodeMap = new HashMap<Clique, Node>();


    public FastMergeStrategy(int k) {

        this.k = k;

    }


    /**
     * Walks through the maximal cliques that make up the communities in either frame,
     * and merges them where appropriate, or extends them if abutting edges that are not already
     * in cliques should be added.
     *
     *
     * @param from
     * @param cdfrom
     * @param to
     * @param cdto
     * @param window
     * @return
     */
    public List<Network> process(Network from, Collection<CommunityFrame> cdfrom, Network to, Collection<CommunityFrame> cdto, int window)  {
        cliqueAdjacencyNodeMap.clear();


        UndirectedJungNetwork merged = new UndirectedJungNetwork();


        Set<Network> cdFrom = new HashSet<Network>(cdfrom);
        Set<Network> cdTo = new HashSet<Network>(cdto);

        Set<Node> nodeset = new HashSet<Node>();
        Map<Set<Node>, Edge> edgemap = new HashMap<Set<Node>, Edge>();
        Set<Set<Node>> newEdges = new HashSet<Set<Node>>();

        log.debug("Start state");
        log.debug("From network: "+from.getNodes().size()+" nodes, "+from.getEdges().size()+" edges, "+cdfrom.size()+"communities");
        log.debug("To network: "+to.getNodes().size()+" nodes, "+to.getEdges().size()+" edges, "+cdto.size()+"communities");

        //create new graph, separating shared edges from unshared edges during the merge process
        int fedges = from.getEdges().size();

        //first loop just merges the networks, and builds up a bunch of bookkeeping
        for (Edge e : U.multiIterator(from.getEdges(), to.getEdges())) {
            boolean second = fedges-- < 0;


            Set<Node> pair = new HashSet<Node>();
            pair.add(e.getEndpoints()[0]);
            pair.add(e.getEndpoints()[1]);


            //segregate edges
            if (!second) {
                newEdges.add(pair);
            } else {
                if (!newEdges.remove(pair)) {
                    newEdges.add(pair);
                }
            }


            Edge existing = edgemap.get(pair);
            if (existing == null) {
                Node node1 = e.getEndpoints()[0], node2 = e.getEndpoints()[1];
                if (!nodeset.contains(node1)) {
                    merged.addVertex(node1);
                    nodeset.add(node1);
                }

                if (!nodeset.contains(node2)) {
                    merged.addVertex(node2);
                    nodeset.add(node2);

                }

                existing = merged.addEdge(node1, node2, e.getWeight());

            } else {

                //update weight if necessary
                if (e.getWeight() > existing.getWeight()) {
                    existing.setWeight(e.getWeight());
                }
            }

            //build map of existing edges in merged
            edgemap.put(pair, existing);
        }

        log.debug("Newedges: "+newEdges.size());
        log.debug("Merged size:"+merged.getVertexCount()+" nodes, "+merged.getEdgeCount()+"  edges");
        log.debug("edgemap size "+edgemap.size());
        log.debug("nodeset size "+nodeset.size());
        //At this point:
        //newedges should contain just those edges that are unshared between the two graphs
        //merged contains the merged network
        //edgemap is a map of pairs of nodes to edges in merged
        //nodeset is just a set of nodes

        log.info("Done merging networks");

        HashMap<Clique, Clique> unsharedCliques = new HashMap<Clique, Clique>();
        HashMap<Clique, Clique> unsharedCliques2 = new HashMap<Clique, Clique>();
        HashMap<Clique, Clique> sharedCliques = new HashMap<Clique, Clique>();
        Set<Set<String>> cliqueEdges = new HashSet<Set<String>>();
        int i = 0;

        UndirectedJungNetwork cliqueAdjacencyGraph = new UndirectedJungNetwork();

        for (Network cd : cdFrom) {
            Set<Clique> curr = new HashSet<Clique>();
            for (Clique c : ((CommunityFrame) cd).getCliques()) {
                Clique nc = new Clique(i++, c.getNodes());
                curr.add(nc);
                unsharedCliques.put(nc, nc);
                cliqueEdges.addAll(c.getEdgeSet());
            }
            addAdjacentCliques(curr, cliqueAdjacencyGraph);

        }
        log.debug("CAG: " + cliqueAdjacencyGraph.getVertexCount() + " nodes, "+cliqueAdjacencyGraph.getEdgeCount() + " edges");

        //cliqeuAdjacencyGraph is a graph of all cliques from cdFrom - representing existing communities
        //unsharedCliques contains all cliques from cdFrom
        //cliquedge contains all edges

        for (Network cd : cdTo) {
            Set<Clique> curr = new HashSet<Clique>();
            for (Clique c : ((CommunityFrame) cd).getCliques()) {

                Clique tomove = unsharedCliques.remove(c);
                if (tomove != null) {
                    curr.add(tomove);
                    sharedCliques.put(tomove, tomove);
                } else {
                    Clique nc = new Clique(i++, c.getNodes());
                    curr.add(nc);
                    //maybe not necessary?
                    unsharedCliques2.put(nc, nc);
                    cliqueEdges.addAll(nc.getEdgeSet());

                }

            }
            addAdjacentCliques(curr, cliqueAdjacencyGraph);
        }
          log.debug("CAG: " + cliqueAdjacencyGraph.getVertexCount() + " nodes, "+cliqueAdjacencyGraph.getEdgeCount() + " edges");
        //cliqueAdjacencyGraph now is a graph of all cliques from merging cdFrom & cdTo
        //sharedCliques contains cliques in both networks
        // /unsharedCliques (1 & 2) contains cliques unique to each of the two graphs respectively
        //cliqueEdges are all edges in any clique in either graph

        //prune edges we already know about
        newEdges.removeAll(cliqueEdges);
        //now newedges is just those edges unshared between the two graphs that are not in any clique;
        //these are the ones we need to look at


        //prune edges connecting nodes of insufficient degree
        for (Iterator<Set<Node>> it = newEdges.iterator(); it.hasNext(); ) {
            Set<Node> s = it.next();
            for (Node n : s) {
                if (merged.degree(n) < k - 1) {
                    it.remove();
                    break;
                }

            }

            // findCliques(edgemap.get(s))


        }
        log.debug("Found " + newEdges.size() + " edges to be processed");

        Set<Clique> ncliques = new HashSet<Clique>();
        for (Set<Node> edge : newEdges) {
            Edge e = edgemap.get(edge);
            List<Node> l = Arrays.asList(merged.getEndpoints(e).getFirst(), merged.getEndpoints(e).getSecond());
            ncliques.addAll(NetworkUtils.findCliquesContaining(merged, new HashSet<Node>(l), k));
        }

        log.debug("Found " + ncliques.size() + " new cliques");
        log.debug("CAG: " + cliqueAdjacencyGraph.getVertexCount() + " nodes, "+cliqueAdjacencyGraph.getEdgeCount() + " edges");
        //build linkes between any cliques that may need to be connected
        //brute force, could simplify
        for (Clique u1 : unsharedCliques.keySet()) {
            for (Clique u2 : unsharedCliques2.keySet()) {
                if (u1.overlap(u2) >= k - 1) {
                    Node n1 = cliqueAdjacencyNodeMap.get(u2);
                    Node n2 = cliqueAdjacencyNodeMap.get(u1);
                   // log.debug("Adding new edge between cliques");
                    cliqueAdjacencyGraph.addEdge(n1, n2, 1.0f);
                }
            }
        }

        log.debug("CAG: " + cliqueAdjacencyGraph.getVertexCount() + " nodes, "+cliqueAdjacencyGraph.getEdgeCount() + " edges");
        //add new cliques to graph and build any necessary connections
        //brute force, could simplify
        for (Clique c : ncliques) {
            Node nnode = new NodeImpl(c.getId() + "");
            cliqueAdjacencyGraph.addVertex(nnode);
            nnode.setProperty("clique", c);
            cliqueAdjacencyNodeMap.put(c, nnode);

            for (Node enode : cliqueAdjacencyGraph.getVertices()) {
                if (enode == nnode) continue;
                Clique e = (Clique) enode.getProperty("clique");
                if (e.overlap(c) >= k - 1) {
                   // log.debug("Adding new edge with emergent clique");
                    cliqueAdjacencyGraph.addEdge(enode, nnode, 1.0f);

                }
            }
        }
        log.debug("CAG: " + cliqueAdjacencyGraph.getVertexCount() + " nodes, "+cliqueAdjacencyGraph.getEdgeCount() + " edges");

        //the clique adjacency graph now has all communities in the merged graph; extract all connected components

        WeakComponentClusterer<Node, Edge> components = new WeakComponentClusterer<Node, Edge>();
        Set<Set<Node>> comps = components.transform(cliqueAdjacencyGraph);

        log.debug("Found "+comps.size()+" components");

        Set<Network> result = new HashSet<Network>();
        Set<Node> nodes = new HashSet<Node>(cliqueAdjacencyGraph.getVertices());
        Set<Clique> cliques = new HashSet<Clique>();
        //System.out.println("Identified "+comps.size()+" components ");
        for (Set<Node> community : comps) {
            UndirectedJungNetwork d = new UndirectedJungNetwork();
            for (Node n : community) {
                Clique c = (Clique) n.getProperty("clique");
                cliques.add(c);
                // d.addClique(c);
                nodes.remove(n);

                for (Set<String> e : c.getEdgeSet()) {
                    String[] s = e.toArray(new String[2]);
                    d.addEdge(new NodeImpl(s[0]), new NodeImpl(s[1]), 1.0f);
                }
            }
            result.add(d);
        }

        log.info(nodes.size() + " of " + cliqueAdjacencyGraph.getVertexCount() + " cliques unaccounted for");
        log.info("Found " + cliques.size() + " unique cliques in " + result.size() + "components");
        return new ArrayList<Network>(result);


    }

    //build a graph of adjacent cliques
    public void addAdjacentCliques(Set<Clique> cliques, UndirectedJungNetwork graph) {
        List<Clique> cliqit = new ArrayList<Clique>(cliques);
        for (int i = 0; i < cliqit.size(); i++) {
            Clique c = cliqit.get(i);
            Node n = cliqueAdjacencyNodeMap.get(c);
            if (n == null) {
                n = new NodeImpl(c.getId() + "");
                n.setProperty("clique", c);
                graph.addVertex(n);
                cliqueAdjacencyNodeMap.put(c, n);

            }
            for (int j = i + 1; j < cliqit.size(); j++) {
                Clique c2 = cliqit.get(j);
                Node n2 = cliqueAdjacencyNodeMap.get(c2);
                if (n2 == null) {
                    n2 = new NodeImpl(c2.getId() + "");
                    n2.setProperty("clique", c2);
                    graph.addVertex(n2);
                    cliqueAdjacencyNodeMap.put(c2, n2);

                }

                if (graph.findEdge(n, n2) == null && c.overlap(c2) >= k - 1) graph.addEdge(n, n2, 1.0f);

            }

        }

    }

    public static void main(String args[]) throws CommunityFinderException, IOException {
        TevaParameters param = new TevaParameters();
        param.setProperty(TevaParameters.WORKING_DIRECTORY, "../REASONTEVA/work");

       // for (int i = 0; i < 30; i++) {


            CFinderCommunityFinder finder = new CFinderCommunityFinder(false, false, new TevaParameters());
            File one = new File("../REASONTEVA/work/CFinderNetwork.TEvA.0.net");
            File two = new File("../REASONTEVA/work/CFinderNetwork.TEvA.1.net");

            Network none = NetworkUtils.readNetworkFile(one);
            Network ntwo = NetworkUtils.readNetworkFile(two);

            List<CommunityFrame> frames1 = finder.findCommunities(one, 4, 0);
            List<CommunityFrame> frames2 = finder.findCommunities(two, 4, 1);

            FastMergeStrategy strategy = new FastMergeStrategy(4);
            List<Network> result = strategy.process(none, frames1, ntwo, frames2, 0);
            for (Network n : result) {
                //log.info("Network: " + n.getNodes());
            }
       // }


    }


}


<|start_filename|>src/edu/mit/cci/teva/engine/Community.java<|end_filename|>
package edu.mit.cci.teva.engine;


import edu.mit.cci.teva.serialization.CommunityFrameMapJaxbAdapter;

import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlID;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * User: jintrone
 * Date: 10/1/12
 * Time: 10:32 PM
 */

@XmlRootElement(name = "Community")
@XmlAccessorType(XmlAccessType.NONE)
public class Community {

    public static int ID_COUNT = 0;


    @XmlElement(name = "assignment")
    private List<ConversationChunk> assignments = new ArrayList<ConversationChunk>();


    @XmlJavaTypeAdapter(CommunityFrameMapJaxbAdapter.class)
    public Map<Integer, CommunityFrame> history = new HashMap<Integer, CommunityFrame>();

    @XmlID
    public String id;

    private int _id;

    public boolean expired = false;

    public boolean isExpired() {
        return expired;
    }

    public void expire() {
        this.expired = true;
    }


    private int maxbin = -1;
    private int minbin = -1;

    public static void reset() {
        ID_COUNT = 0;
    }

    public String getId() {
        return id;
    }

    private Community() {


    }

    public static Community create() {
        return new Community(ID_COUNT++);
    }

    public Community(int id) {
        this._id =id;
        this.id =  "c"+_id;
        ID_COUNT = Math.max(ID_COUNT, _id);

    }

    public int getNumericId() {
        return _id;
    }


    public void addFrame(CommunityFrame frame) {
        frame.setCommunity(this);
        history.put(frame.getWindow(), frame);
        maxbin = Math.max(frame.getWindow(), maxbin);
        minbin = minbin == -1 ? frame.getWindow() : Math.min(frame.getWindow(), minbin);
    }


    public int getMaxBin() {
        if (maxbin == -1 && !history.isEmpty()) {
            for (int key:history.keySet()) {
                maxbin = Math.max(maxbin,key);
            }
        }
        return maxbin;
    }

    public int getMinBin() {
       if (maxbin == -1 && !history.isEmpty()) {
            for (int key:history.keySet()) {
                minbin = (minbin==-1)?key:Math.min(minbin, key);
            }
        }
        return minbin;
    }


    public CommunityFrame getCommunityAtBin(int i) {
        return history.get(i);
    }

    public String getName() {
        return "Community" + id;
    }

//    public double bestSimilarity(Collection<String> words) {
//        double max = 0;
//        for (CommunityFrame snapshot : history.values()) {
//            max = Math.max(max, similarity(snapshot.nodes, words));
//        }
//        return max;
//    }

//    public double similarity(Community other) {
//        return similarity(getMaximalNodeset(),other.getMaximalNodeset());
//    }

    public static double similarity(Collection<String> from, Collection<String> to) {
        double count = 0;
        for (String s : to) {
            if (from.contains(s)) {
                count++;
            }
        }

        Set<String> total = new HashSet<String>(from);
        total.addAll(to);
        return count / (double) total.size();
    }


    public String toString() {
        return getName();
    }

    void afterUnmarshal(Unmarshaller u, Object parent) {
        for (CommunityFrame f:history.values()) {
            f.setCommunity(this);
        }
    }

    public boolean equals(Community c) {
        return this.getId().equals(c.getId()) && history.values().equals(c.history.values());
    }


    public void addAssignment(ConversationChunk chunk) {
      assignments.add(chunk);
    }

//    public List<String> getMaximalRepresentation() {
//       if (representation == null) {
//           representation = Utils.getCommunityRepresentation(maximal,5);
//       }
//        return representation;
//    }


    public List<ConversationChunk> getAssignments() {
        return assignments;
    }
}


<|start_filename|>src/edu/mit/cci/text/wordij/TextToNetworkGenerator.java<|end_filename|>
package edu.mit.cci.text.wordij;

import edu.mit.cci.sna.Network;


import java.util.List;

/**
 * Generates a network based on adjacency relationships in an underlying sequential data frame.
 *
 * Consider a network constructed by placing a link between all adjacent tokens in an underlying
 * data frame. Let us define the degree of indirection between any two nodes as the unweighted
 * length of the shortest path between these nodes minus 1.
 *
 * For example, in the sequence "A B C", "A" and "B" have a degree of indirection = 0, and
 * "A" and "C" have a degree of indirection =1.
 *
 * It is assumed that implementers will add links to the network and weight them
 * according to degree of indirection as defined above.
 *
 * User: jintrone
 * Date: 5/11/11
 * Time: 5:32 PM
 */
public interface TextToNetworkGenerator {

    public Network calculateWeights(List<String> precedingTokens, List<String> targettokens);
}


<|start_filename|>src/edu/mit/cci/text/windowing/WindowingUtils.java<|end_filename|>
package edu.mit.cci.text.windowing;

import edu.mit.cci.teva.model.Conversation;
import edu.mit.cci.teva.model.Post;
import edu.mit.cci.teva.util.TevaUtils;
import edu.mit.cci.teva.util.WindowablePostAdapter;
import edu.mit.cci.text.preprocessing.Tokenizer;
import edu.mit.cci.util.U;
import org.apache.log4j.Logger;

import java.util.*;

/**
 * User: jintrone
 * Date: 9/29/12
 * Time: 10:32 AM
 */
public class WindowingUtils {


    private static Logger log = Logger.getLogger(WindowingUtils.class);

    public static <T extends Windowable> Date[][] analyzeSingleThreadBySize(List<T> current, int numTokensWidth, int numTokensDelta) {
        List<DateWidth> buffer = new ArrayList<DateWidth>();
        List<Date[]> windows = new ArrayList<Date[]>();
        int wcount = 0;
        Iterator<T> it = current.iterator();
        while (it.hasNext()) {
            T next = it.next();
            DateWidth dw = new DateWidth(next.getStart(), next.getTokens().size());
            buffer.add(dw);

            wcount += dw.width;

            if (wcount >= numTokensWidth || !it.hasNext()) {
                Date[] d = new Date[]{buffer.get(0).date, buffer.get(buffer.size() - 1).date};
                windows.add(d);
                int temp = 0;
                while (temp < numTokensDelta) {
                    int size = buffer.remove(0).width;
                    temp += size;


                }
                wcount -= temp;


            }


        }
        return windows.toArray(new Date[windows.size()][]);

    }


    /**
     * Seeks to analyze multiple threads to obtain a set of windows in which the maximum thread length is
     *
     * @param current
     * @param numTokensWidth
     * @param numTokensDelta
     * @param <T>
     * @return
     */
    public static <T extends Windowable> Date[][] analyzeMultipleThreadsBySize(Conversation c, Tokenizer<String> t, int numTokensWidth, int numTokensDelta) {

        List<Date[]> windows = new ArrayList<>();
        List<Post> posts = TevaUtils.getAllSortedPosts(c);
        int target = numTokensWidth - numTokensDelta;


        Map<String, Integer> totals = new HashMap<>();
        List<Integer> sizes = new ArrayList<>();

        int firstInWin = 0;
        windows.add(new Date[]{posts.get(0).getTime(), null});

        for (int i = 0; i < posts.size(); i++) {
            log.info("Processing post "+i);
            if (i%1000 == 0) {
                log.info("Processing post "+i);
            }
            Post p = posts.get(i);
            List<String> tok = t.tokenize(posts.get(i).getContent());
            sizes.add(tok.size());
            int val = (totals.containsKey(p.getThreadid())?totals.get(p.getThreadid()):0) + tok.size();
            totals.put(p.getThreadid(),val);

            if (val > numTokensWidth) {
                U.last(windows)[1] = p.getTime();
                if (i == posts.size() - 1) break;

                for (int delta = 0; delta < numTokensDelta; firstInWin++) {
                    int size = sizes.remove(0);

                    String tid = posts.get(firstInWin).getThreadid();
                    totals.put(tid, totals.get(tid) - size);
                    if (tid.equals(p.getThreadid())) {
                        delta+=size;
                    }

                }

                windows.add(new Date[]{posts.get(firstInWin).getTime(), null});
            }
        }
        if (U.last(windows)[1] == null) {
            U.last(windows)[1] = U.last(posts).getTime();
        }
        return windows.toArray(new Date[windows.size()][]);
    }

    static class DateWidth {
        Date date;
        int width;

        public DateWidth(Date date, int width) {
            this.date = date;
            this.width = width;
        }

    }
}


<|start_filename|>src/edu/mit/cci/teva/engine/TevaParameters.java<|end_filename|>
package edu.mit.cci.teva.engine;

import edu.mit.cci.sna.jung.JungUtils;
import edu.mit.cci.teva.serialization.ParameterAdapter;
import edu.mit.cci.teva.util.ScoringMethod;
import org.apache.log4j.Logger;

import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

/**
 * User: jintrone
 * Date: 10/2/12
 * Time: 11:57 AM
 */

@XmlJavaTypeAdapter(ParameterAdapter.class)
public class TevaParameters extends Properties {

    private static Logger logger = Logger.getLogger(TevaParameters.class);

    public static String MINIMUM_LINK_WEIGHT = "minimum_link_weight";
    public static String FIXED_CLIQUE_SIZE = "fixed_clique_size";
    public static String EXPIRE_CONSUMED_COMMUNITIES = "expire_consumed_communities";
    public static String SCORING_METHOD = "scoring_method";
    public static String CFINDER_EXECUTABLE_PATH = "cfinder_executable_path";
    public static String CFINDER_LICENSE_PATH="cfinder_license_path";
    public static String FILENAME_IDENITIFIER = "filename_identifier";
    public static final String WORD_REPLACEMENT_DICTIONARY = "word_replacement_dictionary";
    public static final String STOPWORD_LIST = "stopword_list";
    public static final String SLIDING_WINDOW_DELTA = "sliding_window_delta";
    public static final String SLIDING_WINDOW_SIZE = "sliding_window_size";
    public static final String WORDIJ_MAX_WEIGHT = "wordij_max_weight";
    public static final String WORDIJ_INDIRECTION_SIZE = "wordij_indirection_size";
    public static final String WORDIJ_TUPLE_SIZE = "wordij_tuple_size";
    public static final String WORKING_DIRECTORY = "working_directory";
    private static final String OVERWRITE_NETWORKS = "overwrite_networks";
    private static final String OVERWRITE_ANALYSES  = "overwrite_analyses";
    private static final String SKIP_NETWORK_GENERATION = "skip_network_generation";
    private static final String MEMBERSHIP_WINDOW_SIZE = "membership_window_size";
    private static final String MEMBERSHIP_WINDOW_DELTA = "membership_window_delta";
    private static final String COS_MAX_CLIQUES_EXECUTABLE = "cos_max_cliques_executable";
    private static final String COS_EXECUTABLE = "cos_executable";
    private static final String CPM_PARAMETERS = "cpm_parameters";
    private static final String PARALLEL_NETWORK_MERGE_POLICY="parallel_network_merge_policy";


    public TevaParameters(InputStream input) throws IOException {
        super();
        if (input == null) {
            logger.info("Could not load properties - falling back to default");
            this.load(getClass().getResourceAsStream("/teva.default.properties"));

        }   else {
            this.load(input);
        }

        logger.info("Loaded properties: "+this);
    }

    public TevaParameters() throws IOException {
        super();
        this.load(getClass().getResourceAsStream("/teva.default.properties"));

    }



    /** Minimum link weight to be considered when extracting communities
     *
     * @return
     */

    
    public float getMinimumLinkWeight() {
        return Float.parseFloat(getProperty(MINIMUM_LINK_WEIGHT, ".5"));
    }


    public void setMinimumLinkWeight(float f) {
        this.setProperty(MINIMUM_LINK_WEIGHT,f+"");
    }

    /**
     * Clique size for community extraction
     * @return
     */
    
    public int getFixedCliqueSize() {
        return Integer.parseInt(getProperty(FIXED_CLIQUE_SIZE,"4"));
    }

    public void setFixedCliqueSize(int i) {
        this.setProperty(FIXED_CLIQUE_SIZE,i+"");
    }


    /**
     * Whether or not communities that are "consumed" should continue to be considered progenitors for subsequent
     * communities
     * @return
     */
    
    public boolean expireConsumedCommunities() {
        return Boolean.parseBoolean(getProperty(EXPIRE_CONSUMED_COMMUNITIES,"false"));
    }

    public void setExpireConsumedCommunities(boolean b) {
        this.setProperty(EXPIRE_CONSUMED_COMMUNITIES,""+b);
    }


    /**
     * Scoring method to be used when establishing the mapping between communities in subsequent time steps
     * @return
     */
    
    public ScoringMethod getScoringMethod() {
        String method = getProperty(SCORING_METHOD,"SIMILARITY");
        return ScoringMethod.Method.valueOf(method);
    }

    public void setScoringMethod(String method) {
        setProperty(SCORING_METHOD,method);
    }

    
    public String getCFinderExecutable() {
        return getProperty(CFINDER_EXECUTABLE_PATH);
    }

    public void setCFinderExecutable(String s) {
        setProperty(CFINDER_EXECUTABLE_PATH,s);
    }

    
    public String getCFinderLicensePath() {
        return getProperty(CFINDER_LICENSE_PATH);
    }

    public void setCFinderLicensePath(String path) {
        setProperty(CFINDER_LICENSE_PATH,path);
    }

    
    public String getFilenameIdentifier() {
        return getProperty(FILENAME_IDENITIFIER,"TEvA");
    }

    public void setFilenameIdentifier(String id) {
        setProperty(FILENAME_IDENITIFIER,id);
    }


    public void setReplacementDictionary(String file) {
        setProperty(WORD_REPLACEMENT_DICTIONARY,file);
    }

    
    public String getReplacementDictionary() {
        return getProperty(WORD_REPLACEMENT_DICTIONARY,null);
    }


    public void setStopwordList(String stopwordList) {
        setProperty(STOPWORD_LIST,stopwordList);
    }

    
    public String getStopwordList() {
        return getProperty(STOPWORD_LIST,null);
    }

    public void setWindowSize(long size) {
        setProperty(SLIDING_WINDOW_SIZE,size+"");
    }

    
    public long getWindowSize() {
        return Long.parseLong(getProperty(SLIDING_WINDOW_SIZE, "-1"));
    }

    public void setWindowDelta(long delta) {
        setProperty(SLIDING_WINDOW_DELTA,delta+"");
    }

    
    public long getWindowDelta() {
        return Long.parseLong(getProperty(SLIDING_WINDOW_DELTA,"-1"));
    }

    public void setWordijIndirection(int size) {
        setProperty(WORDIJ_INDIRECTION_SIZE,size+"");
    }

    
    public int getWordijIndirection() {
        return Integer.parseInt(getProperty(WORDIJ_INDIRECTION_SIZE,3+""));
    }

    public void setWordijTupleSize(int size) {
        setProperty(WORDIJ_TUPLE_SIZE,size+"");
    }

    
    public int getWordijTupleSize() {

        return Integer.parseInt(getProperty(WORDIJ_TUPLE_SIZE,2+""));
    }

    public void setWorkingDirectory(String s) {
        setProperty(WORKING_DIRECTORY,s);
    }

    
    public String getWorkingDirectory() {
        return getProperty(WORKING_DIRECTORY,"/temp");
    }

    public void setOverwriteNetworks(boolean b) {
        setProperty(OVERWRITE_NETWORKS,b+"");
    }

    public boolean getOverwriteNetworks() {
        return Boolean.parseBoolean(getProperty(OVERWRITE_NETWORKS, false + ""));
    }

    public void setOverwriteAnalyses(boolean b) {
        setProperty(OVERWRITE_ANALYSES,b+"");
    }

    public boolean getOverwriteAnalyses() {
        return Boolean.parseBoolean(getProperty(OVERWRITE_ANALYSES, true + ""));
    }

    public void setSkipNetworkGeneration(boolean b) {
        setProperty(SKIP_NETWORK_GENERATION,b+"");
    }

    public boolean getSkipNetworkGeneration() {
        return Boolean.parseBoolean(getProperty(SKIP_NETWORK_GENERATION,false+""));
    }

    public void setMembershipWindowSize(long l) {
        setProperty(MEMBERSHIP_WINDOW_SIZE,l+"");
    }

    public long getMembershipWindowSize() {
       return Long.parseLong(getProperty(MEMBERSHIP_WINDOW_SIZE,"30000"));
    }

    public long getMembershipWindowDelta() {
        return Long.parseLong(getProperty(MEMBERSHIP_WINDOW_DELTA,"30000"));
    }

    public void setMembershipWindowDelta(long membershipWindowDelta) {
        setProperty(MEMBERSHIP_WINDOW_DELTA,""+membershipWindowDelta);
    }

    public String getCosMaxCliquesExecutable() {
        return getProperty(COS_MAX_CLIQUES_EXECUTABLE,"/usr/local/bin/maximal_cliques" );
    }

    public void setCosMaxCliquesExecutable(String s) {
        setProperty(COS_MAX_CLIQUES_EXECUTABLE,s);
    }

     public String getCosExecutable() {
        return getProperty(COS_EXECUTABLE,"/usr/local/bin/cos" );
    }

    public void setCosExecutable(String s) {
        setProperty(COS_EXECUTABLE,s);
    }

    public String getCpmParameters() {
        return getProperty(CPM_PARAMETERS,"");
    }

    public void setCpmParameters(String params) {
        setProperty(CPM_PARAMETERS,params);
    }

    public String getParallelNetworkMergePolicy() {
       return getProperty(PARALLEL_NETWORK_MERGE_POLICY, JungUtils.MergePolicy.MAX.name());
    }

    public void setWordijMaxWeight(float weight) {
        setProperty(WORDIJ_MAX_WEIGHT,weight+"");
    }

    public float getWordijMaxWeight() {
        return Math.max(.1f,Math.min(Float.parseFloat(getProperty(WORDIJ_MAX_WEIGHT, 1.0f + "")),1.0f));
    }
}