gt:             return getResults(passages, neStrs, docNos, false);
context: package info.ephyra.search.searchers;
  
  import info.ephyra.io.MsgPrinter;
  import info.ephyra.search.Result;
  
  import org.w3c.dom.Document;
  import org.w3c.dom.Node;
  import org.w3c.dom.NodeList;
  import org.xml.sax.InputSource;
  
  import java.io.StringReader;
  import java.util.ArrayList;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import javax.xml.parsers.DocumentBuilder;
  import javax.xml.parsers.DocumentBuilderFactory;
  
  import lemurproject.indri.ParsedDocument;
  import lemurproject.indri.QueryEnvironment;
  import lemurproject.indri.ScoredExtentResult;
  
  /**
   * <p>A <code>KnowledgeMiner</code> that deploys the Indri IR system to
   * search a local text corpus. The search results are paragraphs.</p>
   *
   * <p>It runs as a separate thread, so several queries can be performed in
   * parallel.</p>
   *
   * <p>This class extends the class <code>KnowledgeMiner</code>.</p>
   *
   * @author Nico Schlaefer
   * @version 2007-07-26
   */
  public class IndriKM extends KnowledgeMiner {
      /**
       * Maximum total number of search results.
       */
      private static final int MAX_RESULTS_TOTAL = 20;
  //	private static final int MAX_RESULTS_TOTAL = 0;	
      /**
       * Maximum number of search results per query.
       */
      private static final int MAX_RESULTS_PERQUERY = 20;
  //	private static final int MAX_RESULTS_PERQUERY = 50;	
      /**
       * Maximum number of documents fetched at a time.
       */
      private static final int MAX_DOCS = 20;
  //	private static final int MAX_DOCS = 20;	
      /**
       * <p>Regular expression that matches characters that cause problems in
       * Indri queries and thus should be removed from query strings.</p>
       *
       * <p>Indri allows the following characters:
       * <ul>
       * <li>'\u0080'..'\u00ff'</li>
       * <li>'a'..'z'</li>
       * <li>'A'..'Z'</li>
       * <li>'0'..'9'</li>
       * <li>'_'</li>
       * <li>'-'</li>
       * <li>'.' (only allowed if in between digits)</li>
       * <li>whitespaces</li>
       * <li>'"'</li>
       * </ul>
       * However, for some of the special characters Indri fails to retrieve
       * results and therefore they are excluded.
       * </p>
       */
      private static final String FORBIDDEN_CHAR = "[^\\w\\.\\s\"]";
  
      /**
       * Directories of Indri indices.
       */
      private String[] indriDirs;
      /**
       * URLs of Indri servers.
       */
      private String[] indriUrls;
  
      /**
       * Creates a new Indri knowledge miner and sets the directories of indices
       * or the URLs of servers.
       *
       * @param locations directories of indices or URLs of servers
       * @param isServers <code>true</code> iff the first parameter provides URLs of servers
       */
      public IndriKM(String[] locations, boolean isServers) {
          if (isServers) indriUrls = locations;
          else indriDirs = locations;
      }
  
      /**
       * Gets a list of all Indri index directories that have been specified with
       * environment variables 'WIKI_INDEX', 'WIKI_INDEX2', 'WIKI_INDEX3' etc.
       * One environment variable can specify multiple indices which are queried
       * with the same knowledge miner.
       *
       * @return Indri index directories grouped by knowledge miners
       */
      public static String[][] getIndriIndices() {
          ArrayList<String[]> indices = new ArrayList<String[]>();
  
          String index = System.getenv("WIKI_INDEX");
          //   System.out.println("Index: " + index);
          if (index != null && index.length() > 0)
              indices.add(index.split(";"));
          for (int i = 2; ; i++) {
              index = System.getenv("WIKI_INDEX" + i);
              if (index != null && index.length() > 0)
                  indices.add(index.split(";"));
              else break;
          }
  
          return indices.toArray(new String[indices.size()][]);
      }
  
      /**
       * Gets a list of all Indri server URLs that have been specified with
       * environment variables 'INDRI_SERVER', 'INDRI_SERVER2', 'INDRI_SERVER3'
       * etc. One environment variable can specify multiple servers which are
       * queried with the same knowledge miner.
       *
       * @return Indri server URLs grouped by knowledge miners
       */
      public static String[][] getIndriServers() {
          ArrayList<String[]> servers = new ArrayList<String[]>();
  
          String server = System.getenv("INDRI_SERVER");
          if (server != null && server.length() > 0)
              servers.add(server.split(";"));
          for (int i = 2; ; i++) {
              server = System.getenv("INDRI_SERVER" + i);
              if (server != null && server.length() > 0)
                  servers.add(server.split(";"));
              else break;
          }
  
          return servers.toArray(new String[servers.size()][]);
      }
  
      /**
       * Returns a representation of the query string that is suitable for Indri.
       *
       * @param qs query string
       * @return query string for Indri
       */
      public static String transformQueryString(String qs) {
          // drop characters that are not properly supported by Indri
          // ('.' is only allowed in between digits)
          qs = qs.replaceAll("&\\w++;", " ");
          qs = qs.replaceAll(FORBIDDEN_CHAR, " ");
          String dotsRemoved = "";
          for (int i = 0; i < qs.length(); i++)
              if (qs.charAt(i) != '.' ||
                      (i > 0 && i < qs.length() - 1 &&
                              Character.isDigit(qs.charAt(i - 1)) &&
                              Character.isDigit(qs.charAt(i + 1))))
                  dotsRemoved += qs.charAt(i);
          qs = dotsRemoved;
  
          // replace ... OR ... by #or(... ...)
          Matcher m = Pattern.compile(
                  "((\\([^\\(\\)]*+\\)|\\\"[^\\\"]*+\\\"|[^\\s\\(\\)]++) OR )++" +
                          "(\\([^\\(\\)]*+\\)|\\\"[^\\\"]*+\\\"|[^\\s\\(\\)]++)").matcher(qs);
          while (m.find())
              qs = qs.replace(m.group(0), "#or(" + m.group(0) + ")");
          qs = qs.replace(" OR", "");
  
          // replace ... AND ... by #combine(... ...)
          m = Pattern.compile(
                  "((\\([^\\(\\)]*+\\)|\\\"[^\\\"]*+\\\"|[^\\s\\(\\)]++) AND )++" +
                          "(\\([^\\(\\)]*+\\)|\\\"[^\\\"]*+\\\"|[^\\s\\(\\)]++)").matcher(qs);
          while (m.find())
              qs = qs.replace(m.group(0), "#combine(" + m.group(0) + ")");
          qs = qs.replace(" AND", "");
  
          // replace "..." by #1(...)
          m = Pattern.compile("\"([^\"]*+)\"").matcher(qs);
          while (m.find())
              qs = qs.replace(m.group(0), "#1(" + m.group(1) + ")");
  
          // form passage query
  //		qs = "#combine[p](" + qs + ")";
          qs = "#combine(" + qs + ")";
  
          return qs;
      }
  
      /**
       * Returns the maximum total number of search results.
       *
       * @return maximum total number of search results
       */
      protected int getMaxResultsTotal() {
          return MAX_RESULTS_TOTAL;
      }
  
      /**
       * Returns the maximum number of search results per query.
       *
       * @return maximum total number of search results
       */
      protected int getMaxResultsPerQuery() {
          return MAX_RESULTS_PERQUERY;
      }
  
      /**
       * Queries the Indri indices or servers and returns an array containing up
       * to <code>MAX_RESULTS_PERQUERY</code> search results.
       *
       * @return Indri search results
       */
      protected Result[] doSearch() {
          try {
              // create query environment
              QueryEnvironment env = new QueryEnvironment();
  
              // add Indri indices or servers
              if (indriDirs != null && indriDirs.length > 0) {
                  for (String indriDir : indriDirs) env.addIndex(indriDir);
              } else if (indriUrls != null && indriUrls.length > 0) {
                  for (String indriUrl : indriUrls) env.addServer(indriUrl);
              } else {
                  MsgPrinter.printErrorMsg("Directories of Indri indices or " +
                          "URLs of Indri servers required.");
                  System.exit(1);
              }
  
              // run an Indri query, returning up to MAX_RESULTS_PERQUERY results
              MsgPrinter.printStatusMsg("3.2.1 Searching...query string: " + query.getQueryString());
              MsgPrinter.printStatusMsg("3.2.1 Searching...transformed query string: " + transformQueryString(query.getQueryString()));
  
              ScoredExtentResult[] results =
                      env.runQuery(transformQueryString(query.getQueryString()),
                              MAX_RESULTS_PERQUERY);
              MsgPrinter.printStatusMsg("3.2.1 Getting answers....searching...result len: " + results.length);
              // get passages and document numbers
              String[] passages = new String[results.length];
              String[] neStrs = new String[results.length];
              for (int i = 0; i < results.length; i += MAX_DOCS) {
                  // fetch MAX_DOCS documents at a time (for memory efficiency)
                  ScoredExtentResult[] partResults =
                          new ScoredExtentResult[Math.min(MAX_DOCS, results.length - i)];
                  for (int j = i; j < i + partResults.length; j++)
                      partResults[j - i] = results[j];
  
                  ParsedDocument[] documents = env.documents(partResults);
  
                  for (int j = 0; j < partResults.length; j++) {
                      int passageBegin = partResults[j].begin;
                      int passageEnd = partResults[j].end;
                      int byteBegin = documents[j].positions[passageBegin].begin;
                      int byteEnd = documents[j].positions[passageEnd - 1].end;
  
                      byte[] doc = documents[j].text.getBytes("UTF-8");
                      passages[j + i] = new String(doc, byteBegin, byteEnd - byteBegin);
  
                      // parse ne field
                      String content = documents[j].content;
                      DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
                      DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
                      String xmlFileString = "<ROOT>" + content + "</ROOT>";
  //                    System.out.println("xml String: \n "+xmlFileString);
                      Document document = docBuilder.parse(
                              new InputSource(new StringReader(xmlFileString)));
                      NodeList docs = document.getElementsByTagName("NE");
                      if (docs.getLength() > 0) {
                          Node docNode = docs.item(0);
  //                        System.out.println("neNODE: \n"+docNode);
  //                        String neStr = docNode.getTextContent(); //not work due to old dependency
  //                        System.out.println("neStr: \n"+neStr);
  
                          // see http://stackoverflow.com/a/14016460/4105935
                          neStrs [j+i] = docNode.getFirstChild().getNodeValue();
                      }else{
                          MsgPrinter.printErrorMsgTimestamp("ne field not found");
                      }
  //                    System.out.println(neStr);
                  }
              }
              String[] docNos = env.documentMetadata(results, "docno");
  //            MsgPrinter.printStatusMsg("searching results docNos=========>>>>>:" + Arrays.toString(docNos));
  //            System.out.println("passages:=========>>>>>:"+ Arrays.toString(passages));
  
              // close query environment
              env.close();
  
              // return results
  