title: Unable to run Java program that imports org.apache.lucene.analysis.Analyzer
body: <p>I'm unable to run a java program that imports org.apache.lucene.analysis.Analyzer;</p>
  
  <pre><code>import java.io.BufferedReader;
  import java.io.BufferedWriter;
  import java.io.FileReader;
  import java.io.FileWriter;
  import java.io.StringReader;
  import java.util.HashMap;
  import java.util.Map;
  
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.IntWritable;
  import org.apache.hadoop.io.LongWritable;
  import org.apache.hadoop.io.Text;
  import org.apache.lucene.analysis.Analyzer;
  import org.apache.lucene.analysis.TokenStream;
  import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
  import org.apache.mahout.classifier.naivebayes.BayesUtils;
  import org.apache.mahout.classifier.naivebayes.NaiveBayesModel;
  import org.apache.mahout.classifier.naivebayes.StandardNaiveBayesClassifier;
  import org.apache.mahout.common.Pair;
  import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
  import org.apache.mahout.math.RandomAccessSparseVector;
  import org.apache.mahout.math.Vector;
  import org.apache.mahout.math.Vector.Element;
  import org.apache.mahout.vectorizer.DefaultAnalyzer;
  import org.apache.mahout.vectorizer.TFIDF;
  
  import com.google.common.collect.ConcurrentHashMultiset;
  import com.google.common.collect.Multiset;
  
  
  public class Classifier {
  
   public static Map&lt;String, Integer&gt; readDictionnary(Configuration conf, Path dictionnaryPath) {
    Map&lt;String, Integer&gt; dictionnary = new HashMap&lt;String, Integer&gt;();
    for (Pair&lt;Text, IntWritable&gt; pair : new SequenceFileIterable&lt;Text, IntWritable&gt;(dictionnaryPath, true, conf)) {
     dictionnary.put(pair.getFirst().toString(), pair.getSecond().get());
    }
    return dictionnary;
   }
  
   public static Map&lt;Integer, Long&gt; readDocumentFrequency(Configuration conf, Path documentFrequencyPath) {
    Map&lt;Integer, Long&gt; documentFrequency = new HashMap&lt;Integer, Long&gt;();
    for (Pair&lt;IntWritable, LongWritable&gt; pair : new SequenceFileIterable&lt;IntWritable, LongWritable&gt;(documentFrequencyPath, true, conf)) {
     documentFrequency.put(pair.getFirst().get(), pair.getSecond().get());
    }
    return documentFrequency;
   }
  
   public static void main(String[] args) throws Exception {
  
    System.out.println("Start time :" + System.currentTimeMillis());
    if (args.length &lt; 5) {
     System.out.println("Arguments: [model] [label index] [dictionnary] [document frequency] [tweet file]");
     return;
    }
    String modelPath = args[0];
    String labelIndexPath = args[1];
    String dictionaryPath = args[2];
    String documentFrequencyPath = args[3];
    String testFilePath = args[4];
  
    Configuration configuration = new Configuration();
  
    // model is a matrix (wordId, labelId) =&gt; probability score
    NaiveBayesModel model = NaiveBayesModel.materialize(new Path(modelPath), configuration);
  
    StandardNaiveBayesClassifier classifier = new StandardNaiveBayesClassifier(model);
  
    // labels is a map label =&gt; classId
    Map&lt;Integer, String&gt; labels = BayesUtils.readLabelIndex(configuration, new Path(labelIndexPath));
    Map&lt;String, Integer&gt; dictionary = readDictionnary(configuration, new Path(dictionaryPath));
    Map&lt;Integer, Long&gt; documentFrequency = readDocumentFrequency(configuration, new Path(documentFrequencyPath));
  
  
    // analyzer used to extract word from tweet
    Analyzer analyzer = new DefaultAnalyzer();
  
    int labelCount = labels.size();
    int documentCount = documentFrequency.get(-1).intValue();
  
    System.out.println("Number of labels: " + labelCount);
    System.out.println("Number of documents in training set: " + documentCount);
    BufferedReader reader = new BufferedReader(new FileReader(testFilePath));
  
    String outputFile = "/home/hduser/result.txt";
    FileWriter f1 = new FileWriter(outputFile,true); 
    BufferedWriter out = new BufferedWriter(f1);
  
  
  
    int correctCounter=0;
    int totalCounter=0;
    while(true)
    {
     String line = reader.readLine();
     if (line == null) {
      break;
     }
  
     String[] arr = line.split(" ");
     String catId = arr[0];
     String label = arr[1];
  
     String msg = line.substring(arr[0].length() + arr[1].length() + 2);
  
  
     Multiset&lt;String&gt; words = ConcurrentHashMultiset.create();
  
     // extract words from Msg
     TokenStream ts = analyzer.reusableTokenStream("text", new StringReader(msg));
     CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     ts.reset();
     int wordCount = 0;
     while (ts.incrementToken()) {
      if (termAtt.length() &gt; 0) {
       String word = ts.getAttribute(CharTermAttribute.class).toString();
       Integer wordId = dictionary.get(word);
       // if the word is not in the dictionary, skip it
       if (wordId != null) {
        words.add(word);
        wordCount++;
       }
      }
     }
  
     // create vector wordId =&gt; weight using tfidf
     Vector vector = new RandomAccessSparseVector(10000);
     TFIDF tfidf = new TFIDF();
     for (Multiset.Entry&lt;String&gt; entry:words.entrySet()) {
      String word = entry.getElement();
      int count = entry.getCount();
      Integer wordId = dictionary.get(word);
      Long freq = documentFrequency.get(wordId);
      double tfIdfValue = tfidf.calculate(count, freq.intValue(), wordCount, documentCount);
      vector.setQuick(wordId, tfIdfValue);
     }
     // With the classifier, we get one score for each label 
     // The label with the highest score is the one the tweet is more likely to
     // be associated to
     Vector resultVector = classifier.classifyFull(vector);
     //double bestScore = -Double.MAX_VALUE;
     double bestScore =Double.MAX_VALUE;
     int bestCategoryId = -1;
     String resultStr=catId+" ";
     for(Element element: resultVector) 
     {
      int categoryId = element.index();
      double score = -1 * element.get();
      if (score &lt; bestScore) {
       bestScore = score;
       bestCategoryId = categoryId;
      }
      //System.out.print("  " + labels.get(categoryId) + ": " + score);
      if(resultStr.equalsIgnoreCase(catId + " "))
      {
       resultStr=resultStr + labels.get(categoryId) + " " + score;
      }
      else
      {
       resultStr=resultStr + "   " + labels.get(categoryId) + " " + score;
      }
     }
     try
     {
  
       out.write(resultStr);
       out.write("\n");
  
     }
     catch(Exception e)
     {
  
     }
  
     //System.out.println(label + " =&gt; " + labels.get(bestCategoryId));
     out1.write(label + " =&gt; " + labels.get(bestCategoryId));
      out1.write("\n");
     totalCounter++;
      if(label.equalsIgnoreCase(labels.get(bestCategoryId)))
      {
  
       correctCounter++;
       System.out.println("correctCounter : " + correctCounter);
      }
    };
     //Close the output stream
    System.out.println("correctCounter : " + correctCounter + " TotalCounter :" + totalCounter);
    System.out.println("End time :" + System.currentTimeMillis());
    System.out.println("Accuracy : " +  (double)correctCounter/totalCounter);
     out.close();
   }
  }
  </code></pre>
  
  <p>Configurations on my ubuntu system: Hadoop 1.2.0, Mahout 0.7, Lucene 1.4.1, Java 1.6</p>
  
  <p>I compiled the java program without any errors and generated a jar file:</p>
  
  <pre><code>java c -classpath /usr/local/hadoop/*:/usr/local/mahout/*:/usr/local/lucene/* -d Classifier_Class/ Classifier.java
  
  jar -cvf Classify.jar -C Classifier_CLass/ .    
  </code></pre>
  
  <p>When I tried executing this jar using hadoop I'm facing the following error:</p>
  
  <pre><code>hadoop jar Classify.jar {input arguments}
  Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/lucene/analysis/Analyzer
      at java.lang.Class.forName0(Native Method)
      at java.lang.Class.forName(Class.java:270)
      at org.apache.hadoop.util.RunJar.main(RunJar.java:153)
  Caused by: java.lang.ClassNotFoundException: org.apache.lucene.analysis.Analyzer
      at java.net.URLClassLoader$1.run(URLClassLoader.java:217)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:205)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:323)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:268)
      ... 3 more
  </code></pre>