title: java.lang.ClassNotFoundException: text.DefaultSource
body: <p>I have a scala maven application with spark. I use Intellij Idea. I did an executable-jar from it but when I try to launch it via windows console there is an error about lack of some class. I can't figure out there is a problem because I've added it in  block in my .pom file. Also when I look inside .jar I see that library with that class:</p>
  
  <p><em>needed library in .jar:</em>
  <img src="https://i.stack.imgur.com/lxVuR.png" alt="needed library in .jar"></p>
  
  <p>I tried to use both plugins: maven-shade-plugin and maven-assembly-plugin and result is the same. And I tried to set this library explicitly in classpath through project structure->libraries in Intellij:</p>
  
  <p><em>classpath in IDEA:</em>
  <img src="https://i.stack.imgur.com/Xjx0A.png" alt="classpath in idea">.</p>
  
  <p>Any help would be greatly appreciated!
  Here is my code: </p>
  
  <pre><code>import org.apache.spark.broadcast.Broadcast
  import org.apache.spark.ml.recommendation.{ALS, ALSModel}
  import org.apache.spark.sql.functions._
  import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
  import scala.collection.{Map, Set}
  import scala.collection.mutable.ArrayBuffer
  import scala.util.Random
  object RunRecommender {
  
    def main(args: Array[String]): Unit = {
      val spark: SparkSession =SparkSession.builder()
        .master("local")
        .appName("Recommender Engines with Audioscrobbler data")
        .config("spark.sql.warehouse.dir", "spark-warehouse")
        .getOrCreate()
  
      val rawUserArtistData: Dataset[String] = spark.read.textFile("user_artist_data.txt")
      val rawArtistData: Dataset[String] = spark.read.textFile("artist_data.txt")
      val rawArtistAlias: Dataset[String] = spark.read.textFile("artist_alias.txt")
  
      val runRecommender: RunRecommender = new RunRecommender(spark)
      runRecommender.preparation(rawUserArtistData, rawArtistData, rawArtistAlias)
      runRecommender.model(rawUserArtistData, rawArtistData, rawArtistAlias)
      runRecommender.evaluate(rawUserArtistData, rawArtistAlias)
      runRecommender.recommend(rawUserArtistData, rawArtistData, rawArtistAlias)
    }
  
  }
  </code></pre>
  
  <p>and here is my .pom file:</p>
  
  <pre><code>&lt;?xml version="1.0" encoding="UTF-8"?&gt;
  &lt;project xmlns="http://maven.apache.org/POM/4.0.0"
           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
           xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"&gt;
      &lt;modelVersion&gt;4.0.0&lt;/modelVersion&gt;
  
      &lt;groupId&gt;recommends&lt;/groupId&gt;
      &lt;artifactId&gt;recommends&lt;/artifactId&gt;
      &lt;packaging&gt;jar&lt;/packaging&gt;
      &lt;name&gt;Recommender Engine with Audioscrobbler data&lt;/name&gt;
      &lt;version&gt;1.0-SNAPSHOT&lt;/version&gt;
  
      &lt;repositories&gt;
      &lt;repository&gt;
          &lt;id&gt;mavencentral&lt;/id&gt;
          &lt;name&gt;Maven Central&lt;/name&gt;
          &lt;url&gt;https://repo1.maven.org/maven2/&lt;/url&gt;
          &lt;layout&gt;default&lt;/layout&gt;
      &lt;/repository&gt;
      &lt;/repositories&gt;
  
      &lt;build&gt;
          &lt;pluginManagement&gt;
              &lt;plugins&gt;
                  &lt;plugin&gt;
                      &lt;groupId&gt;net.alchim31.maven&lt;/groupId&gt;
                      &lt;artifactId&gt;scala-maven-plugin&lt;/artifactId&gt;
                      &lt;version&gt;3.2.1&lt;/version&gt;
                  &lt;/plugin&gt;
                  &lt;plugin&gt;
                      &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
                      &lt;artifactId&gt;maven-compiler-plugin&lt;/artifactId&gt;
                      &lt;version&gt;2.0.2&lt;/version&gt;
                  &lt;/plugin&gt;
              &lt;/plugins&gt;
          &lt;/pluginManagement&gt;
  
          &lt;plugins&gt;
              &lt;plugin&gt;
                  &lt;groupId&gt;net.alchim31.maven&lt;/groupId&gt;
                  &lt;artifactId&gt;scala-maven-plugin&lt;/artifactId&gt;
                  &lt;version&gt;3.2.1&lt;/version&gt;
                  &lt;configuration&gt;
                      &lt;archive&gt;
                          &lt;manifest&gt;
                              &lt;mainClass&gt;RunRecommender&lt;/mainClass&gt;
                          &lt;/manifest&gt;
                      &lt;/archive&gt;
                  &lt;/configuration&gt;
                      &lt;executions&gt;
                          &lt;execution&gt;
                              &lt;id&gt;scala-compile-first&lt;/id&gt;
                              &lt;phase&gt;process-resources&lt;/phase&gt;
                              &lt;goals&gt;
                                  &lt;goal&gt;add-source&lt;/goal&gt;
                                  &lt;goal&gt;compile&lt;/goal&gt;
                              &lt;/goals&gt;
                          &lt;/execution&gt;
                          &lt;execution&gt;
                              &lt;id&gt;scala-test-compile&lt;/id&gt;
                              &lt;phase&gt;process-test-resources&lt;/phase&gt;
                              &lt;goals&gt;
                                  &lt;goal&gt;testCompile&lt;/goal&gt;
                              &lt;/goals&gt;
                          &lt;/execution&gt;
                      &lt;/executions&gt;
              &lt;/plugin&gt;
              &lt;plugin&gt;
                  &lt;artifactId&gt;maven-shade-plugin&lt;/artifactId&gt;
                  &lt;executions&gt;
                      &lt;execution&gt;
                          &lt;phase&gt;package&lt;/phase&gt;
                          &lt;goals&gt;
                              &lt;goal&gt;shade&lt;/goal&gt;
                          &lt;/goals&gt;
                          &lt;configuration&gt;
                              &lt;createDependencyReducedPom&gt;false&lt;/createDependencyReducedPom&gt;
                              &lt;filters&gt;
                                  &lt;filter&gt;
                                      &lt;artifact&gt;*:*&lt;/artifact&gt;
                                      &lt;excludes&gt;
                                          &lt;exclude&gt;META-INF/*.SF&lt;/exclude&gt;
                                          &lt;exclude&gt;META-INF/*.DSA&lt;/exclude&gt;
                                          &lt;exclude&gt;META-INF/*.RSA&lt;/exclude&gt;
                                      &lt;/excludes&gt;
                                  &lt;/filter&gt;
                              &lt;/filters&gt;
                              &lt;transformers&gt;
                                  &lt;transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"&gt;
                                      &lt;mainClass&gt;RunRecommender&lt;/mainClass&gt;
                                  &lt;/transformer&gt;
                              &lt;/transformers&gt;
                          &lt;/configuration&gt;
                      &lt;/execution&gt;
                  &lt;/executions&gt;
              &lt;/plugin&gt;
          &lt;/plugins&gt;
      &lt;/build&gt;
  
      &lt;dependencies&gt;
          &lt;dependency&gt;
              &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
              &lt;artifactId&gt;spark-core_2.11&lt;/artifactId&gt;
              &lt;version&gt;2.0.0&lt;/version&gt;
          &lt;/dependency&gt;
          &lt;dependency&gt;
              &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
              &lt;artifactId&gt;spark-mllib_2.11&lt;/artifactId&gt;
              &lt;version&gt;2.0.0&lt;/version&gt;
          &lt;/dependency&gt;
          &lt;dependency&gt;
              &lt;groupId&gt;org.apache.spark&lt;/groupId&gt;
              &lt;artifactId&gt;spark-sql_2.10&lt;/artifactId&gt;
              &lt;version&gt;2.0.0&lt;/version&gt;
          &lt;/dependency&gt;
      &lt;/dependencies&gt;
  
  &lt;/project&gt;
  </code></pre>
  
  <p>and here is stack-trace when I try to run jar:</p>
  
  <pre><code>16/11/06 11:56:40 WARN SparkContext: Use an existing SparkContext, some configuration may not take effect.
  16/11/06 11:56:40 INFO SharedState: Warehouse path is 'spark-warehouse'.
  Exception in thread "main" java.lang.ClassNotFoundException: Failed to find data source: text. Please find packages at http://spark-packages.org
          at org.apache.spark.sql.execution.datasources.DataSource.lookupDataSource(DataSource.scala:145)
          at org.apache.spark.sql.execution.datasources.DataSource.providingClass$lzycompute(DataSource.scala:78)
          at org.apache.spark.sql.execution.datasources.DataSource.providingClass(DataSource.scala:78)
          at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:310)
          at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
          at org.apache.spark.sql.DataFrameReader.text(DataFrameReader.scala:492)
          at org.apache.spark.sql.DataFrameReader.textFile(DataFrameReader.scala:528)
          at org.apache.spark.sql.DataFrameReader.textFile(DataFrameReader.scala:501)
          at RunRecommender$.main(RunRecommender.scala:20)
          at RunRecommender.main(RunRecommender.scala)
  Caused by: java.lang.ClassNotFoundException: text.DefaultSource
          at java.net.URLClassLoader.findClass(Unknown Source)
          at java.lang.ClassLoader.loadClass(Unknown Source)
          at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
          at java.lang.ClassLoader.loadClass(Unknown Source)
          at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$5$$anonfun$apply$1.apply(DataSource.scala:130)
          at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$5$$anonfun$apply$1.apply(DataSource.scala:130)
          at scala.util.Try$.apply(Try.scala:192)
          at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$5.apply(DataSource.scala:130)
          at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$5.apply(DataSource.scala:130)
          at scala.util.Try.orElse(Try.scala:84)
          at org.apache.spark.sql.execution.datasources.DataSource.lookupDataSource(DataSource.scala:130)
          ... 9 more
  16/11/06 11:56:40 INFO SparkContext: Invoking stop() from shutdown hook
  1
  </code></pre>