1

I seen all the treads related to this issue and they are all very clear that the posters were cross compiling with two versions of Scala. In my case I make sure I only have one version 2.11 but I still get the same error. Any help is appreciated, thanks. My Spark Env:

   /___/ .__/\_,_/_/ /_/\_\   version 2.0.0.2.5.3.0-37
  /_/   
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_67)

My pom.xml:

    <properties>
    <spark.version>2.2.1</spark.version>
    <scala.version>2.11.8</scala.version>
    <scala.library.version>2.11.8</scala.library.version>
    <scala.binary.version>2.11</scala.binary.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.source.version>1.7</java.source.version>
    <java.compile.version>1.7</java.compile.version>
    <kafka.version>0-10</kafka.version>
</properties>

<dependencies>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>com.typesafe.scala-logging</groupId>
        <artifactId>scala-logging-slf4j_${scala.binary.version}</artifactId>
        <version>2.1.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming-kafka-${kafka.version}_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-library</artifactId>
        <version>${scala.library.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>0.11.0.2</version>
    </dependency>
</dependencies>

This is the exception:

    at org.apache.spark.streaming.kafka010.DirectKafkaInputDStream$$anonfun$start$1.apply(DirectKafkaInputDStream.scala:246)
at org.apache.spark.streaming.kafka010.DirectKafkaInputDStream$$anonfun$start$1.apply(DirectKafkaInputDStream.scala:245)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.mutable.AbstractSet.scala$collection$SetLike$$super$map(Set.scala:45)
at scala.collection.SetLike$class.map(SetLike.scala:93)
at scala.collection.mutable.AbstractSet.map(Set.scala:45)
at org.apache.spark.streaming.kafka010.DirectKafkaInputDStream.start(DirectKafkaInputDStream.scala:245)
at org.apache.spark.streaming.DStreamGraph$$anonfun$start$5.apply(DStreamGraph.scala:47)
at org.apache.spark.streaming.DStreamGraph$$anonfun$start$5.apply(DStreamGraph.scala:47)
at scala.collection.parallel.mutable.ParArray$ParArrayIterator.foreach_quick(ParArray.scala:145)
at scala.collection.parallel.mutable.ParArray$ParArrayIterator.foreach(ParArray.scala:138)
at scala.collection.parallel.ParIterableLike$Foreach.leaf(ParIterableLike.scala:975)
at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply$mcV$sp(Tasks.scala:54)
at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53)
at scala.collection.parallel.Task$$anonfun$tryLeaf$1.apply(Tasks.scala:53)
at scala.collection.parallel.Task$class.tryLeaf(Tasks.scala:56)
at scala.collection.parallel.ParIterableLike$Foreach.tryLeaf(ParIterableLike.scala:972)
at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask$class.compute(Tasks.scala:165)
at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:514)
at scala.concurrent.forkjoin.RecursiveAction.exec(RecursiveAction.java:160)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

When I grep "_2.1" in the output of command: mvn dependency:tree -Dverbose I don't see any references to 2.10.

    [INFO] +- org.apache.spark:spark-core_2.11:jar:2.2.1:compile
[INFO] |  +- com.twitter:chill_2.11:jar:0.8.0:compile
[INFO] |  +- org.apache.spark:spark-launcher_2.11:jar:2.2.1:compile
[INFO] |  |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- org.apache.spark:spark-network-common_2.11:jar:2.2.1:compile
[INFO] |  +- org.apache.spark:spark-network-shuffle_2.11:jar:2.2.1:compile
[INFO] |  |  +- (org.apache.spark:spark-network-common_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- org.apache.spark:spark-unsafe_2.11:jar:2.2.1:compile
[INFO] |  |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  |  +- (com.twitter:chill_2.11:jar:0.8.0:compile - omitted for duplicate)
[INFO] |  +- org.json4s:json4s-jackson_2.11:jar:3.2.11:compile
[INFO] |  |  +- org.json4s:json4s-core_2.11:jar:3.2.11:compile
[INFO] |  |  |  +- org.json4s:json4s-ast_2.11:jar:3.2.11:compile
[INFO] |  |  |        +- org.scala-lang.modules:scala-xml_2.11:jar:1.0.1:compile
[INFO] |  |  |        \- (org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.1:compile - omitted for conflict with 1.0.4)
[INFO] |  +- com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.6.5:compile
[INFO] |  +- org.apache.spark:spark-tags_2.11:jar:2.2.1:compile
[INFO] +- org.apache.spark:spark-sql_2.11:jar:2.2.1:compile
[INFO] |  +- org.apache.spark:spark-sketch_2.11:jar:2.2.1:compile
[INFO] |  |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- (org.apache.spark:spark-core_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- org.apache.spark:spark-catalyst_2.11:jar:2.2.1:compile
[INFO] |  |  +- (org.apache.spark:spark-core_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  |  +- (org.apache.spark:spark-unsafe_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  |  +- (org.apache.spark:spark-sketch_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] +- org.apache.spark:spark-hive_2.11:jar:2.2.1:compile
[INFO] |  +- (org.apache.spark:spark-core_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- (org.apache.spark:spark-sql_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] +- com.typesafe.scala-logging:scala-logging-slf4j_2.11:jar:2.1.2:compile
[INFO] |  +- com.typesafe.scala-logging:scala-logging-api_2.11:jar:2.1.2:compile
[INFO] +- org.apache.spark:spark-streaming-kafka-0-10_2.11:jar:2.2.1:compile
[INFO] |  +- org.apache.kafka:kafka_2.11:jar:0.10.0.1:compile
[INFO] |  |  +- org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4:compile
[INFO] |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] +- org.apache.spark:spark-streaming_2.11:jar:2.2.1:compile
[INFO] |  +- (org.apache.spark:spark-core_2.11:jar:2.2.1:compile - omitted for duplicate)
[INFO] |  +- (org.apache.spark:spark-tags_2.11:jar:2.2.1:compile - omitted for duplicate)

Also I should state that I am using a Uber jar to run in the Spark server using spark-submit. The Uber is including the jars below. I included the scala jars as a last resource to solve the problem but it does not matter if do or don't.

            <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.1.0</version>
            <configuration>
                <shadedArtifactAttached>false</shadedArtifactAttached>
                <keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
                <artifactSet>
                    <includes>
                        <include>org.apache.kafka:spark*</include>
                        <include>org.apache.spark:spark-streaming-kafka-${kafka.version}_${scala.binary.version}
                        </include>
                        <include>org.apache.kafka:kafka_${scala.binary.version}</include>
                        <include>org.apache.kafka:kafka-clients</include>
                        <include>org.apache.spark:*</include>
                        <include>org.scala-lang:scala-library</include>
                    </includes>
                    <excludes>
                        <exclude>org.apache.hadoop:*</exclude>
                        <exclude>com.fasterxml:*</exclude>
                    </excludes>
                </artifactSet>
                <transformers>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                        <resource>META-INF/services/javax.ws.rs.ext.Providers</resource>
                    </transformer>
                </transformers>
            </configuration>
            <executions>
                <execution>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
Improve this question
2
  • 1
    Why compile with 2.2.1 if you start stating you use 2.0.0? Commented Jan 31, 2018 at 19:14
  • That is a good question, as a next step I was going to downgrade but I don't see why that would be. I also edited my question because the problem only happen when I deploy the uber jar and I run with spar-submit. Of course in IntelliJ it runs fine. Commented Jan 31, 2018 at 19:26

2 Answers 2

Reset to default
2

Scala 2.11 does not work with Java 7: https://scala-lang.org/download/2.11.8.html. Please update Java to 8

Improve this answer
Sign up to request clarification or add additional context in comments.

5 Comments

I switched to 1.7 because I was getting this error: Exception in thread "main" java.lang.UnsupportedClassVersionError: com/syf/dl/kafka/demo/SparkStreamToHdfs : Unsupported major.minor version 52.0
In the server i get: bash-4.1$ java -version java version "1.7.0_141" OpenJDK Runtime Environment (rhel-2.6.10.1.el6_9-x86_64 u141-b02) OpenJDK 64-Bit Server VM (build 24.141-b02, mixed mode)
@Fabio You must compile with Java 8 :) Then you will not have this error :)
@Gaweda there is no JDK1.8 on the server. I don't think that I can compile with Java 1.8 and Scala 2.11.8 and deploy on a server that has JDK 1.7 and apparently Scala 2.11.8. I am not sure how that would work. I just did another test where I change the POM to use JDK 1.7 and scala 2.10.6 and Spark 2.0.0 . And I still get Unsupported major.minor version 52.0
@Fabio Both compile and runtime version must be 8
0

Finally I got it to work for my given environment. Changes I did was Scala 2.10.6 Java 1.7 Spark 2.0.0.

For completeness here is my pom.xml:

   <properties>
    <spark.version>2.0.0</spark.version>
    <scala.version>2.10.6</scala.version>
    <scala.library.version>2.10.6</scala.library.version>
    <scala.binary.version>2.10</scala.binary.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
    <java.source.version>1.7</java.source.version>
    <java.compile.version>1.7</java.compile.version>
    <kafka.version>0-10</kafka.version>
</properties>

<dependencies>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-core_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-hive_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>com.typesafe.scala-logging</groupId>
        <artifactId>scala-logging-slf4j_${scala.binary.version}</artifactId>
        <version>2.1.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming-kafka-${kafka.version}_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.scala-lang</groupId>
        <artifactId>scala-library</artifactId>
        <version>${scala.library.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming_${scala.binary.version}</artifactId>
        <version>${spark.version}</version>
    </dependency>
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>0.11.0.2</version>
    </dependency>
</dependencies>

<build>
    <sourceDirectory>src/main/java</sourceDirectory>
    <testSourceDirectory>src/test/java</testSourceDirectory>
    <resources>
        <resource>
            <directory>src/main/resources</directory>
        </resource>
    </resources>

    <plugins>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>3.1.0</version>
            <configuration>
                <shadedArtifactAttached>false</shadedArtifactAttached>
                <keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
                <artifactSet>
                    <includes>
                        <include>org.apache.kafka:spark*</include>
                        <include>org.apache.spark:spark-streaming-kafka-${kafka.version}_${scala.binary.version}
                        </include>
                        <include>org.apache.kafka:kafka_${scala.binary.version}</include>
                        <include>org.apache.kafka:kafka-clients</include>
                        <include>org.apache.spark:*</include>
                        <include>org.scala-lang:scala-library</include>
                    </includes>
                    <excludes>
                        <exclude>org.apache.hadoop:*</exclude>
                        <exclude>com.fasterxml:*</exclude>
                    </excludes>
                </artifactSet>
                <transformers>
                    <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                        <resource>META-INF/services/javax.ws.rs.ext.Providers</resource>
                    </transformer>
                </transformers>
            </configuration>
            <executions>
                <execution>
                    <goals>
                        <goal>shade</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>

        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
                <source>${java.source.version}</source>
                <target>${java.compile.version}</target>
            </configuration>
        </plugin>

        <plugin>
            <groupId>org.scala-tools</groupId>
            <artifactId>maven-scala-plugin</artifactId>
            <version>2.15.2</version>
            <executions>
                <execution>
                    <id>compile</id>
                    <goals>
                        <goal>compile</goal>
                    </goals>
                    <phase>compile</phase>
                </execution>
                <execution>
                    <id>test-compile</id>
                    <goals>
                        <goal>testCompile</goal>
                    </goals>
                    <phase>test-compile</phase>
                </execution>
                <execution>
                    <phase>process-resources</phase>
                    <goals>
                        <goal>compile</goal>
                    </goals>
                </execution>
            </executions>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-eclipse-plugin</artifactId>
            <version>2.9</version>
            <configuration>
                <sourceIncludes>
                    <sourceInclude>**/*.scala</sourceInclude>
                </sourceIncludes>
                <projectNameTemplate>[artifactId]</projectNameTemplate>
                <projectnatures>
                    <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
                    <projectnature>org.eclipse.m2e.core.maven2Nature</projectnature>
                    <projectnature>org.eclipse.jdt.core.javanature</projectnature>
                </projectnatures>
                <buildcommands>
                    <buildcommand>org.eclipse.m2e.core.maven2Builder</buildcommand>
                    <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
                </buildcommands>
                <classpathContainers>
                    <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER"</classpathContainer>
                </classpathContainers>
                <excludes>
                    <exclude>org.scala-lang:scala-library</exclude>
                    <exclude>org.scala-lang:scala-compiler</exclude>
                </excludes>
            </configuration>
        </plugin>
        <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-jar-plugin</artifactId>
            <version>2.6</version>
            <configuration>
                <archive>
                    <manifestEntries>
                        <Implementation-Version>${project.version}</Implementation-Version>
                        <SCM-Revision>1.0</SCM-Revision>
                    </manifestEntries>
                </archive>
            </configuration>
        </plugin>
    </plugins>
</build>
Improve this answer

2 Comments

So my answer was correct, 2.11 with Java 8, update of Java is required and preferred, in other downgrade of Scala will work
Yes you point me to the fact that 2.11 and Java 7 does not work. However the correct answer was to downgrade to 2.10 since 1.8 was not available but I shall mark your answer as correct.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.