Hi I am learning Scala Spark, and trying to build a simple code. But getting an error as shown below. I have imported all the dependencies in pom.xml file form maven repo also. but still it is giving me class not found exception. Please help. Below in the screenshot of the error, code and pom.xml file.
Exception in thread "main" java.lang.NoClassDefFoundError:org/apache/spark/sql/SparkSession$
at com.first.spark.Demo$.main(Demo.scala:13)
at com.first.spark.Demo.main(Demo.scala)
Caused by: java.lang.ClassNotFoundException: org.apache.spark.sql.SparkSession$
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 2 more
pom.xml file
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.example</groupId>
<artifactId>Scala_2</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies> <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>2.4.5</version>
</dependency>
<dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
<version>2.4.8</version>
<scope>provided</scope>
</dependency>
</dependencies>
Spak Sql Code
package com.first.spark
import org.apache.avro.LogicalTypes.Date
import org.apache.spark.sql.catalyst.expressions.Month
import org.apache.spark.sql.{SparkSession, functions}
import org.apache.spark.sql.functions.{avg, col, month, round, year}
import org.apache.spark.{SparkConf, SparkContext}
object Demo {
def main(args: Array[String]):Unit = {
val spark = SparkSession.builder
.appName(name = "ReadMultilintJson")
.config("spark.master", "local")
.getOrCreate()
val aaon = spark.read.option("header", true).option("inferSchema", true)
.csv("/Users/p0p029i/Downloads/Spark_Sql Datasets/614_m6_StockMarket_DataSets/AAON.csv")
val aaondf = aaon.withColumn("Year", year(col("Date")))
.withColumn(colName = "Month", month(col("Date")))
val aaongroup = aaondf.groupBy("Year", "Month")
.agg(round(avg(col("Adj Close")),2)).as("AvgClsPrice")
val aaonsorted = aaongroup.orderBy(col("year").desc,col("Month").desc)
aaonsorted.show()
}
}