1

I have such test class:

package com.my.spark.test

import org.apache.spark.SparkContext
import org.scalatest.{ConfigMap, BeforeAndAfterAll, FunSuite}

class MyTrainingSuiteIT extends FunSuite with BeforeAndAfterAll {

  private[this] var _sc: SparkContext = null
  private[this] val defaultCoresNumber = 1
  private[this] val defaultMaster = s"local[$defaultCoresNumber]"
  private[this] val defaultName = "some-spark-integration-test"

  override def beforeAll(configMap: ConfigMap): Unit = {
    super.beforeAll()
    val mode = configMap.get("mode").get
    mode match {
      case "local" =>
        val coresNumber = configMap.get("cores").get
        _sc = new SparkContext(s"local[$coresNumber]", defaultName)
      case "docker" =>
        println("Docker was chosen")
        _sc = new SparkContext(defaultMaster, defaultName)
      case "cluster" =>
        val clusterType = configMap.get("clusterType").get
        println(s"Cluster of type [$clusterType] was chosen.")
        _sc = new SparkContext(defaultMaster, defaultName)
      case _ =>
        println("Unknow mode was chosen")
        _sc = new SparkContext(defaultMaster, defaultName)
    }
  }

  override def afterAll(): Unit = {
    _sc.stop()
    _sc = null
    super.afterAll()
  }

  test("Context testing") {
    assert(defaultMaster == s"local[$defaultCoresNumber]")
  }
}

And I try to perform test with commandline and scalatest(I need to do it without sbt or maven):

scala -classpath /home/huge/list/of/jars.jar org.scalatest.tools.Runner -R /home/hspark/my_spark_project/target/project-tests.jar -s com.my.spark.test.MyTrainingSuiteIT -Dmode=local -Dcores=2

After that ScalaTest window is opened and I see error:

Event: Run Aborted

Message: An exception or error caused a run to abort: akka.remote.RemoteActorRefProvider. (java.lang.String, akka.actor.ActorSystem$Settings, akka.event.EventStream, akka.actor.Scheduler, akka.actor.DynamicAccess) 
Summary: Total number of tests run: 0

Suites: completed 0, aborted 0
Tests: succeeded 0, failed 0, canceled 0, ignored 0, pending 0 
Exception:
java.lang.NoSuchMethodException
java.lang.Class.getConstructor0(Class.java:2902) 
java.lang.Class.getDeclaredConstructor(Class.java:2066)  
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$2.apply(DynamicAccess.scala:77) 
scala.util.Try$.apply(Try.scala:161) 
akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:74) 
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:85) 
akka.actor.ReflectiveDynamicAccess$$anonfun$createInstanceFor$3.apply(DynamicAccess.scala:85) scala.util.Success.flatMap(Try.scala:200) 
akka.actor.ReflectiveDynamicAccess.createInstanceFor(DynamicAccess.scala:85)
akka.actor.ActorSystemImpl.<init>(ActorSystem.scala:546) 
akka.actor.ActorSystem$.apply(ActorSystem.scala:111) 
akka.actor.ActorSystem$.apply(ActorSystem.scala:104) 
org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) 
org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) 
org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:52) 
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1920) 
scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) 
org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1911) 
org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:55) 
org.apache.spark.rpc.akka.AkkaRpcEnvFactory.create(AkkaRpcEnv.scala:253) 
org.apache.spark.rpc.RpcEnv$.create(RpcEnv.scala:53) 
org.apache.spark.SparkEnv$.create(SparkEnv.scala:254) 
org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194) 
org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277) 
org.apache.spark.SparkContext.<init>(SparkContext.scala:450) 
org.apache.spark.SparkContext.<init>(SparkContext.scala:162) 
com.my.spark.test.MyTrainingSuiteIT.beforeAll(MyTrainingSuiteIT.scala:19) 
org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) 
com.my.spark.test.MyTrainingSuiteIT.run(MyTrainingSuiteIT.scala:6) 
org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) 
org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) 
org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557)
scala.collection.immutable.List.foreach(List.scala:318) 
org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) 
org.scalatest.tools.RunnerJFrame$RunnerThread$$anonfun$run$1.apply(RunnerJFrame.scala:1565) 
org.scalatest.tools.RunnerJFrame$RunnerThread$$anonfun$run$1.apply(RunnerJFrame.scala:1563) 
org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) 
org.scalatest.tools.RunnerJFrame$RunnerThread.run(RunnerJFrame.scala:1563)

And the question is, what should I do to perform tests correctly? Why doesn't it work?

1 Answer 1

1

The reason is that there were several akka implementation in classpath. More details here https://stackoverflow.com/a/34685021/4054598

Sign up to request clarification or add additional context in comments.

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.