Convert RDD of Array[Array[String]] to DataFrame

You can do something like,

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Row, SparkSession}
import java.util.UUID
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.StructType

    val l1: Array[Array[String]] = Array(
      Array[String]("K1", "V1"),
      Array[String]("K2", "V2"),
      Array[String]("K3", "V3"),
      Array[String]("K5", "V5"),
      Array[String]("K7", "V7"))

    val l2: Array[Array[String]] = Array(
      Array[String]("K1", "V1"),
      Array[String]("K3", "V3"),
      Array[String]("K21", "V21"))

    val spark = SparkSession.builder().master("local").getOrCreate()
    val sc = spark.sparkContext

    val rdd = sc.parallelize(Array(l1, l2)).flatMap(x => {
      val id = UUID.randomUUID().toString
      x.map(y => Row(id, y(0), y(1)))
    })

    val schema = new StructType()
      .add("id", "String")
      .add("key", "String")
      .add("value", "String")

    val df = spark
      .createDataFrame(rdd, schema)
      .groupBy("id")
      .pivot("key").agg(last("value"))
      .drop("id")

    df.printSchema()
    df.show(false)

The schema and output looks something like,

root
 |-- K1: string (nullable = true)
 |-- K2: string (nullable = true)
 |-- K21: string (nullable = true)
 |-- K3: string (nullable = true)
 |-- K5: string (nullable = true)
 |-- K7: string (nullable = true)

+---+----+----+---+----+----+
|K1 |K2  |K21 |K3 |K5  |K7  |
+---+----+----+---+----+----+
|V1 |null|V21 |V3 |null|null|
|V1 |V2  |null|V3 |V5  |V7  |
+---+----+----+---+----+----+

Note: this will produce null in missing places as shown in outputs. pivot basically transposes the data set based on some column Hope this answers your question!