How to map json object in to a java object in apache spark

Question

I have following json object and I want to map this json into java object please help me.

{
  "address": {
    "building": "1007",
    "coord": [
      -73.856077,
      40.848447
    ],
    "street": "Morris Park Ave",
    "zipcode": "10462"
  },
  "borough": "Bronx",
  "cuisine": "Bakery",
  "grades": [
    {
      "date": {
        "$date": 1393804800000
      },
      "grade": "A",
      "score": 2
    },
    {
      "date": {
        "$date": 1378857600000
      },
      "grade": "A",
      "score": 6
    },
    {
      "date": {
        "$date": 1358985600000
      },
      "grade": "A",
      "score": 10
    },
    {
      "date": {
        "$date": 1322006400000
      },
      "grade": "A",
      "score": 9
    },
    {
      "date": {
        "$date": 1299715200000
      },
      "grade": "B",
      "score": 14
    }
  ],
  "name": "Morris Park Bake Shop",
  "restaurant_id": "30075445"
}

When I use scala printschema methot it shows like below, Please help me to map this schema with java object.

root
 |-- address: struct (nullable = true)
 |    |-- building: string (nullable = true)
 |    |-- coord: array (nullable = true)
 |    |    |-- element: double (containsNull = true)
 |    |-- street: string (nullable = true)
 |    |-- zipcode: string (nullable = true)
 |-- borough: string (nullable = true)
 |-- cuisine: string (nullable = true)
 |-- grades: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- date: struct (nullable = true)
 |    |    |    |-- $date: long (nullable = true)
 |    |    |-- grade: string (nullable = true)
 |    |    |-- score: long (nullable = true)
 |-- name: string (nullable = true)
 |-- restaurant_id: string (nullable = true)

dont post images in the question you can copy paste the schema like above. — Ashwini
– Ashwini, Commented May 2, 2020 at 5:21

Ram Ghadiyaram · Accepted Answer · 2020-05-02 05:19:36Z

If you wan to convert json string to java you can use Gson..

Gson g = new Gson(); yourvenillabean p = g.fromJson(jsonString, yourvanillabean.class)

see complete example here

Read more here

since you added spark tag If I am right the below is spark way of handling json if you wan to create a dataframe...

you can convert this json in to DataFrame like below

package examples;


import org.apache.spark.SparkConf;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.List;

public class JsonDF {

    SparkSession sparkSession;

    public static void main(String[] args) {


 SparkConf conf = new SparkConf();
        conf.setMaster("local[2]");


        SparkSession spark = SparkSession
                .builder()
                .appName("Dataset-json")
                .master("local[4]")
                .getOrCreate();
        System.out.println("Session created");
        String str = "{\n" +
                "  \"address\": {\n" +
                "    \"building\": \"1007\",\n" +
                "    \"coord\": [\n" +
                "      -73.856077,\n" +
                "      40.848447\n" +
                "    ],\n" +
                "    \"street\": \"Morris Park Ave\",\n" +
                "    \"zipcode\": \"10462\"\n" +
                "  },\n" +
                "  \"borough\": \"Bronx\",\n" +
                "  \"cuisine\": \"Bakery\",\n" +
                "  \"grades\": [\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1393804800000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 2\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1378857600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 6\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1358985600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 10\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1322006400000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 9\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1299715200000\n" +
                "      },\n" +
                "      \"grade\": \"B\",\n" +
                "      \"score\": 14\n" +
                "    }\n" +
                "  ],\n" +
                "  \"name\": \"Morris Park Bake Shop\",\n" +
                "  \"restaurant_id\": \"30075445\"\n" +
                "}";
        List<String> data = Arrays.asList(str);

        Dataset<String> ds = spark.createDataset(data, Encoders.STRING());
        spark.read().json(ds).show(false);


    }
}

Result :

+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+
|address                                                |borough|cuisine|grades                                                                                                                         |name                 |restaurant_id|
+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+
|[1007, [-73.856077, 40.848447], Morris Park Ave, 10462]|Bronx  |Bakery |[[[1393804800000], A, 2], [[1378857600000], A, 6], [[1358985600000], A, 10], [[1322006400000], A, 9], [[1299715200000], B, 14]]|Morris Park Bake Shop|30075445     |
+-------------------------------------------------------+-------+-------+-------------------------------------------------------------------------------------------------------------------------------+---------------------+-------------+

UPDATE :

Since you want to convert json to java class you can use this to do that with your json string...

After convertion you can get a class like this:

public class Application {
  Address AddressObject;
  private String borough;
  private String cuisine;
  ArrayList<Object> grades = new ArrayList<Object>();
  private String name;
  private String restaurant_id;


 // Getter Methods 

  public Address getAddress() {
    return AddressObject;
  }

  public String getBorough() {
    return borough;
  }

  public String getCuisine() {
    return cuisine;
  }

  public String getName() {
    return name;
  }

  public String getRestaurant_id() {
    return restaurant_id;
  }

 // Setter Methods 

  public void setAddress( Address addressObject ) {
    this.AddressObject = addressObject;
  }

  public void setBorough( String borough ) {
    this.borough = borough;
  }

  public void setCuisine( String cuisine ) {
    this.cuisine = cuisine;
  }

  public void setName( String name ) {
    this.name = name;
  }

  public void setRestaurant_id( String restaurant_id ) {
    this.restaurant_id = restaurant_id;
  }
}
public class Address {
  private String building;
  ArrayList<Object> coord = new ArrayList<Object>();
  private String street;
  private String zipcode;


 // Getter Methods 

  public String getBuilding() {
    return building;
  }

  public String getStreet() {
    return street;
  }

  public String getZipcode() {
    return zipcode;
  }

 // Setter Methods 

  public void setBuilding( String building ) {
    this.building = building;
  }

  public void setStreet( String street ) {
    this.street = street;
  }

  public void setZipcode( String zipcode ) {
    this.zipcode = zipcode;
  }
}

which is compatible to the spark schema you got..

root
 |-- address: struct (nullable = true)
 |    |-- building: string (nullable = true)
 |    |-- coord: array (nullable = true)
 |    |    |-- element: double (containsNull = true)
 |    |-- street: string (nullable = true)
 |    |-- zipcode: string (nullable = true)
 |-- borough: string (nullable = true)
 |-- cuisine: string (nullable = true)
 |-- grades: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- date: struct (nullable = true)
 |    |    |    |-- $date: long (nullable = true)
 |    |    |-- grade: string (nullable = true)
 |    |    |-- score: long (nullable = true)
 |-- name: string (nullable = true)
 |-- restaurant_id: string (nullable = true)

Here is an example of How to create a Dataframe in spark that contains complex Java Beans

Thank you for your reply :) . I need to create a class called Restaurant and load records as instances of that class. How to create this resturent class. Please Help
you can use site24x7.com/tools/json-to-java.html copy json string and convert in to java object online
Thank you for your valuable time, I Still having the problem in this mapping part I created the class called Resturent after converting my json. In spark side where to put my schema to map with json..? Please help me
Could you look at the solution below using Jackson API and comment please.
@L.S Couple of questions : Why you want to only adapt converting mapping etc... whcih are traditional, why cant you use direct api with straitght forward techniques are available ? I mean why Bean ... mapping... encoding .... only why not direct way like for ex: .read.json("examples/src/main/resources/people.json");

VN'sCorner · Accepted Answer · 2020-05-02 13:35:04Z

I guess the intention is to create Java Object from json String.Its going to be a long one but pretty easy to implement. This solution is based on Jackson API.ObjectMapper is the main api used for data-binding.

Step 1 : Include jackson in maven dependency if you are using pom.xml otherwise download respective jar files from maven repository https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core

<dependencies>
    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>2.10.1</version>
    </dependency>
  </dependencies>

Step 2: Create POJO Classes.These object will be created from JSON.

We need POJO's for below and will take the structure below,

         AddressDetails  
           Address
           Grades  
             Grade   
               Date

I will add the code for POJO's bottom up.

Date :

package org.personal.TestProject.jsontoobject;

public class Date {
    private float $date;

    // Getter Methods
    public float get$date() {
        return $date;
    }

    // Setter Methods
    public void set$date(float $date) {
        this.$date = $date;
    }
}

Grade :

package org.personal.TestProject.jsontoobject;

    public class Grade {

        Date date = new Date();
        private String grade;
        private float score;

        // Getter Methods
        public float getScore() {
            return score;
        }
        public String getGrade() {
            return grade;
        }
        public Date getDate() {
            return date;
        }

        // Setter Methods
        public void setScore(float score) {
            this.score = score;
        }
        public void setGrade(String grade) {
            this.grade = grade;
        }

        public void setDate(Date date) {
            this.date = date;
        }
    }

Grades:

package org.personal.TestProject.jsontoobject;

import java.util.ArrayList;
import java.util.List;
import org.personal.TestProject.jsontoobject.Grade;

public class Grades {

    private List<Grade> grades = new ArrayList<Grade>();

    public List<Grade> getGrades() {
        return grades;
    }

    public void setGrades(List<Grade> grades) {
        this.grades = grades;
    }
}

Address:

package org.personal.TestProject.jsontoobject;
import java.util.ArrayList;
import java.util.List;

public class Address {
    private String building;
    List<Float> coord = new ArrayList<Float>();
    private String street;
    private String zipcode;


    // Getter Methods

    public String getBuilding() {
        return building;
    }
    public List<Float> getCoords() {
        return coord;
    }

    public String getStreet() {
        return street;
    }
    public String getZipcode() {
        return zipcode;
    }

    // Setter Methods
    public void setBuilding(String building) {
        this.building = building;
    }
    public void setCoord(List<Float> coord) {
        this.coord = coord;
    }
    public void setStreet(String street) {
        this.street = street;
    }
    public void setZipcode(String zipcode) {
        this.zipcode = zipcode;
    }
}

AddressDetails :

package org.personal.TestProject.jsontoobject;

import java.util.ArrayList;
import java.util.List;
import org.personal.TestProject.jsontoobject.Grade;
import org.personal.TestProject.jsontoobject.Address;

public class AddressDetails {

    Address address = new Address();
    private String borough;
    private String cuisine;
    List<Grade> grades = new ArrayList<Grade>();
    private String name;
    private String restaurant_id;

    //Setters
    public List<Grade> getGrades() {
        return grades;
    }
    public Address getAddress() {
        return address;
    }
    public String getBorough() {
        return borough;
    }
    public String getCuisine() {
        return cuisine;
    }
    public String getName() {
        return name;
    }
    public String getRestaurant_id() {
        return restaurant_id;
    }

    //Getters
    public void setGrades(List<Grade> grades) {
        this.grades = grades;
    }
    public void setAddress(Address address) {
        this.address = address;
    }
    public void setBorough(String borough) {
        this.borough = borough;
    }  
    public void setCuisine(String cuisine) {
        this.cuisine = cuisine;
    }    
    public void setName(String name) {
        this.name = name;
    }   
    public void setRestaurant_id(String restaurant_id) {
        this.restaurant_id = restaurant_id;
    }
}

Step 3: We use ObjectMapper to convert json(file or String) into an Object.Now you may use getters to access fields from the object as desired.

package org.personal.TestProject.jsontoobject;

        import java.io.File;
        import java.util.ArrayList;
        import java.util.List;

        import com.fasterxml.jackson.databind.ObjectMapper;
        import org.personal.TestProject.jsontoobject.AddressDetails;

public class ReadJsonFile {

    public static void main(String args[]){

        ObjectMapper mapper = new ObjectMapper();

        String json = "{\n" +
                "  \"address\": {\n" +
                "    \"building\": \"1007\",\n" +
                "    \"coord\": [\n" +
                "      -73.856077,\n" +
                "      40.848447\n" +
                "    ],\n" +
                "    \"street\": \"Morris Park Ave\",\n" +
                "    \"zipcode\": \"10462\"\n" +
                "  },\n" +
                "  \"borough\": \"Bronx\",\n" +
                "  \"cuisine\": \"Bakery\",\n" +
                "  \"grades\": [\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1393804800000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 2\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1378857600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 6\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1358985600000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 10\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1322006400000\n" +
                "      },\n" +
                "      \"grade\": \"A\",\n" +
                "      \"score\": 9\n" +
                "    },\n" +
                "    {\n" +
                "      \"date\": {\n" +
                "        \"$date\": 1299715200000\n" +
                "      },\n" +
                "      \"grade\": \"B\",\n" +
                "      \"score\": 14\n" +
                "    }\n" +
                "  ],\n" +
                "  \"name\": \"Morris Park Bake Shop\",\n" +
                "  \"restaurant_id\": \"30075445\"\n" +
                "}";

        AddressDetails value = null;
        try {
            value = mapper.readValue(json, AddressDetails.class);
             // For converting a json file to object use below code
            //value = mapper.readValue(new File("result.json"), AddressDetails.class);
            // Use getter methods to Access any fields on the object
            System.out.println("Restaurant ID " +value.getRestaurant_id());
        } catch (Exception e) {
            e.printStackTrace();
        }     
    }

}

Please mark/vote the answer. That will help developers in need.

Collectives™ on Stack Overflow

How to map json object in to a java object in apache spark

2 Answers 2

UPDATE :

6 Comments

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

UPDATE :

6 Comments

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related