Parse TCP JSON Stream and emit each object via Socket.io

Question

I am working with a data feed that sends a JSON stream over a TCP socket and I'm using node.js/socket.io to emit the stream to a browser client.

Everything is working except I need each JSON object to emitted as a separate message from the socket.io server. In the client the messages are received like this:

//Message 1:
{"type":"TYPE_1","odds":[{"eventId":"foo","odds":[{"oddId":foo,"oddType":"LIVE","source":"foo"}]}]}
//Message 2:
{"type":"TYPE_2","odds":[{"eventId":"foo","odds":[{"oddId":foo,"oddType":"LIVE","source":"foo"}]}]}
{"type":"TYPE_3","odds":[{"eventId":"foo","odds":[{"oddId":foo,"oddType":"LIVE","source":"foo"}]}]}
//Message 3:
{"type":"TYPE_4","odds":[{"eventId":"foo","od
//Message 4:
ds":[{"oddId":foo,"oddType":"LIVE","source":"foo"}]}]}

The data feed docs state: "All messages sending to your application will form a JSON stream (no delimiter between messages), so you may need a decoder that support JSON stream."

So the stream is strictly correct but I need each object as separate message.

I have looked at https://www.npmjs.com/package/JSONStream and others but am very new to nodejs and socketio and am not sure how to implement them in to the server.

Have also read How can I parse the first JSON object on a stream in JS, nodejs JSON.parse(data_from_TCP_socket), http://www.sebastianseilund.com/json-socket-sending-json-over-tcp-in-node.js-using-sockets.

I think it's something to do with buffer chunk lengths and them being too big so the messages get split but that could be wrong! I'm guessing I need a delimiter check that balances brackets but not sure how to go about it or if the right approach.

My Server Script:

var app = require('express')();
var http = require('http').Server(app);
var io = require('socket.io')(http);
var net = require('net');
var port = 8992;              // Datafeed port
var host = '127.0.0.1';      // Datafeed IP address

//Whenever someone connects this gets executed
io.on('connection', function(socket){
  console.log('A user connected to me the server');

  //Whenever someone disconnects this piece of code executed
  socket.on('disconnect', function () {
    console.log('A user disconnected');
  });
});

//Create a TCP socket to read data from datafeed
var socket = net.createConnection(port, host);

socket.on('error', function(error) {
  console.log("Error Connecting");
});

socket.on('connect', function(connect) {
  console.log('connection established');
  socket.write('{"type":"SUBSCRIBE"}');
});

socket.on('data', function(data) {
  //console.log('DATA ' + socket.remoteAddress + ': ' + data);
  var data = data.toString();
  io.sockets.emit('event', JSON.stringify(data));
});

http.listen(3000, function(){
  console.log('listening on *:3000');
});

My Client:

<!DOCTYPE html>
<html>
  <head><title>Hello world</title></head>
  <script src="https://cdn.socket.io/socket.io-1.4.5.js"></script>
  <script src="http://code.jquery.com/jquery-1.11.1.js"></script>
  <script>
    var socket = io.connect('http://localhost:3000');
  </script>
  <body>
    <form action="">
      <input id="m" autocomplete="off" /><button>Send</button>
    </form>
    <ul id="messages"></ul>
    <script>
      socket.on('event', function(data){
        var t = JSON.parse(data.toString('utf8'));
        $('#messages').prepend($('<li>').text(t));
        console.log('Got event from Server:', t);
      });
    </script>
  </body>
</html>

Any help or guidance would be amazing as really struggling with this.

mscdex · Accepted Answer · 2016-08-19 16:53:20Z

4

A common delimiter to use is a newline character (\n). If you have that appended to your JSON messages it will be very easy to parse the messages. For example:

var sockBuf = '';
socket.setEncoding('utf8');
socket.on('data', function(data) {
  sockBuf += data;
  var i;
  var l = 0;
  while ((i = sockBuf.indexOf('\n', l)) !== -1) {
    io.sockets.emit('event', sockBuf.slice(l, i));
    l = i + 1;
  }
  if (l)
    sockBuf = sockBuf.slice(l);
});

or a more efficient, but slightly less simple solution:

var sockBuf = '';
socket.setEncoding('utf8');
socket.on('data', function(data) {
  var i = data.indexOf('\n');
  if (i === -1) {
    sockBuf += data;
    return;
  }
  io.sockets.emit('event', sockBuf + data.slice(0, i));
  var l = i + 1;
  while ((i = data.indexOf('\n', l)) !== -1) {
    io.sockets.emit('event', data.slice(l, i));
    l = i + 1;
  }
  sockBuf = data.slice(l);
});

edited Aug 19, 2016 at 16:53

answered Aug 19, 2016 at 16:46

mscdex

107k15 gold badges201 silver badges159 bronze badges

Sign up to request clarification or add additional context in comments.

15 Comments

David Pede Over a year ago

Thanks for this but have just tested both versions and nothing is emitted from the server now. Any idea what the issue might be?

mscdex Over a year ago

Your TCP server needs to be writing the JSON messages with newlines after them, are you sure it's now doing that?

David Pede Over a year ago

Yes have just output the raw data from the data feed to console and it shows each json node starts on newline

David Pede Over a year ago

I changed the delimeter in your code examples to '}{' just to check and the server emits but with the same problem as my question

mscdex Over a year ago

I don't know then, the code I suggested works for me locally with a simple TCP server that writes newline-delimited JSON strings to connected sockets.

|

Collectives™ on Stack Overflow

Parse TCP JSON Stream and emit each object via Socket.io

1 Answer 1

15 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

15 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related