How to marshal array to binary and unmarshal binary to array in golang?

Question

I want to use gob to encode and decode object, I do it like this:

type transProp struct {
    a []int
    b []float64
}

func (p transProp) MarshalBinary() ([]byte, error) {
    // A simple encoding: plain text.
    var b bytes.Buffer
    fmt.Fprintln(&b, p.a, p.b)
    return b.Bytes(), nil
}

// UnmarshalBinary modifies the receiver so it must take a pointer receiver.
func (p *transProp) UnmarshalBinary(data []byte) error {
    // A simple encoding: plain text.
    b := bytes.NewBuffer(data)
    _, err := fmt.Fscanln(b, &p.a, &p.b)
    return err
}

func TestGobEncode(t *testing.T) {
    p := transProp{
        a: []int{3, 4, 5},
        b: []float64{1.0, 2.0},
    }

    var network bytes.Buffer
    enc := gob.NewEncoder(&network)
    err := enc.Encode(p)
    if err != nil {
        log.Fatal("encode:", err)
    }
    dec := gob.NewDecoder(&network)
    var p1 transProp
    err = dec.Decode(&p1)
    if err != nil {
        log.Fatal("decode:", err)
    }
    fmt.Println(p1)
}

But, this report error like this:

decode:can't scan type: *[]int

Println/Scanln aren't meant to be used for serialization. You could probably implement fmt.Scanner, but you might as well do it directly in the Marshal methods you have. — Mr_Pink
– Mr_Pink, Commented May 31, 2017 at 17:38

putu · Accepted Answer · 2017-06-01 14:58:54Z

If you can change the visibility of transProp fields to public, e.g.

type transProp struct {
    A []int
    B []float64
}

then you don't need to implement custom binary marshaller/unmarshaller. You can use default gob decoder/encoder.

However, if you can't, there are many options.

The simplest one, define another wrapper struct that export related fields, wrap transProp, then use default gob encoder/decoder, e.g.

type wrapTransProp struct {
    A []int
    B []float64
}

func (p transProp) MarshalBinary() ([]byte, error) {
    //Wrap struct
    w := wrapTransProp{p.a, p.b}

    //use default gob encoder
    var buf bytes.Buffer
    enc := gob.NewEncoder(&buf)
    if err := enc.Encode(w); err != nil {
        return nil, err
    }
    return buf.Bytes(), nil
}
func (p *transProp) UnmarshalBinary(data []byte) error {
    w := wrapTransProp{}

    //Use default gob decoder
    reader := bytes.NewReader(data)
    dec := gob.NewDecoder(reader)
    if err := dec.Decode(&w); err != nil {
        return err
    }

    p.a = w.A
    p.b = w.B
    return nil
}

Custom marshaller/unmarshaller with custom data layout. There are many implementation possibilities. Several considerations:

Byte order, little/big endian?
Packet/stream layout?

An example implementation of big-endian with stream format:

// Big-Endian
// Size  : 4,         4,      1,        n,     4,       n
// Types : uint32,    uint32, uint8,    []int, uint32,  []float64
// Data  : #numbytes, #nInt,  #intSize, p.a,   #nFloat, p.b

The challenge is in how to represent int since it's architecture dependent. Sample implementation will be:

func (p transProp) MarshalBinary() ([]byte, error) {
    //Get sizeof int (in bits) from strconv package
    szInt := strconv.IntSize / 8
    nInt := len(p.a)
    nFloat := len(p.b)

    nStream := 4 + 4 + 1 + nInt*szInt + 4 + nFloat*8
    stream := make([]byte, nStream)
    pos := 0

    //total number of bytes
    binary.BigEndian.PutUint32(stream, uint32(nStream))
    pos += 4

    //num of items in p.a
    binary.BigEndian.PutUint32(stream[pos:], uint32(nInt))
    pos += 4

    //int size
    stream[pos] = uint8(szInt)
    pos++

    //items in a
    switch szInt {
    case 1:
        for _, v := range p.a {
            stream[pos] = uint8(v)
            pos++
        }
    case 2: //16-bit
        for _, v := range p.a {
            binary.BigEndian.PutUint16(stream[pos:], uint16(v))
            pos += 2
        }
    case 4: //32-bit
        for _, v := range p.a {
            binary.BigEndian.PutUint32(stream[pos:], uint32(v))
            pos += 4
        }
    case 8: //64-bit
        for _, v := range p.a {
            binary.BigEndian.PutUint64(stream[pos:], uint64(v))
            pos += 8
        }
    }

    //number of items in p.b
    binary.BigEndian.PutUint32(stream[pos:], uint32(nFloat))
    pos += 4

    //items in b
    s := stream[pos:pos] //slice len=0, capacity=nFloat
    buf := bytes.NewBuffer(s)
    if err := binary.Write(buf, binary.BigEndian, p.b); err != nil {
        return nil, err
    }

    return stream, nil
}

func (p *transProp) UnmarshalBinary(data []byte) error {
    buf := bytes.NewBuffer(data)

    var intSize uint8
    var k, nBytes, nInt, nFloat uint32

    //num bytes
    if err := binary.Read(buf, binary.BigEndian, &nBytes); err != nil {
        return err
    }
    if len(data) < int(nBytes) {
        return errors.New("len(data) < #Bytes")
    }

    //num of int items
    if err := binary.Read(buf, binary.BigEndian, &nInt); err != nil {
        return err
    }

    //int size
    if err := binary.Read(buf, binary.BigEndian, &intSize); err != nil {
        return err
    }

    //read int into p.a
    pos := 0
    stream := buf.Bytes()
    p.a = make([]int, nInt)
    switch intSize {
    case 1:
        for pos = 0; pos < int(nInt); pos++ {
            p.a[pos] = int(stream[pos])
        }
    case 2:
        for k = 0; k < nInt; k++ {
            p.a[k] = int(binary.BigEndian.Uint16(stream[pos:]))
            pos += 2
        }
    case 4:
        for k = 0; k < nInt; k++ {
            p.a[k] = int(binary.BigEndian.Uint32(stream[pos:]))
            pos += 4
        }
    case 8:
        for k = 0; k < nInt; k++ {
            p.a[k] = int(binary.BigEndian.Uint64(stream[pos:]))
            pos += 8
        }
    }

    //advance buffer
    buf.Next(pos)

    //num of float64 items
    if err := binary.Read(buf, binary.BigEndian, &nFloat); err != nil {
        return err
    }

    //items in b
    p.b = make([]float64, nFloat)
    if err := binary.Read(buf, binary.BigEndian, p.b); err != nil {
        return err
    }

    return nil
}

It is very nice to have a working [Un]MarshalBinary code like that. I was afraid of doing nonsense with such an approach, but your safeguards make it look robust enough. Thanks!

Collectives™ on Stack Overflow

How to marshal array to binary and unmarshal binary to array in golang?

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related