Merge pull request #103 from thanhpk/reduce-memory-usage-while-serial…

…izing

use bufio.Writer instead of binary.Write to reduce memory footprint while serializing

bitset.go

@@ -897,7 +897,18 @@ func (b *BitSet) WriteTo(stream io.Writer) (int64, error) {
 	}
 
 	// Write set
-	err = binary.Write(stream, binaryOrder, b.set)
+	// current implementation of bufio.Writer is more memory efficient than
+	// binary.Write for large set
+	writer := bufio.NewWriter(stream)
+	var item = make([]byte, binary.Size(uint64(0))) // for serializing one uint64
+	for i := range b.set {
+		binaryOrder.PutUint64(item, b.set[i])
+		if nn, err := writer.Write(item); err != nil {
+			return int64(i*binary.Size(uint64(0)) + nn), err
+		}
+	}
+
+	err = writer.Flush()
 	return int64(b.BinaryStorageSize()), err
 }
 
@@ -917,9 +928,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) {
 	}
 
 	// Read remaining bytes as set
-	err = binary.Read(stream, binaryOrder, newset.set)
-	if err != nil {
-		return 0, err
+	// current implementation bufio.Reader is more memory efficient than
+	// binary.Read for large set
+	reader := bufio.NewReader(stream)
+	var item = make([]byte, binary.Size(uint64(0))) // one uint64
+	for i := uint64(0); i < length; i++ {
+		if _, err := reader.Read(item); err != nil {
+			if err == io.EOF {
+				break // done
+			}
+			return 0, err
+		}
+		newset.set[i] = binaryOrder.Uint64(item)
 	}
 
 	*b = *newset
@@ -929,25 +949,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) {
 // MarshalBinary encodes a BitSet into a binary form and returns the result.
 func (b *BitSet) MarshalBinary() ([]byte, error) {
 	var buf bytes.Buffer
-	writer := bufio.NewWriter(&buf)
-
-	_, err := b.WriteTo(writer)
+	_, err := b.WriteTo(&buf)
 	if err != nil {
 		return []byte{}, err
 	}
 
-	err = writer.Flush()
-
 	return buf.Bytes(), err
 }
 
 // UnmarshalBinary decodes the binary form generated by MarshalBinary.
 func (b *BitSet) UnmarshalBinary(data []byte) error {
 	buf := bytes.NewReader(data)
-	reader := bufio.NewReader(buf)
-
-	_, err := b.ReadFrom(reader)
-
+	_, err := b.ReadFrom(buf)
 	return err
 }