all: switch from merkle tree to serial hash

blob/blob.go

@@ -1,11 +1,12 @@
 package blob
 
 import (
-	"github.com/pkg/errors"
 	"io"
 	"io/ioutil"
 	"os"
 	"sync"
+
+	"github.com/pkg/errors"
 )
 
 type SectorReader interface {
@@ -22,12 +23,15 @@ type Blob interface {
 	io.Closer
 	Readable
 	Transaction() (Transaction, error)
+	Seek(uint16)
+	At() uint16
 }
 
 type blobImpl struct {
-	f    *os.File
-	name string
-	mu   sync.Mutex
+	f          *os.File
+	sectorSize uint16
+	name       string
+	mu         sync.Mutex
 }
 
 func newFromFile(name string, f *os.File) Blob {
@@ -41,6 +45,14 @@ func (b *blobImpl) Name() string {
 	return b.name
 }
 
+func (b *blobImpl) Seek(sectorSize uint16) {
+	b.sectorSize = sectorSize
+}
+
+func (b *blobImpl) At() uint16 {
+	return b.sectorSize
+}
+
 func (b *blobImpl) ReadSector(id uint8) (Sector, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -55,10 +67,11 @@ func (b *blobImpl) ReadAt(p []byte, off int64) (int, error) {
 
 func (b *blobImpl) Transaction() (Transaction, error) {
 	return &txImpl{
-		name:      b.name,
-		cloner:    b.txCloner,
-		committer: b.txCommitter,
-		remover:   b.txRemover,
+		name:       b.name,
+		sectorSize: b.sectorSize,
+		cloner:     b.txCloner,
+		committer:  b.txCommitter,
+		remover:    b.txRemover,
 	}, nil
 }
 

blob/blob_test.go

@@ -2,9 +2,10 @@ package blob
 
 import (
 	"crypto/rand"
-	"github.com/stretchr/testify/require"
 	"io"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 // note: transaction functionality is tested in transaction_test.go
@@ -18,13 +19,13 @@ func TestBlob_Reading(t *testing.T) {
 	require.Equal(t, "foobar", bl.Name())
 
 	var expSector Sector
-	_, err = f.ReadAt(expSector[:], 65536)
+	_, err = f.ReadAt(expSector[:], 256)
 	require.NoError(t, err)
 	actSector, err := bl.ReadSector(1)
 	require.NoError(t, err)
 	require.Equal(t, expSector, actSector)
 	actData := make([]byte, 10, 10)
-	_, err = bl.ReadAt(actData, 65536)
+	_, err = bl.ReadAt(actData, 256)
 	require.NoError(t, err)
 	require.EqualValues(t, expSector[:10], actData)
 }

blob/consts.go

@@ -2,8 +2,8 @@ package blob
 
 const (
 	HeaderLen      = 1 + 1 + 63 + 8 + 32 + 65
-	SectorLen      = 4096
-	SectorCount    = 256
+	SectorLen      = 256
+	SectorCount    = 4096
 	Size           = SectorLen * SectorCount
 	CurrentVersion = 1
 )

blob/hash.go

@@ -0,0 +1,90 @@
+package blob
+
+import (
+	"io"
+
+	"fnd/crypto"
+
+	"golang.org/x/crypto/blake2b"
+)
+
+var (
+	ZeroHash         crypto.Hash
+	ZeroSectorHashes SectorHashes
+)
+
+type SectorHashes [SectorCount]crypto.Hash
+
+func (s SectorHashes) Encode(w io.Writer) error {
+	for _, h := range s {
+		if _, err := w.Write(h[:]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *SectorHashes) Decode(r io.Reader) error {
+	var res SectorHashes
+	var hash crypto.Hash
+	for i := 0; i < len(res); i++ {
+		if _, err := r.Read(hash[:]); err != nil {
+			return err
+		}
+		res[i] = hash
+	}
+
+	*s = res
+	return nil
+}
+
+func (s SectorHashes) DiffWith(other SectorHashes) int {
+	if s == other {
+		return -1
+	}
+
+	var i int
+	for i = 0; i < len(s); i++ {
+		if s[i] != other[i] {
+			break
+		}
+	}
+	return i
+}
+
+func (s SectorHashes) Tip() crypto.Hash {
+	var i int
+	for i = 0; i < SectorCount; i++ {
+		if s[i] == ZeroHash {
+			break
+		}
+	}
+	if i == 0 {
+		return crypto.ZeroHash
+	}
+	return s[i-1]
+}
+
+func SerialHashSector(sector Sector, prevHash crypto.Hash) crypto.Hash {
+	var res crypto.Hash
+	hasher, _ := blake2b.New256(nil)
+	hasher.Write(prevHash[:])
+	hasher.Write(sector[:])
+	h := hasher.Sum(nil)
+	copy(res[:], h)
+	return res
+}
+
+// SerialHash returns serial hash of the contents of the reader br
+func SerialHash(br io.Reader, prevHash crypto.Hash, sectorSize uint16) (SectorHashes, error) {
+	var res SectorHashes
+	var sector Sector
+	for i := 0; i < int(sectorSize); i++ {
+		if _, err := br.Read(sector[:]); err != nil {
+			return ZeroSectorHashes, err
+		}
+		res[i] = SerialHashSector(sector, prevHash)
+		prevHash = res[i]
+	}
+	return res, nil
+}

blob/io.go

@@ -1,8 +1,9 @@
 package blob
 
 import (
-	"github.com/pkg/errors"
 	"io"
+
+	"github.com/pkg/errors"
 )
 
 var (
@@ -47,7 +48,7 @@ func WriteBlobAt(w io.WriterAt, b []byte, off int64) (int, error) {
 	return w.WriteAt(b, off)
 }
 
-func WriteSector(w io.WriterAt, id uint8, sector Sector) error {
+func WriteSector(w io.WriterAt, id uint16, sector Sector) error {
 	_, err := w.WriteAt(sector[:], int64(id)*int64(SectorLen))
 	return err
 }

blob/io_test.go

@@ -1,9 +1,10 @@
 package blob
 
 import (
-	"github.com/stretchr/testify/require"
 	"io"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 type zeroReader struct{}
@@ -110,7 +111,7 @@ func TestReadSector(t *testing.T) {
 		},
 		{
 			255,
-			16711680,
+			65280,
 		},
 	}
 	r := &readerWrapper{
@@ -189,7 +190,7 @@ func TestWriteSector(t *testing.T) {
 		},
 		{
 			255,
-			16711680,
+			65280,
 		},
 	}
 	w := &writerWrapper{

blob/merkle.go

@@ -2,19 +2,21 @@ package blob
 
 import (
 	"bytes"
-	"fnd/crypto"
-	"github.com/pkg/errors"
 	"io"
 	"math"
 	"math/bits"
 	"strings"
+
+	"fnd/crypto"
+
+	"github.com/pkg/errors"
 )
 
 const (
 	MerkleTreeHeight = 8
 	MerkleProofLen   = 8 * 32
 
-	SubsectorSize        = 4096
+	SubsectorSize        = 256
 	SubsectorCountBlob   = Size / SubsectorSize
 	SubsectorCountSector = SectorLen / SubsectorSize
 	SubsectorProofLevel  = 8
@@ -38,11 +40,10 @@ var (
 		{0xaa, 0xbd, 0xcb, 0xb2, 0x3b, 0xfc, 0xea, 0xfe, 0x71, 0xf3, 0x83, 0x4a, 0x17, 0xec, 0x1d, 0x24, 0xbd, 0x4e, 0xf2, 0xda, 0xed, 0x68, 0xd2, 0xcc, 0xc3, 0xc2, 0x98, 0x9f, 0xd0, 0x92, 0xe3, 0x53}: {0x7d, 0x1e, 0x84, 0xe6, 0x2d, 0x7e, 0xc9, 0xf6, 0xbc, 0x3f, 0x88, 0x66, 0x75, 0x73, 0x3d, 0xa0, 0x6d, 0xaf, 0x02, 0x77, 0xad, 0x8f, 0xfc, 0xf7, 0xc5, 0x39, 0x93, 0x8c, 0xa5, 0xee, 0x9c, 0xf2},
 	}
 
-	zero4kSector     = make([]byte, 4096)
-	zero4kSectorHash = crypto.Hash{0x68, 0x6e, 0xde, 0x92, 0x88, 0xc3, 0x91, 0xe7, 0xe0, 0x50, 0x26, 0xe5, 0x6f, 0x2f, 0x91, 0xbf, 0xd8, 0x79, 0x98, 0x7a, 0x04, 0x0e, 0xa9, 0x84, 0x45, 0xda, 0xbc, 0x76, 0xf5, 0x5b, 0x8e, 0x5f}
-
-	EmptyBlobMerkleRoot = crypto.Hash{0x7d, 0x1e, 0x84, 0xe6, 0x2d, 0x7e, 0xc9, 0xf6, 0xbc, 0x3f, 0x88, 0x66, 0x75, 0x73, 0x3d, 0xa0, 0x6d, 0xaf, 0x02, 0x77, 0xad, 0x8f, 0xfc, 0xf7, 0xc5, 0x39, 0x93, 0x8c, 0xa5, 0xee, 0x9c, 0xf2}
-	EmptyBlobBaseHash   = crypto.Hash{0x53, 0x2a, 0x12, 0xf0, 0x9f, 0xeb, 0xf8, 0x52, 0x14, 0x19, 0x95, 0x99, 0x73, 0xad, 0x53, 0x46, 0x94, 0x4c, 0x2b, 0x22, 0xbf, 0x76, 0x4d, 0x0e, 0x1a, 0x34, 0x25, 0x5b, 0x65, 0x64, 0xfe, 0x4b}
+	zero4kSector        = make([]byte, 4096)
+	zero4kSectorHash    = crypto.Hash{0x68, 0x6e, 0xde, 0x92, 0x88, 0xc3, 0x91, 0xe7, 0xe0, 0x50, 0x26, 0xe5, 0x6f, 0x2f, 0x91, 0xbf, 0xd8, 0x79, 0x98, 0x7a, 0x04, 0x0e, 0xa9, 0x84, 0x45, 0xda, 0xbc, 0x76, 0xf5, 0x5b, 0x8e, 0x5f}
+	EmptyBlobMerkleRoot = crypto.Hash{0x0c, 0x63, 0x8a, 0xfe, 0x73, 0x32, 0x49, 0x76, 0x16, 0x59, 0x07, 0xbc, 0x33, 0x86, 0x2e, 0xbf, 0x74, 0xe9, 0x71, 0x32, 0x66, 0x6e, 0x6a, 0x68, 0x15, 0x4f, 0xe9, 0xf7, 0x42, 0xf4, 0x45, 0x7f}
+	EmptyBlobBaseHash   = crypto.Hash{0xe, 0x57, 0x51, 0xc0, 0x26, 0xe5, 0x43, 0xb2, 0xe8, 0xab, 0x2e, 0xb0, 0x60, 0x99, 0xda, 0xa1, 0xd1, 0xe5, 0xdf, 0x47, 0x77, 0x8f, 0x77, 0x87, 0xfa, 0xab, 0x45, 0xcd, 0xf1, 0x2f, 0xe3, 0xa8}
 	ZeroMerkleBase      MerkleBase
 )
 

blob/merkle_test.go

@@ -4,20 +4,15 @@ import (
 	"bytes"
 	"crypto/rand"
 	"encoding/hex"
-	"fnd/testutil/testfs"
-	"github.com/stretchr/testify/require"
 	"io"
 	"testing"
+
+	"fnd/testutil/testfs"
+
+	"github.com/stretchr/testify/require"
 )
 
-const expProof = "532a12f09febf8521419959973ad5346944c2b22bf764d0e1a34255b6564fe4b" +
-	"ffe2cf7ecd1b993235746be21e91c8e61a1e22dace9850912585416501e98447" +
-	"60522e01341fe96254668ba1bc2c79dd6fc66b3784c2eb39d4f07319c6232657" +
-	"ae0182ab7803fc44d085c1c734a552fffdb0f744179f0d95bd60dd6f8f1818af" +
-	"f34c7d70b652eba48e02b4717e1f0a2be9337b0751ccf7bf3644674cbe6423d5" +
-	"3200b99bfcd82c64c818b1a2b26e14bf784fe9188a559b6b38a6dda4fb553147" +
-	"b86be3a8b288b0ef0b7ee5a9852d118167a50c8471bfb9fb8f0c7992f452c79c" +
-	"aabdcbb23bfceafe71f3834a17ec1d24bd4ef2daed68d2ccc3c2989fd092e353"
+const expProof = "d87e2783f806556be18f0d7d71324c9efa47d76dcb6ab20ebfb1bd11fcdd8e16159fb570f424dd23426b626a4274af3fb0673a3d94df9ec9ea5b32a23fb4888cd394b989e56276a3b452ffc4c693c292fd3939b3b88dbb5f07caf89434e4c598ccc660bb61bf612c077f45f098cbe1c2fc417349705b9dffb4b4357dce2839bdc587910d908005e8d451b4305190ceabff410d145a569f8d2de57c89dfac823b16b0b07f28b35127b90ee00b4cc5612ae2d2d9e2a3343ac9c0261ef96d812d43a63c26e267529de752e0f066a3a882364f28da1f02e2a1dcd852580ee804d6c4fe623c1fbe4bdad753ab5a0c281c90bcc87c59fd3c65c6723a87b72b501ae0ea"
 
 func TestEncodeDecode(t *testing.T) {
 	f, done := testfs.NewTempFile(t)
@@ -47,14 +42,14 @@ func TestMerkleize(t *testing.T) {
 	blob := newFromFile("foobar", f)
 	mt, err := Merkleize(NewReader(blob))
 	require.NoError(t, err)
-	require.Equal(t, "7d1e84e62d7ec9f6bc3f886675733da06daf0277ad8ffcf7c539938ca5ee9cf2", hex.EncodeToString(mt.Root().Bytes()))
+	require.Equal(t, "f60de559740b01fe11dd241dee2729782caabd36613c469806a5449aaeef0a27", hex.EncodeToString(mt.Root().Bytes()))
 
 	_, err = f.WriteAt([]byte{0x01, 0x99, 0x99, 0x99, 0xff, 0xad, 0xfc, 0x11, 0x99}, int64(SectorLen*2))
 	require.NoError(t, err)
 
 	mt, err = Merkleize(NewReader(blob))
 	require.NoError(t, err)
-	require.Equal(t, "e9aa8bdf96bd79398d64a31865d2dd4d95eee2ba54b06a4e30b85e9e838497dd", hex.EncodeToString(mt.Root().Bytes()))
+	require.Equal(t, "75a87ea080e57925f81c6bf732bf72fb1f6465ee2ecbfaf7580b444fd3990cc7", hex.EncodeToString(mt.Root().Bytes()))
 }
 
 func TestGenProof(t *testing.T) {
@@ -95,6 +90,6 @@ func TestProtocolBase(t *testing.T) {
 	base := mt.ProtocolBase()
 
 	for i := 0; i < len(base); i++ {
-		require.Equal(t, "532a12f09febf8521419959973ad5346944c2b22bf764d0e1a34255b6564fe4b", hex.EncodeToString(base[i][:]))
+		require.Equal(t, "d87e2783f806556be18f0d7d71324c9efa47d76dcb6ab20ebfb1bd11fcdd8e16", hex.EncodeToString(base[i][:]))
 	}
 }

blob/pathify.go

@@ -3,7 +3,7 @@ package blob
 import (
 	"bytes"
 	"encoding/hex"
-	"fnd.localhost/handshake/primitives"
+	"github.com/mslipper/handshake/primitives"
 	"path"
 )
 

blob/seal.go

@@ -2,23 +2,26 @@ package blob
 
 import (
 	"fnd/crypto"
-	"fnd.localhost/dwire"
+	"fnd/dwire"
+
 	"golang.org/x/crypto/blake2b"
-	"time"
 )
 
-func SealHash(name string, ts time.Time, merkleRoot crypto.Hash, reservedRoot crypto.Hash) crypto.Hash {
+func SealHash(name string, epochHeight, sectorSize uint16, sectorTipHash crypto.Hash, reservedRoot crypto.Hash) crypto.Hash {
 	h, _ := blake2b.New256(nil)
-	if _, err := h.Write([]byte("FNBLOB")); err != nil {
+	if _, err := h.Write([]byte("DDRPBLOB")); err != nil {
 		panic(err)
 	}
 	if err := dwire.EncodeField(h, name); err != nil {
 		panic(err)
 	}
-	if err := dwire.EncodeField(h, ts); err != nil {
+	if err := dwire.EncodeField(h, epochHeight); err != nil {
+		panic(err)
+	}
+	if err := dwire.EncodeField(h, sectorSize); err != nil {
 		panic(err)
 	}
-	if _, err := h.Write(merkleRoot[:]); err != nil {
+	if _, err := h.Write(sectorTipHash[:]); err != nil {
 		panic(err)
 	}
 	if _, err := h.Write(reservedRoot[:]); err != nil {
@@ -30,7 +33,7 @@ func SealHash(name string, ts time.Time, merkleRoot crypto.Hash, reservedRoot cr
 	return out
 }
 
-func SignSeal(signer crypto.Signer, name string, ts time.Time, merkleRoot crypto.Hash, reservedRoot crypto.Hash) (crypto.Signature, error) {
-	h := SealHash(name, ts, merkleRoot, reservedRoot)
+func SignSeal(signer crypto.Signer, name string, epochHeight, sectorSize uint16, sectorTipHash crypto.Hash, reservedRoot crypto.Hash) (crypto.Signature, error) {
+	h := SealHash(name, epochHeight, sectorSize, sectorTipHash, reservedRoot)
 	return signer.Sign(h)
 }