Stream Merkle root computation when downloading

go.mod

@@ -1,6 +1,6 @@
 module lukechampine.com/us
 
-go 1.12
+go 1.13
 
 require (
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da

merkle/merkle.go

@@ -2,6 +2,8 @@
 package merkle // import "lukechampine.com/us/merkle"
 
 import (
+	"io"
+
 	"gitlab.com/NebulousLabs/Sia/crypto"
 	"lukechampine.com/us/renterhost"
 )
@@ -42,3 +44,19 @@ func MetaRoot(roots []crypto.Hash) crypto.Hash {
 	}
 	return s.root()
 }
+
+// ReaderRoot returns the Merkle root of the supplied stream, which must contain
+// an integer multiple of segments.
+func ReaderRoot(r io.Reader) (crypto.Hash, error) {
+	var s stack
+	leaf := make([]byte, SegmentSize)
+	for {
+		if _, err := io.ReadFull(r, leaf); err == io.EOF {
+			break
+		} else if err != nil {
+			return crypto.Hash{}, err
+		}
+		s.appendLeaf(leaf)
+	}
+	return s.root(), nil
+}

merkle/merkle_test.go

@@ -1,8 +1,12 @@
 package merkle
 
 import (
+	"bytes"
+	"io"
+	"math/bits"
 	"reflect"
 	"testing"
+	"testing/iotest"
 
 	"gitlab.com/NebulousLabs/Sia/crypto"
 	"golang.org/x/crypto/blake2b"
@@ -788,3 +792,146 @@ func BenchmarkVerifyPrecomputedDiffProof(b *testing.B) {
 		VerifyDiffProof(actions, numSectors, treeHashes, leafHashes, oldRoot, newRoot, appendRoots)
 	}
 }
+
+func TestReaderRoot(t *testing.T) {
+	var sector [renterhost.SectorSize]byte
+	frand.Read(sector[:])
+	leafHashes := make([]crypto.Hash, SegmentsPerSector)
+	for i := range leafHashes {
+		leafHashes[i] = leafHash(sector[i*SegmentSize:][:SegmentSize])
+	}
+
+	root, err := ReaderRoot(bytes.NewReader(sector[:]))
+	if err != nil {
+		t.Fatal(err)
+	} else if root != SectorRoot(&sector) {
+		t.Fatal("root mismatch")
+	}
+
+	// test some random tree sizes against MetaRoot
+	for i := 0; i < 10; i++ {
+		numSegs := 1 << frand.Intn(bits.TrailingZeros(SegmentsPerSector))
+		root, err := ReaderRoot(bytes.NewReader(sector[:numSegs*SegmentSize]))
+		if err != nil {
+			t.Fatal(err)
+		} else if root != MetaRoot(leafHashes[:numSegs]) {
+			t.Fatal("root mismatch")
+		}
+	}
+
+	// test bad reader
+	_, err = ReaderRoot(iotest.OneByteReader(bytes.NewReader(sector[:])))
+	if err == io.ErrUnexpectedEOF {
+		t.Fatal("expected io.ErrUnexpectedEOF, got", err)
+	}
+}
+
+func BenchmarkReaderRoot(b *testing.B) {
+	var sector [renterhost.SectorSize]byte
+	r := bytes.NewReader(sector[:])
+	b.SetBytes(renterhost.SectorSize)
+	for i := 0; i < b.N; i++ {
+		ReaderRoot(r)
+		r.Seek(0, io.SeekStart)
+	}
+}
+
+func TestRangeProofVerifier(t *testing.T) {
+	// same as TestBuildVerifyProof, but using RangeProofVerifier
+
+	var sector [renterhost.SectorSize]byte
+	frand.Read(sector[:])
+	sectorRoot := SectorRoot(&sector)
+	segmentRoots := make([]crypto.Hash, SegmentsPerSector)
+	for i := range segmentRoots {
+		segmentRoots[i] = leafHash(sector[i*SegmentSize:][:SegmentSize])
+	}
+
+	verifyProof := func(proof []crypto.Hash, data []byte, start, end int, root crypto.Hash) bool {
+		rpv := NewRangeProofVerifier(start, end)
+		rpv.ReadFrom(bytes.NewReader(data))
+		return rpv.Verify(proof, root)
+	}
+
+	proof := BuildProof(&sector, 0, 1, nil)
+	hash := leafHash(sector[:64])
+	for i := range proof {
+		hash = nodeHash(hash, proof[i])
+	}
+	if !verifyProof(proof, sector[:64], 0, 1, hash) {
+		t.Error("RangeProofVerifier failed to verify a known correct proof")
+	}
+
+	proof = BuildProof(&sector, SegmentsPerSector-1, SegmentsPerSector, nil)
+	hash = leafHash(sector[len(sector)-64:])
+	for i := range proof {
+		hash = nodeHash(proof[len(proof)-i-1], hash)
+	}
+	if !verifyProof(proof, sector[len(sector)-64:], SegmentsPerSector-1, SegmentsPerSector, hash) {
+		t.Error("RangeProofVerifier failed to verify a known correct proof")
+	}
+
+	proof = BuildProof(&sector, 10, 11, nil)
+	hash = leafHash(sector[10*64:][:64])
+	hash = nodeHash(hash, proof[2])
+	hash = nodeHash(proof[1], hash)
+	hash = nodeHash(hash, proof[3])
+	hash = nodeHash(proof[0], hash)
+	for i := 4; i < len(proof); i++ {
+		hash = nodeHash(hash, proof[i])
+	}
+	if !verifyProof(proof, sector[10*64:11*64], 10, 11, hash) {
+		t.Error("RangeProofVerifier failed to verify a known correct proof")
+	}
+
+	// this is the largest possible proof
+	midl, midr := SegmentsPerSector/2-1, SegmentsPerSector/2+1
+	proof = BuildProof(&sector, midl, midr, nil)
+	left := leafHash(sector[midl*64:][:64])
+	for i := 0; i < len(proof)/2; i++ {
+		left = nodeHash(proof[len(proof)/2-i-1], left)
+	}
+	right := leafHash(sector[(midr-1)*64:][:64])
+	for i := len(proof) / 2; i < len(proof); i++ {
+		right = nodeHash(right, proof[i])
+	}
+	if !verifyProof(proof, sector[midl*64:midr*64], midl, midr, hash) {
+		t.Error("RangeProofVerifier failed to verify a known correct proof")
+	}
+
+	// test some random proofs against VerifyProof
+	for i := 0; i < 5; i++ {
+		start := frand.Intn(SegmentsPerSector - 1)
+		end := start + frand.Intn(SegmentsPerSector-start) + 1
+		proof := BuildProof(&sector, start, end, nil)
+		if !verifyProof(proof, sector[start*SegmentSize:end*SegmentSize], start, end, sectorRoot) {
+			t.Errorf("BuildProof constructed an incorrect proof for range %v-%v", start, end)
+		}
+	}
+
+	// test a proof with precomputed inputs
+	leftRoots := make([]crypto.Hash, SegmentsPerSector/2)
+	for i := range leftRoots {
+		leftRoots[i] = leafHash(sector[i*SegmentSize:][:SegmentSize])
+	}
+	left = MetaRoot(leftRoots)
+	precalc := func(i, j int) (h crypto.Hash) {
+		if i == 0 && j == SegmentsPerSector/2 {
+			h = left
+		}
+		return
+	}
+	proof = BuildProof(&sector, SegmentsPerSector-1, SegmentsPerSector, precalc)
+	recalcProof := BuildProof(&sector, SegmentsPerSector-1, SegmentsPerSector, nil)
+	if !reflect.DeepEqual(proof, recalcProof) {
+		t.Fatal("precalc failed")
+	}
+
+	// test malformed inputs
+	if verifyProof(nil, make([]byte, SegmentSize), 0, 1, crypto.Hash{}) {
+		t.Error("RangeProofVerifier verified an incorrect proof")
+	}
+	if verifyProof([]crypto.Hash{{}}, sector[:], 0, SegmentsPerSector, crypto.Hash{}) {
+		t.Error("RangeProofVerifier verified an incorrect proof")
+	}
+}

merkle/proof.go

@@ -1,6 +1,7 @@
 package merkle
 
 import (
+	"io"
 	"math/bits"
 	"sort"
 	"unsafe"
@@ -423,3 +424,57 @@ func modifyLeaves(leafHashes []crypto.Hash, actions []renterhost.RPCWriteAction,
 	}
 	return leafHashes
 }
+
+// A RangeProofVerifier allows for proofs to be verified in streaming fashion.
+type RangeProofVerifier struct {
+	start, end int
+	roots      []crypto.Hash
+}
+
+// ReadFrom implements io.ReaderFrom.
+func (rpv *RangeProofVerifier) ReadFrom(r io.Reader) (int64, error) {
+	var total int64
+	i, j := rpv.start, rpv.end
+	for i < j {
+		subtreeSize := nextSubtreeSize(i, j)
+		n := int64(subtreeSize * SegmentSize)
+		root, err := ReaderRoot(io.LimitReader(r, n))
+		if err != nil {
+			return total, err
+		}
+		total += n
+		rpv.roots = append(rpv.roots, root)
+		i += subtreeSize
+	}
+	return total, nil
+}
+
+// Verify verifies the supplied proof, using the data ingested from ReadFrom.
+func (rpv *RangeProofVerifier) Verify(proof []crypto.Hash, root crypto.Hash) bool {
+	if len(proof) != ProofSize(SegmentsPerSector, rpv.start, rpv.end) {
+		return false
+	}
+	var s stack
+	consume := func(roots *[]crypto.Hash, i, j int) {
+		for i < j && len(*roots) > 0 {
+			subtreeSize := nextSubtreeSize(i, j)
+			height := bits.TrailingZeros(uint(subtreeSize)) // log2
+			s.insertNodeHash((*roots)[0], height)
+			*roots = (*roots)[1:]
+			i += subtreeSize
+		}
+	}
+	consume(&proof, 0, rpv.start)
+	consume(&rpv.roots, rpv.start, rpv.end)
+	consume(&proof, rpv.end, SegmentsPerSector)
+	return s.root() == root
+}
+
+// NewRangeProofVerifier returns a RangeProofVerifier for the sector range
+// [start, end).
+func NewRangeProofVerifier(start, end int) *RangeProofVerifier {
+	return &RangeProofVerifier{
+		start: start,
+		end:   end,
+	}
+}