add encode and decode data method

build/storage/storage.go

@@ -9,6 +9,8 @@ import (
 
 type Data = io.Reader
 
+type Hash = [32]byte
+
 type SectorRef struct {
 	ID        abi.SectorID
 	ProofType abi.RegisteredSealProof

build/storiface/filetype.go

@@ -109,12 +109,8 @@ func ParseSectorID(baseName string) (abi.SectorID, error) {
 	}, nil
 }
 
-func SectorName(sid abi.SectorID, numbers ...int32) string {
-	out := fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
-	for _, number := range numbers{
-		out = fmt.Sprintf("%s-%d", out, number)
-	}
-	return out
+func SectorName(sid abi.SectorID) string {
+	return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
 }
 
 func PathByType(sps SectorPaths, fileType SectorFileType) string {

cmd/bench/main.go

@@ -24,8 +24,10 @@ func main() {
 		Usage:   "Benchmark performance of seal and window-post",
 		Version: "1.11.1",
 		Commands: []*cli.Command{
+			test,
 			testSealCmd,
 			testAggregationCmd,
+			testSplitDataCmd,
 		},
 	}
 
@@ -35,6 +37,16 @@ func main() {
 	}
 }
 
+var test = &cli.Command{
+	Name:   "test",
+	Usage:  "Test interface",
+	Action: func(c *cli.Context) error {
+		// Test 8MiB sector
+		seal.Test()
+		return nil
+	},
+}
+
 var testSealCmd = &cli.Command{
 	Name:   "test-seal",
 	Usage:  "Test interface",
@@ -48,6 +60,19 @@ var testSealCmd = &cli.Command{
 	},
 }
 
+var testSplitDataCmd = &cli.Command{
+	Name:   "test-split",
+	Usage:  "Test interface",
+	Action: func(c *cli.Context) error {
+		// Test 8MiB sector
+		err := seal.TestSplitDataInToPieces()
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
 var testAggregationCmd = &cli.Command{
 	Name:   "test-aggregation",
 	Usage:  "Test interface",

go.mod

@@ -24,6 +24,7 @@ require (
 	go.opencensus.io v0.23.0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	github.com/libp2p/go-libp2p-core v0.8.5
+	github.com/minio/sha256-simd v0.1.1
 )
 
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

seal/basicfs/fs.go

@@ -28,7 +28,7 @@ func (b *Provider) GetRoot() string {
 	return b.Root
 }
 
-func (b *Provider) AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType, numbers ...int32) (storiface.SectorPaths, func(), error) {
+func (b *Provider) AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
 	if err := os.Mkdir(filepath.Join(b.Root, storiface.FTUnsealed.String()), 0755); err != nil && !os.IsExist(err) { // nolint
 		return storiface.SectorPaths{}, nil, err
 	}
@@ -68,7 +68,7 @@ func (b *Provider) AcquireSector(ctx context.Context, id storage.SectorRef, exis
 			return storiface.SectorPaths{}, nil, ctx.Err()
 		}
 
-		path := filepath.Join(b.Root, fileType.String(), storiface.SectorName(id.ID, numbers...))
+		path := filepath.Join(b.Root, fileType.String(), storiface.SectorName(id.ID))
 
 		prevDone := done
 		done = func() {

seal/test_seal.go

@@ -3,9 +3,11 @@ package seal
 import(
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"math/rand"
+	"path/filepath"
 	"time"
 
 	"golang.org/x/xerrors"
@@ -16,6 +18,7 @@ import(
 	spproof "fil_integrate/build/proof"
 	"fil_integrate/build"
 	"fil_integrate/build/storage"
+	"fil_integrate/build/storiface"
 	// "fil_integrate/extern/sector-storage/ffiwrapper"
 	"fil_integrate/seal/basicfs"
 )
@@ -130,7 +133,7 @@ func TestAggregateWindowPoSt(
 
 	for i := 0; i < numAggregate; i++{
 		log.Infof("[%d] Generating Window-Post", i)
-		proof, _, err := sb.GenProofForWindowPoSt(ctx, 1000, sealedSectorsinfo[i], challenge[:])
+		proof, _, err := sb.GenerateWindowPoStProofs(ctx, 1000, sealedSectorsinfo[i], challenge[:])
 		if err != nil {
 			return err
 		}
@@ -214,7 +217,7 @@ func TestAggregateWindowPoSt(
 
 func TestSealAndUnseal() error {
 	//********************need (sb,ctx,sid,sectorSize,file,seed,ticket,challenge)****************//
-	sdir, err := homedir.Expand("~/tmp")
+	sdir, err := homedir.Expand("~/tmp/bench")
 	if err != nil {
 		return err
 	}
@@ -293,7 +296,7 @@ func TestSealAndUnseal() error {
 		SealProof:    sid.ProofType,
 	})
 
-	proof2, err := sb.GenProofForC2(ctx, sid, seed, ticket, pieces, cids)
+	proof2, err := sb.GenerateCommit2Proof(ctx, sid, seed, ticket, pieces, cids)
 	if err != nil {
 		return err
 	}
@@ -346,7 +349,7 @@ func TestSealAndUnseal() error {
 		return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
 	}
 
-	proof, _, err := sb.GenProofForWindowPoSt(ctx, sid.ID.Miner, sealedSectors, challenge[:])
+	proof, _, err := sb.GenerateWindowPoStProofs(ctx, sid.ID.Miner, sealedSectors, challenge[:])
 
 	wpvi := spproof.WindowPoStVerifyInfo{
 		Randomness:        challenge[:],
@@ -367,6 +370,97 @@ func TestSealAndUnseal() error {
 
 }
 
+func TestSplitDataInToPieces() error {
+	sdir, err := homedir.Expand("~/tmp/bench")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(sdir, 0775) //nolint:gosec
+	if err != nil {
+		return xerrors.Errorf("creating sectorbuilder dir: %w", err)
+	}
+
+	tsdir, err := ioutil.TempDir(sdir, "bench")
+	if err != nil {
+		return err
+	}
+	// defer func() {
+	// 	if err := os.RemoveAll(tsdir); err != nil {
+	// 		log.Warn("remove all: ", err)
+	// 	}
+	// }()
+
+	// TODO: pretty sure this isnt even needed?
+	if err := os.MkdirAll(tsdir, 0775); err != nil {
+		return err
+	}
+	sbfs := &basicfs.Provider{
+			Root: tsdir,
+		}
+	sb, err := New(sbfs)
+	if err != nil{
+		return err
+	}
+	ctx := context.TODO()
+	sectorSize := abi.SectorSize(4*1024*1024)
+
+	root, err := homedir.Expand("~/tmp")
+	if err != nil {
+		return err
+	}
+	filename := filepath.Join(root, "input.dat")
+	err = generateRandomData(filename)
+	if err != nil {
+		return err
+	}
+	in, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	finalHash, err := sb.EncodeDataToPieces(ctx, sectorSize, in)
+	if err != nil{
+		return err
+	}
+
+	filename = filepath.Join(root, "output.dat")
+	out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	err = decodePiecesToData(sb, ctx, tsdir, sectorSize, finalHash, out)
+	if err != nil {
+		return err
+	}
+
+	ok, err := checkDecodedFile(root)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		fmt.Println("decode pieces failed")
+	} else {
+		fmt.Println("decode pieces success")
+	}
+
+	return nil
+}
+
+func Test() int {
+	var buf1 []byte
+	var buf2 []byte
+	buf1 = append(buf1, 0,1,2,3)
+	buf2 = append(buf2, 10,20,30,40)
+	buf1 = append(buf2, buf1...)
+	fmt.Println(buf1, len(buf1), buf1[4])
+	fmt.Println(buf2, len(buf2), buf2[4])
+	return 0
+}
+
 func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
 	spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
 	if err != nil {
@@ -374,4 +468,149 @@ func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
 	}
 
 	return spt
+}
+
+func generateRandomData(filename string) error {
+	Datasize := 128*1024*1024
+	buf := make([]byte, Datasize)
+	b := []byte("random string!")
+	for i:=0; i<Datasize; i=i+32{
+		temphash := blake2b.Sum256(b)
+		copy(b, temphash[:])
+		copy(buf[i:i+32],temphash[:])
+	}
+	_ = os.Remove(filename)
+	f,err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.Write(buf[:])
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func decodePiecesToData(sb *Sealer, ctx context.Context, tsdir string, sectorSize abi.SectorSize, finalHash storage.Hash, out io.Writer) error {
+	var hashData []storage.Hash
+	DataLen := abi.PaddedPieceSize(sectorSize).Unpadded() - 8
+	filename := filepath.Join(tsdir, "pieces", fmt.Sprintf("%x.dat", finalHash[:]))
+	fmt.Printf("Decode: %x.dat\n", finalHash[:])
+	file, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	hasPre, preHash, metaData, commData, err := sb.DecodePiece(ctx, sectorSize, file, 0, storiface.UnpaddedByteIndex(DataLen))
+	if err != nil {
+		return err
+	}
+
+	for ; hasPre; {
+		commData = append(hashData, commData...)
+		filename = filepath.Join(tsdir, "pieces", fmt.Sprintf("%x.dat", preHash[:]))
+		fmt.Printf("Decode: %x.dat\n", preHash[:])
+		file, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		hasPre, preHash, metaData, hashData, err = sb.DecodePiece(ctx, sectorSize, file, 0, storiface.UnpaddedByteIndex(DataLen))
+		if err != nil{
+			return err
+		}
+	}
+
+	for _, pieceHash := range commData {
+		filename = filepath.Join(tsdir, "pieces", fmt.Sprintf("%x.dat", pieceHash[:]))
+		fmt.Printf("Decode: %x.dat\n", pieceHash[:])
+		file, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+		_, _, data, _, err := sb.DecodePiece(ctx, sectorSize, file, 0, storiface.UnpaddedByteIndex(DataLen))
+		if err != nil {
+			return err
+		}
+		for wbuf := data[:]; len(wbuf) > 0; {
+			n, err := out.Write(wbuf)
+			if err != nil{
+				return err
+			}
+			wbuf = wbuf[n:]
+		}
+	}
+	for wbuf := metaData[:]; len(wbuf) > 0; {
+		n, err := out.Write(wbuf)
+		if err != nil{
+			return err
+		}
+		wbuf = wbuf[n:]
+	}
+	return nil
+}
+
+func checkDecodedFile(root string) (bool, error) {
+	filename := filepath.Join(root, "input.dat")
+	in, err := os.Open(filename)
+	if err != nil {
+		return false, err
+	}
+	defer in.Close()
+
+	filename = filepath.Join(root, "output.dat")
+	out, err := os.Open(filename)
+	if err != nil {
+		return false, err
+	}
+	defer out.Close()
+
+	inBuf := make([]byte, 2<<20)
+	outBuf := make([]byte, 2<<20)
+
+	for{
+		var readin int
+		var readout int
+		for wbuf := inBuf[:]; len(wbuf) > 0; {
+			n, err := in.Read(wbuf)
+			if err != nil && err != io.EOF{
+				return false, err
+			}
+
+			wbuf = wbuf[n:]
+			readin += n
+
+			if err == io.EOF {
+				break
+			}
+		}
+		for wbuf := outBuf[:]; len(wbuf) > 0; {
+			n, err := out.Read(wbuf)
+			if err != nil && err != io.EOF{
+				return false, err
+			}
+
+			wbuf = wbuf[n:]
+			readout += n
+			
+			if err == io.EOF {
+				break
+			}
+		}
+
+		if readin != readout {
+			return false, xerrors.Errorf("the output data and input data do not match")
+		}
+
+		if readin == 0 {
+			break
+		}
+
+		for index := 0; index < readin; index++ {
+			if inBuf[index] != outBuf[index] {
+				return false, xerrors.Errorf("the output data and input data do not match")
+			}
+		}
+	}
+	return true, nil
 }
\ No newline at end of file

seal/type.go

@@ -15,9 +15,14 @@ import(
 
 //interface
 type SectorSealer interface{
-	AddPiece(ctx context.Context, sector storage.SectorRef, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data, numbers ...int32) (abi.PieceInfo, error)
+	AddPiece(ctx context.Context, sector storage.SectorRef, existingPieceSizes []abi.UnpaddedPieceSize, pieceSize abi.UnpaddedPieceSize, file storage.Data) (abi.PieceInfo, error)
+	
 	CheckPieceAndDataRoot(sid storage.SectorRef, commd cid.Cid, pieces []abi.PieceInfo) (bool, error)
-	Sealed(ctx context.Context, sid storage.SectorRef, seed abi.InteractiveSealRandomness, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.SectorCids, storage.Proof, error) 
+	Sealed(ctx context.Context, sid storage.SectorRef, seed abi.InteractiveSealRandomness, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.SectorCids, []byte, error) 
+	// Split and encode data into pieces
+	// Pieces structure is [ Tag | MetaData | HashData ] or [ Tag | PreHash | HashData]
+	EncodeDataToPieces(ctx context.Context, sectorSize abi.SectorSize, file storage.Data) (storage.Hash, error)
+	DecodePiece(ctx context.Context, sectorSize abi.SectorSize, in io.Reader, start storiface.UnpaddedByteIndex, end storiface.UnpaddedByteIndex) (bool, storage.Hash, []byte, []storage.Hash, error)
 	GenerateCommit2Proof( ctx context.Context, sid storage.SectorRef, seed abi.InteractiveSealRandomness, ticket abi.SealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Proof, error)
 	GenerateWindowPoStProofs(ctx context.Context, minerID abi.ActorID, sectorInfo []spproof.SectorInfo, randomness abi.PoStRandomness) ([]spproof.PoStProof, []abi.SectorID, error)
 	UnsealedRange(ctx context.Context, sid storage.SectorRef, sectorSize abi.SectorSize, ticket abi.SealRandomness, commd cid.Cid, out io.Writer, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error
@@ -37,7 +42,7 @@ type SectorProvider interface {
 	// * returns storiface.ErrSectorNotFound if a requested existing sector doesn't exist
 	// * returns an error when allocate is set, and existing isn't, and the sector exists
 	AcquireUnsealed(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error)
-	AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType, numbers ...int32) (storiface.SectorPaths, func(), error)
+	AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error)
 }
 
 var _ SectorProvider = &basicfs.Provider{}
\ No newline at end of file