add new test

build/storage/storage.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/ipfs/go-cid"
+
+	"fil_integrate/build/storiface"
 )
 
 type Data = io.Reader
@@ -20,6 +22,14 @@ type SectorRef struct {
 	ProofType abi.RegisteredSealProof
 }
 
+type RangeSector struct {
+	Sector   SectorRef
+	Sealed   cid.Cid
+	Unsealed cid.Cid
+	Offset   storiface.UnpaddedByteIndex
+	Size     abi.UnpaddedPieceSize
+}
+
 type PreCommit1Out []byte
 
 type Commit1Out []byte

cmd/bench/main.go

@@ -24,10 +24,8 @@ func main() {
 		Usage:   "Benchmark performance of seal and window-post",
 		Version: "1.0.1",
 		Commands: []*cli.Command{
-			test,
 			testSealAndWindowPoSt,
 			testSealCmd,
-			testAggregationCmd,
 			testSplitDataCmd,
 		},
 	}
@@ -38,15 +36,6 @@ func main() {
 	}
 }
 
-var test = &cli.Command{
-	Name:   "test",
-	Usage:  "Test interface",
-	Action: func(c *cli.Context) error {
-		seal.Test()
-		return nil
-	},
-}
-
 var testSealAndWindowPoSt = &cli.Command{
 	Name:   "test-all",
 	Usage:  "Test Seal the sectors and generate window post",
@@ -93,47 +82,30 @@ var testSealCmd = &cli.Command{
 var testSplitDataCmd = &cli.Command{
 	Name:   "test-split",
 	Usage:  "Test encode data into pieces",
-	Action: func(c *cli.Context) error {
-		// Test 8MiB sector
-		err := seal.TestSplitDataInToPieces()
-		if err != nil {
-			return err
-		}
-		return nil
-	},
-}
-
-var testAggregationCmd = &cli.Command{
-	Name:   "test-aggregation",
-	Usage:  "Test aggregate some window-post proofs",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:   "sector-size",
-			Value:  "2KiB",
+			Value:  "8MiB",
 			Usage:  "size of the sectors in bytes",
 		},
-		&cli.IntFlag{
-			Name:   "num-sectors",
-			Value:  4,
-			Usage:  "How many sectors used in single window post",
-		},
-		&cli.IntFlag{
-			Name:   "num-agg",
-			Value:  4,
-			Usage:  "How many window-post proofs used to aggregate",
+		&cli.StringFlag{
+			Name:   "data-size",
+			Value:  "256MiB",
+			Usage:  "size of the input file in bytes",
 		},
 	},
 	Action: func(c *cli.Context) error {
-		log.Info("testing")
+		// Test 8MiB sector
+		dataSize, err := units.RAMInBytes(c.String("data-size"))
+		if err != nil {
+			return err
+		}
 		sectorSizeInt, err := units.RAMInBytes(c.String("sector-size"))
 		if err != nil {
 			return err
 		}
 		sectorSize := abi.SectorSize(sectorSizeInt)
-		numSectors := c.Int("num-sectors")
-		numAggregate := c.Int("num-agg")
-
-		err = seal.TestAggregateWindowPoSt(sectorSize, numSectors, numAggregate)
+		err = seal.TestSplitDataInToPieces(sectorSize, uint64(dataSize))
 		if err != nil {
 			return err
 		}

seal/seal_api.go

@@ -21,6 +21,7 @@ import(
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/ipfs/go-cid"
 
+	"fil_integrate/build"
 	"fil_integrate/build/fr32"
 	spieces "fil_integrate/build/pieces"
 	"fil_integrate/build/storiface"
@@ -36,6 +37,8 @@ const TagLen uint32 = 8
 
 const NewestNetworkVersion = network.Version13
 
+var PicesNotEnoughError = xerrors.Errorf("can not use the existing pieces to fill the sector")
+
 type Encoder struct {
 	Root string
 }
@@ -94,6 +97,7 @@ func (sp *Encoder) EncodeDataToPieces(
 		if err != nil {
 			return storage.Piece{}, nil, err
 		}
+		// fmt.Printf("encode1: %x.dat\n", pieceHash[:])
 
 		hashData = append(hashData, pieceHash[:]...)
 		pieces = append(pieces, storage.Piece{
@@ -158,6 +162,7 @@ func (sp *Encoder) EncodeData(
 		if err != nil {
 			return nil, err
 		}
+		// fmt.Printf("encode2: %x.dat\n", prePieceHash[:])
 
 		pieces = append(pieces, storage.Piece{
 			Commitment: prePieceHash, 
@@ -203,8 +208,9 @@ func New(sectors SectorManager) (*Sealer, error) {
 func (sb *Sealer)AddPiece(
 	ctx context.Context, 
 	sector storage.SectorRef, 
-	pieces *[]storage.Piece, 
-) ([]abi.PieceInfo, error) {
+	sortedPieces []storage.Piece, 
+) ([]abi.PieceInfo, []storage.Piece, error) {
+	var index int
 	var addPieces []storage.Piece
 	var pieceSize abi.UnpaddedPieceSize
 	var existingPieceSizes []abi.UnpaddedPieceSize
@@ -212,34 +218,34 @@ func (sb *Sealer)AddPiece(
 
 	ssize, err := sector.ProofType.SectorSize()
 	if err != nil {
-		return nil, err
+		return nil, sortedPieces, err
 	}
 
 	maxPieceSize := abi.PaddedPieceSize(ssize).Unpadded()
 
 	pieceRoot := filepath.Join(sb.sectors.GetRoot(), "pieces")
-	for ;len(*pieces) > 0; {
-		pieceSize += (*pieces)[0].Size
+	for index = 0;index < len(sortedPieces); {
+		pieceSize += sortedPieces[index].Size
 		if pieceSize > maxPieceSize {
-			return nil, xerrors.Errorf("Exists a piece whose size is bigger than 8MiB or is not power of two or the pieces is not sorted")
+			return nil, sortedPieces, xerrors.Errorf("Exists a piece whose size is bigger than 8MiB or is not power of two or the pieces is not sorted")
 		} else if pieceSize == maxPieceSize {
-			addPieces = append(addPieces, (*pieces)[0])
-			(*pieces) = (*pieces)[1:]
+			addPieces = append(addPieces, sortedPieces[index])
+			index++
 			break
 		}
-		addPieces = append(addPieces, (*pieces)[0])
-		(*pieces) = (*pieces)[1:]
+		addPieces = append(addPieces, sortedPieces[index])
+		index++
 	}
 
 	if pieceSize != maxPieceSize {
-		return nil, xerrors.Errorf("can not use the existing pieces to generate 8MiB piece")
+		return nil, sortedPieces, PicesNotEnoughError
 	}
 
 	for _, piece := range(addPieces) {
 		filename := filepath.Join(pieceRoot, fmt.Sprintf("%x.dat", piece.Commitment[:]))
 		file, err := os.Open(filename)
 		if err != nil {
-			return nil, err
+			return nil, sortedPieces, err
 		}
 		defer func(){
 			file.Close()
@@ -248,13 +254,13 @@ func (sb *Sealer)AddPiece(
 		fmt.Printf("Adding %x.dat\n", piece.Commitment[:])
 		pieceInfo, err := sb.addPiece(ctx, sector, existingPieceSizes, piece.Size, file)
 		if err != nil {
-			return nil, err
+			return nil, sortedPieces, err
 		}
 		existingPieceSizes = append(existingPieceSizes, piece.Size)
 		piecesInfo = append(piecesInfo, pieceInfo)
 	}
 
-	return piecesInfo, nil
+	return piecesInfo, sortedPieces[index:], nil
 }
 
 func (sb *Sealer)addPiece( 
@@ -502,9 +508,8 @@ func ToReadableFile(r io.Reader, n int64) (*os.File, func() error, error) {
 func (sb *Sealer)UnsealedRange(
 	ctx        context.Context, 
 	sid        storage.SectorRef, 
-	sectorSize abi.SectorSize,  
-	commd      cid.Cid, 
 	out        io.Writer, 
+	commd      cid.Cid, 
 	offset     storiface.UnpaddedByteIndex, 
 	size       abi.UnpaddedPieceSize,
 ) error {
@@ -521,7 +526,7 @@ func (sb *Sealer)UnsealedRange(
 		}
 	}
 
-	err := sb.UnsealPiece(ctx, sid, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), commd)
+	err := sb.UnsealPiece(ctx, sid, offset, size, commd)
 	if err != nil {
 		return err
 	}
@@ -1033,13 +1038,13 @@ func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorIn
 }
 
 type Verifier struct {
-	Lock *sync.RWMutex
-	SM   map[abi.SectorID]storage.SectorCids
+	lock *sync.RWMutex
+	sm   map[abi.SectorID]storage.SectorCids
 }
 
 var ProofVerifier = Verifier{
-	Lock: new(sync.RWMutex),
-	SM:   make(map[abi.SectorID]storage.SectorCids),
+	lock: new(sync.RWMutex),
+	sm:   make(map[abi.SectorID]storage.SectorCids),
 }
 
 var _ SectorVerifier = Verifier{}
@@ -1048,9 +1053,9 @@ func (v Verifier) VerifySeal(info spproof.SealVerifyInfo) (bool, error) {
 	info.Randomness = Ticket
 	ok, err := ffi.VerifySeal(info)
 	if ok && err == nil {
-		v.Lock.Lock()
-		defer v.Lock.Unlock()
-		v.SM[info.SectorID] = storage.SectorCids{
+		v.lock.Lock()
+		defer v.lock.Unlock()
+		v.sm[info.SectorID] = storage.SectorCids{
 			Sealed: info.SealedCID,
 			Unsealed: info.UnsealedCID,
 		}
@@ -1064,14 +1069,14 @@ func (v Verifier) VerifyAggregateSeals(aggregate spproof.AggregateSealVerifyProo
 	}
 	ok, err := ffi.VerifyAggregateSeals(aggregate)
 	if ok && err == nil {
-		v.Lock.Lock()
-		defer v.Lock.Unlock()
+		v.lock.Lock()
+		defer v.lock.Unlock()
 		for _, info := range aggregate.Infos {
 			sid := abi.SectorID{
 				Miner: aggregate.Miner,
 				Number: info.Number,
 			}
-			v.SM[sid] = storage.SectorCids{
+			v.sm[sid] = storage.SectorCids{
 				Sealed: info.SealedCID,
 				Unsealed: info.UnsealedCID,
 			}
@@ -1088,12 +1093,12 @@ func (v Verifier) VerifyWindowPoSt(
 ) (bool, error) {
 	chanllendedSectors := make([]spproof.SectorInfo, len(sectors))
 	// minerID = sectors[0].ID.Miner
-	v.Lock.RLock()
+	v.lock.RLock()
 	// defer m.Lock.RUnLock()
 	for idx, sid := range(sectors){
-		cids, ok := v.SM[sid.ID]
+		cids, ok := v.sm[sid.ID]
 		if !ok {
-			v.Lock.RUnlock()
+			v.lock.RUnlock()
 			return false, xerrors.Errorf("can not map the sectorID into sector commitment")
 		}
 		chanllendedSectors[idx] = spproof.SectorInfo{
@@ -1102,7 +1107,7 @@ func (v Verifier) VerifyWindowPoSt(
 			SealedCID: cids.Sealed,
 		}
 	}
-	v.Lock.RUnlock()
+	v.lock.RUnlock()
 
 	randomness[31] &= 0x3f
 	return ffi.VerifyWindowPoSt(spproof.WindowPoStVerifyInfo{
@@ -1121,14 +1126,14 @@ func (v Verifier)VerifyAggregateWindowPostProofs(
 ) (bool, error) {
 	var sectorInfos []spproof.SectorInfo
 	sectorCount := make([]uint, len(sectors))
-	v.Lock.RLock()
+	v.lock.RLock()
 	// defer v.Lock.RUnLock()
 	for i, sectorRange := range(sectors) {
 		sectorCount[i] = uint(len(sectorRange))
 		for _, sid := range(sectorRange) {
-			cids, ok := v.SM[sid.ID]
+			cids, ok := v.sm[sid.ID]
 			if !ok {
-				v.Lock.RUnlock()
+				v.lock.RUnlock()
 				return false, xerrors.Errorf("can not map the sectorID into sector commitment")
 			}
 			sectorInfos = append(sectorInfos, spproof.SectorInfo{
@@ -1138,7 +1143,7 @@ func (v Verifier)VerifyAggregateWindowPostProofs(
 			})
 		}
 	}
-	v.Lock.RUnlock()
+	v.lock.RUnlock()
 
 	for i, random := range(randomnesses) {
 		randomnesses[i][31] = random[31] & 0x3f
@@ -1163,17 +1168,13 @@ func DefaultAggregationType() abi.RegisteredAggregationProof {
 	return abi.RegisteredAggregationProof_SnarkPackV1;
 }
 
-func memset(dst, src []byte) int {
-	if dst == nil {
-		return 0
-	}
-	if src == nil {
-		for n := 0; n < len(dst); n++ {
-			dst[n] = 0
-		}
-		return len(dst)
+func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
+	spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
+	if err != nil {
+		panic(err)
 	}
-	return copy(dst, src)
+
+	return spt
 }
 
 func min(x, y uint32) uint32 {
@@ -1181,26 +1182,4 @@ func min(x, y uint32) uint32 {
 		return x
 	}
 	return y
-}
-
-func check(in, out []byte) (bool, error){
-	if len(in) != len(out) {
-		return false, xerrors.Errorf("the %d output data and %d input data do not match", len(out), len(in))
-	}
-
-	for index := 0; index < len(in); index++ {
-		if in[index] != out[index] {
-			return false, xerrors.Errorf("the output data and input data do not match at: %d input is %u, output is %u",index,in[index],out[index])
-		}
-	}
-
-	return true, nil
-}
-
-// func nextUppandedPowerOfTwo(index uint32) abi.UnpaddedPieceSize {
-// 	power := 0
-// 	for index = index / 254; index != 0 ; power += 1 {
-// 		index >>= 1
-// 	}
-// 	return abi.UnpaddedPieceSize(254 * (1 << power))
-// }
\ No newline at end of file
+}
\ No newline at end of file

seal/type.go

@@ -25,7 +25,7 @@ type PieceEncoder interface {
 
 //interface
 type SectorSealer interface{
-	AddPiece(context.Context, storage.SectorRef, *[]storage.Piece) ([]abi.PieceInfo, error)
+	AddPiece(ctx context.Context, sid storage.SectorRef, sortedPieces []storage.Piece) ([]abi.PieceInfo, []storage.Piece, error)
 	
 	// run pre-commit1 and pre-commit2 phase
 	// generate the sealed sector and sector commitment(commd, commr)
@@ -35,7 +35,7 @@ type SectorSealer interface{
 	GenerateCommitProof( ctx context.Context, sid storage.SectorRef, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (spproof.Proof, error)
 	AggregateSealProofs(aggregateInfo spproof.AggregateSealVerifyProofAndInfos, proofs []spproof.Proof) (spproof.Proof, error)
 	
-	UnsealedRange(ctx context.Context, sid storage.SectorRef, sectorSize abi.SectorSize, commd cid.Cid, out io.Writer, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error
+	UnsealedRange(ctx context.Context, sid storage.SectorRef, out io.Writer, commd cid.Cid, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error
 
 	GenerateWindowPoStProofs(ctx context.Context, minerID abi.ActorID, sectorInfo []spproof.SectorInfo, randomness abi.PoStRandomness) ([]spproof.PoStProof, []abi.SectorID, error)
 	AggregateWindowPoStProofs(aggregateInfo spproof.AggregateWindowPostInfos, proofs []spproof.PoStProof) (spproof.Proof, error)