test aggregation

seal/seal_api.go

@@ -3,18 +3,13 @@ package seal
 import(
 	"context"
 	"io"
-	"io/ioutil"
 	"os"
 	// "bytes"
-	"math/rand"
-
-	saproof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
 
 	"golang.org/x/xerrors"
 
 	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/mitchellh/go-homedir"
 	// commcid "github.com/filecoin-project/go-fil-commcid"
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	"fil_integrate/extern/sector-storage/ffiwrapper"
@@ -25,8 +20,6 @@ import(
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
 	"github.com/ipfs/go-cid"
-	"github.com/minio/blake2b-simd"
-	"fil_integrate/build"
 )
 
 var log = logging.Logger("sealing")
@@ -35,9 +28,31 @@ var log = logging.Logger("sealing")
 //[has_pre][MetaLen1..MetaLen4][PieceLen1..PieceLen4]
 const TagLen int = 8
 
+const arp = abi.RegisteredAggregationProof_SnarkPackV1
 
 const NewestNetworkVersion = network.Version13
 
+func DefaultAggregationType() abi.RegisteredAggregationProof {
+	return arp;
+}
+
+func GetCommRFromDir(
+	sb *ffiwrapper.Sealer,
+	ctx context.Context,
+	sectorID abi.SectorID,
+) (cid.Cid, error) {
+	return sb.GetCommRFromDir(ctx, sectorID)
+}
+
+func PutCommRIntoDir(
+	sb *ffiwrapper.Sealer,
+	ctx context.Context,
+	sectorID abi.SectorID,
+	sealedCID cid.Cid,
+) error {
+	return sb.PutCommRIntoDir(ctx, sectorID, sealedCID)
+}
+
 func AddPiece(
 	sb *ffiwrapper.Sealer, 
 	ctx context.Context, 
@@ -195,8 +210,8 @@ func AddPiece(
 func AggregateWindowPoStProofs(
 	sb *ffiwrapper.Sealer, 
 	ctx context.Context,
-	randomnesses []abi.SealRandomness, 
 	aggregate abi.RegisteredAggregationProof, 
+	randomnesses []abi.PoStRandomness,  
 	windowPoStProofs []proof5.PoStProof,
 	sectorCountArr []uint,
 ) ([]byte, error) {
@@ -224,7 +239,7 @@ func VerifyAggregateWindowPostProofs(
 	aggregateType abi.RegisteredAggregationProof,
 	miner abi.ActorID,
 	aggregateProof []byte,
-	randomnesses []abi.SealRandomness,
+	randomnesses []abi.PoStRandomness,
 	sealedSectors [][]proof5.SectorInfo,
 ) (bool, error) {
 	var sectorInfos []proof5.SectorInfo
@@ -308,7 +323,7 @@ func GenProofForC2(
 	return sb.SealCommit2(ctx, sid, c1Out)
 }
 
-func GenProofForWindowPost(
+func GenProofForWindowPoSt(
 	sb *ffiwrapper.Sealer, 
 	ctx context.Context, 
 	minerID abi.ActorID, 
@@ -360,167 +375,3 @@ func UnsealedRange(
 	return nil
 
 }
\ No newline at end of file
-
-func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
-	spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
-	if err != nil {
-		panic(err)
-	}
-
-	return spt
-}
-
-func TestSealAndUnseal() error {
-	//********************need (sb,ctx,sid,sectorSize,file,seed,ticket,challenge)****************//
-	sdir, err := homedir.Expand("~/tmp")
-	if err != nil {
-		return err
-	}
-
-	err = os.MkdirAll(sdir, 0775) //nolint:gosec
-	if err != nil {
-		return xerrors.Errorf("creating sectorbuilder dir: %w", err)
-	}
-
-	tsdir, err := ioutil.TempDir(sdir, "bench")
-	if err != nil {
-		return err
-	}
-	// defer func() {
-	// 	if err := os.RemoveAll(tsdir); err != nil {
-	// 		log.Warn("remove all: ", err)
-	// 	}
-	// }()
-
-	// TODO: pretty sure this isnt even needed?
-	if err := os.MkdirAll(tsdir, 0775); err != nil {
-		return err
-	}
-	sbfs := &basicfs.Provider{
-			Root: tsdir,
-		}
-	sb ,err := ffiwrapper.New(sbfs)
-	if err != nil{
-		return err
-	}
-	ctx := context.TODO()
-	sectorSize := abi.SectorSize(8*1024*1024)
-	sid := storage.SectorRef{
-		ID: abi.SectorID{
-			Miner:  1000,
-			Number: 0,
-		},
-		ProofType: spt(sectorSize),
-	}
-	file := rand.New(rand.NewSource(1587))
-	seed := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}
-	trand := blake2b.Sum256([]byte("ticket-preimage"))
-	ticket := abi.SealRandomness(trand[:])
-	var challenge [32]byte
-	rand.Read(challenge[:])
-	//ADD PIECES
-	var existingPieceSizes []abi.UnpaddedPieceSize
-	var pieces []abi.PieceInfo
-	var sealedSectors []saproof2.SectorInfo
-
-	piece, err := AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
-	if err != nil {
-		return err
-	}
-
-	existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
-	pieces = append(pieces, piece)
-
-	piece, err = AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
-	if err != nil {
-		return err
-	}
-
-	existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
-	pieces = append(pieces, piece)
-
-	//SEAL
-	cids, proof1, c1o, err := Sealed(sb, ctx, sid, seed, ticket, pieces)
-	if err != nil {
-		return err
-	}
-
-	sealedSectors = append(sealedSectors, saproof2.SectorInfo{
-		SealedCID:    cids.Sealed,
-		SectorNumber: sid.ID.Number,
-		SealProof:    sid.ProofType,
-	})
-
-	proof2, err := GenProofForC2(sb , ctx, sid, c1o)
-	if err != nil {
-		return err
-	}
-
-	ok, err := CheckPieceAndDataRoot(sid, cids.Unsealed, pieces)
-	if err != nil {
-		return err
-	}
-
-	if !ok {
-		return xerrors.Errorf("commd and pieces info don't match")
-	}
-
-	//verify proof
-	svi1 := saproof2.SealVerifyInfo{
-		SectorID:              sid.ID,
-		SealedCID:             cids.Sealed,
-		SealProof:             sid.ProofType,
-		Proof:                 proof1,
-		DealIDs:               nil,
-		Randomness:            ticket,
-		InteractiveRandomness: seed,
-		UnsealedCID:           cids.Unsealed,
-	}
-
-	svi2 := saproof2.SealVerifyInfo{
-		SectorID:              sid.ID,
-		SealedCID:             cids.Sealed,
-		SealProof:             sid.ProofType,
-		Proof:                 proof2,
-		DealIDs:               nil,
-		Randomness:            ticket,
-		InteractiveRandomness: seed,
-		UnsealedCID:           cids.Unsealed,
-	}
-
-	ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi1)
-	if err != nil {
-		return err
-	}
-	if !ok {
-		return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
-	}
-
-	ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi2)
-	if err != nil {
-		return err
-	}
-	if !ok {
-		return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
-	}
-
-	proof, _, err := GenProofForWindowPost(sb, ctx, sid.ID.Miner, sealedSectors, challenge[:])
-
-	wpvi := saproof2.WindowPoStVerifyInfo{
-		Randomness:        challenge[:],
-		Proofs:            proof,
-		ChallengedSectors: sealedSectors,
-		Prover:            sid.ID.Miner,
-	}
-
-	ok, err = ffiwrapper.ProofVerifier.VerifyWindowPoSt(ctx, wpvi)
-	if err != nil {
-		return err
-	}
-	if !ok {
-		log.Error("window post verification failed")
-	}
-
-	return nil
-
-}
\ No newline at end of file

seal/test_seal.go

+package seal
+
+import(
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"math/rand"
+	"time"
+
+	"golang.org/x/xerrors"
+	"github.com/mitchellh/go-homedir"
+	"github.com/minio/blake2b-simd"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/specs-storage/storage"
+	saproof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
+	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+
+	"fil_integrate/build"
+	"fil_integrate/extern/sector-storage/ffiwrapper"
+	"fil_integrate/extern/sector-storage/ffiwrapper/basicfs"
+)
+
+func TestAggregateWindowPoSt(
+	sectorSize abi.SectorSize,
+	numSectors int,
+	numAggregate int,
+) error {
+	sdir, err := homedir.Expand("~/tmp")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(sdir, 0775) //nolint:gosec
+	if err != nil {
+		return xerrors.Errorf("creating sectorbuilder dir: %w", err)
+	}
+
+	tsdir, err := ioutil.TempDir(sdir, "bench")
+	if err != nil {
+		return err
+	}
+	// defer func() {
+	// 	if err := os.RemoveAll(tsdir); err != nil {
+	// 		log.Warn("remove all: ", err)
+	// 	}
+	// }()
+
+	// TODO: pretty sure this isnt even needed?
+	if err := os.MkdirAll(tsdir, 0775); err != nil {
+		return err
+	}
+	sbfs := &basicfs.Provider{
+			Root: tsdir,
+		}
+	sb ,err := ffiwrapper.New(sbfs)
+	if err != nil{
+		return err
+	}
+	ctx := context.TODO()
+
+	file := rand.New(rand.NewSource(1587))
+	trand := blake2b.Sum256([]byte("ticket-preimage"))
+	ticket := abi.SealRandomness(trand[:])
+	var challenge [32]byte
+	rand.Read(challenge[:])
+
+	var randomnesses []abi.PoStRandomness
+	var sealedSectorsinfo [][]saproof2.SectorInfo
+	var sectorCount []uint
+	var proofs []proof5.PoStProof
+	sealProofType := spt(sectorSize)
+
+	start := time.Now()
+	for i := 0; i < numAggregate; i++{
+		for j := 0; j < numSectors; j++{
+			var pieces []abi.PieceInfo
+
+			sid := storage.SectorRef{
+				ID: abi.SectorID{
+					Miner:  1000,
+					Number: abi.SectorNumber(i*numSectors+j),
+				},
+				ProofType: sealProofType,
+			}
+
+			piece, err := AddPiece(sb, ctx, sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), file)
+			if err != nil {
+				return err
+			}
+			pieces = append(pieces, piece)
+
+			// log.Infof("[%d] Running replication(1)...", sid.ID.Number)
+
+			pc1out, err := sb.SealPreCommit1(ctx, sid, ticket, pieces)
+			if err != nil {
+				return xerrors.Errorf("commit: %w", err)
+			}
+
+			// log.Infof("[%d] Running replication(2)...", sid.ID.Number)
+			cids, err := sb.SealPreCommit2(ctx, sid, pc1out)
+			if err != nil {
+				return xerrors.Errorf("commit: %w", err)
+			}
+
+			comm_r := cids.Sealed
+			PutCommRIntoDir(sb, ctx, sid.ID, comm_r)
+		}
+	}
+	log.Infof("Sealed [%d] Sectors Done", numSectors*numAggregate)
+	sealed := time.Now()
+
+	for i := 0; i < numAggregate; i++{
+		var sealedSectors []saproof2.SectorInfo
+		for j := 0; j < numSectors; j++{
+			sectorID := abi.SectorID{
+				Miner:  1000,
+				Number: abi.SectorNumber(i*numSectors+j),
+			}
+			comm_r, err := GetCommRFromDir(sb, ctx, sectorID)
+			if err != nil {
+				return err
+			}
+			sealedSectors = append(sealedSectors, saproof2.SectorInfo{
+				SealedCID:    comm_r,
+				SectorNumber: sectorID.Number,
+				SealProof:    sealProofType,
+			})
+		}
+		sealedSectorsinfo = append(sealedSectorsinfo, sealedSectors)
+	}
+	log.Infof("Read [%d] Commitment Rplication Done", numSectors*numAggregate)
+	loadCommr := time.Now()
+
+	for i := 0; i < numAggregate; i++{
+		log.Infof("[%d] Generating Window-Post", i)
+		proof, _, err := GenProofForWindowPoSt(sb, ctx, 1000, sealedSectorsinfo[i], challenge[:])
+		if err != nil {
+			return err
+		}
+
+		proofs = append(proofs, proof...)
+		randomnesses = append(randomnesses, challenge[:])
+		sectorCount = append(sectorCount, uint(numSectors))
+	}
+	log.Infof("Generate [%d] Window-PoSt Done", numAggregate)
+	genWindowPoSt := time.Now()
+
+	aggregateProof, err := AggregateWindowPoStProofs(sb, ctx, DefaultAggregationType(), randomnesses, proofs, sectorCount)
+	if err != nil {
+		return err
+	}
+	aggregateProofs := time.Now()
+
+	PoStType, _ := sealProofType.RegisteredWindowPoStProof()
+	ok, err := VerifyAggregateWindowPostProofs(
+		ctx, 
+		PoStType, 
+		DefaultAggregationType(), 
+		abi.ActorID(1000),
+		aggregateProof,
+		randomnesses,
+		sealedSectorsinfo,
+	)
+	if err != nil {
+		return err
+	}
+	if ok {
+		fmt.Println("Aggregate proof is true")
+	} else{
+		fmt.Println("Aggregate proof is false")
+	}
+	verifyProofs := time.Now()
+
+	fmt.Printf("Seal %d sectors using %s\n", numSectors*numAggregate, sealed.Sub(start))
+	fmt.Printf("Read %d comm_r using %s\n", numAggregate*numSectors, loadCommr.Sub(sealed))
+	fmt.Printf("Generate %d window-post using %s\n", numAggregate, genWindowPoSt.Sub(loadCommr))
+	fmt.Printf("Aggregate %d window-post Proofs using %s\n", numAggregate, aggregateProofs.Sub(genWindowPoSt))
+	fmt.Printf("Verify Proofs using %s\n", verifyProofs.Sub(aggregateProofs))
+
+	return nil
+}
+
+func TestSealAndUnseal() error {
+	//********************need (sb,ctx,sid,sectorSize,file,seed,ticket,challenge)****************//
+	sdir, err := homedir.Expand("~/tmp")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(sdir, 0775) //nolint:gosec
+	if err != nil {
+		return xerrors.Errorf("creating sectorbuilder dir: %w", err)
+	}
+
+	tsdir, err := ioutil.TempDir(sdir, "bench")
+	if err != nil {
+		return err
+	}
+	// defer func() {
+	// 	if err := os.RemoveAll(tsdir); err != nil {
+	// 		log.Warn("remove all: ", err)
+	// 	}
+	// }()
+
+	// TODO: pretty sure this isnt even needed?
+	if err := os.MkdirAll(tsdir, 0775); err != nil {
+		return err
+	}
+	sbfs := &basicfs.Provider{
+			Root: tsdir,
+		}
+	sb ,err := ffiwrapper.New(sbfs)
+	if err != nil{
+		return err
+	}
+	ctx := context.TODO()
+	sectorSize := abi.SectorSize(8*1024*1024)
+	sid := storage.SectorRef{
+		ID: abi.SectorID{
+			Miner:  1000,
+			Number: 0,
+		},
+		ProofType: spt(sectorSize),
+	}
+	file := rand.New(rand.NewSource(1587))
+	seed := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}
+	trand := blake2b.Sum256([]byte("ticket-preimage"))
+	ticket := abi.SealRandomness(trand[:])
+	var challenge [32]byte
+	rand.Read(challenge[:])
+	//ADD PIECES
+	var existingPieceSizes []abi.UnpaddedPieceSize
+	var pieces []abi.PieceInfo
+	var sealedSectors []saproof2.SectorInfo
+
+	piece, err := AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
+	if err != nil {
+		return err
+	}
+
+	existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
+	pieces = append(pieces, piece)
+
+	piece, err = AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
+	if err != nil {
+		return err
+	}
+
+	existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
+	pieces = append(pieces, piece)
+
+	//SEAL
+	cids, proof1, c1o, err := Sealed(sb, ctx, sid, seed, ticket, pieces)
+	if err != nil {
+		return err
+	}
+
+	sealedSectors = append(sealedSectors, saproof2.SectorInfo{
+		SealedCID:    cids.Sealed,
+		SectorNumber: sid.ID.Number,
+		SealProof:    sid.ProofType,
+	})
+
+	proof2, err := GenProofForC2(sb , ctx, sid, c1o)
+	if err != nil {
+		return err
+	}
+
+	ok, err := CheckPieceAndDataRoot(sid, cids.Unsealed, pieces)
+	if err != nil {
+		return err
+	}
+
+	if !ok {
+		return xerrors.Errorf("commd and pieces info don't match")
+	}
+
+	//verify proof
+	svi1 := saproof2.SealVerifyInfo{
+		SectorID:              sid.ID,
+		SealedCID:             cids.Sealed,
+		SealProof:             sid.ProofType,
+		Proof:                 proof1,
+		DealIDs:               nil,
+		Randomness:            ticket,
+		InteractiveRandomness: seed,
+		UnsealedCID:           cids.Unsealed,
+	}
+
+	svi2 := saproof2.SealVerifyInfo{
+		SectorID:              sid.ID,
+		SealedCID:             cids.Sealed,
+		SealProof:             sid.ProofType,
+		Proof:                 proof2,
+		DealIDs:               nil,
+		Randomness:            ticket,
+		InteractiveRandomness: seed,
+		UnsealedCID:           cids.Unsealed,
+	}
+
+	ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi1)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
+	}
+
+	ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi2)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
+	}
+
+	proof, _, err := GenProofForWindowPoSt(sb, ctx, sid.ID.Miner, sealedSectors, challenge[:])
+
+	wpvi := saproof2.WindowPoStVerifyInfo{
+		Randomness:        challenge[:],
+		Proofs:            proof,
+		ChallengedSectors: sealedSectors,
+		Prover:            sid.ID.Miner,
+	}
+
+	ok, err = ffiwrapper.ProofVerifier.VerifyWindowPoSt(ctx, wpvi)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		log.Error("window post verification failed")
+	}
+
+	return nil
+
+}
+
+func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
+	spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
+	if err != nil {
+		panic(err)
+	}
+
+	return spt
+}
\ No newline at end of file