Skip to content

I
interface-test
Commit bd2eb1aa authored by 董子豪's avatar 董子豪
Browse files
Options

test aggregation

parent 77e3ef77
Show whitespace changes
Inline Side-by-side
Showing with 372 additions and 175 deletions
seal/seal_api.go
View file @ bd2eb1aa
......@@ -3,18 +3,13 @@ package seal
import(
"context"
"io"
"io/ioutil"
"os"
// "bytes"
"math/rand"
saproof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
"golang.org/x/xerrors"
proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
logging "github.com/ipfs/go-log/v2"
"github.com/mitchellh/go-homedir"
// commcid "github.com/filecoin-project/go-fil-commcid"
ffi "github.com/filecoin-project/filecoin-ffi"
"fil_integrate/extern/sector-storage/ffiwrapper"
......@@ -25,8 +20,6 @@ import(
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/network"
"github.com/ipfs/go-cid"
"github.com/minio/blake2b-simd"
"fil_integrate/build"
)
var log = logging.Logger("sealing")
......@@ -35,9 +28,31 @@ var log = logging.Logger("sealing")
//[has_pre][MetaLen1..MetaLen4][PieceLen1..PieceLen4]
const TagLen int = 8
const arp = abi.RegisteredAggregationProof_SnarkPackV1
const NewestNetworkVersion = network.Version13
func DefaultAggregationType() abi.RegisteredAggregationProof {
return arp;
}
func GetCommRFromDir(
sb *ffiwrapper.Sealer,
ctx context.Context,
sectorID abi.SectorID,
) (cid.Cid, error) {
return sb.GetCommRFromDir(ctx, sectorID)
}
func PutCommRIntoDir(
sb *ffiwrapper.Sealer,
ctx context.Context,
sectorID abi.SectorID,
sealedCID cid.Cid,
) error {
return sb.PutCommRIntoDir(ctx, sectorID, sealedCID)
}
func AddPiece(
sb *ffiwrapper.Sealer,
ctx context.Context,
......@@ -195,8 +210,8 @@ func AddPiece(
func AggregateWindowPoStProofs(
sb *ffiwrapper.Sealer,
ctx context.Context,
randomnesses []abi.SealRandomness,
aggregate abi.RegisteredAggregationProof,
randomnesses []abi.PoStRandomness,
windowPoStProofs []proof5.PoStProof,
sectorCountArr []uint,
) ([]byte, error) {
......@@ -224,7 +239,7 @@ func VerifyAggregateWindowPostProofs(
aggregateType abi.RegisteredAggregationProof,
miner abi.ActorID,
aggregateProof []byte,
randomnesses []abi.SealRandomness,
randomnesses []abi.PoStRandomness,
sealedSectors [][]proof5.SectorInfo,
) (bool, error) {
var sectorInfos []proof5.SectorInfo
......@@ -308,7 +323,7 @@ func GenProofForC2(
return sb.SealCommit2(ctx, sid, c1Out)
}
func GenProofForWindowPost(
func GenProofForWindowPoSt(
sb *ffiwrapper.Sealer,
ctx context.Context,
minerID abi.ActorID,
......@@ -360,167 +375,3 @@ func UnsealedRange(
return nil
}
\ No newline at end of file
func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
if err != nil {
panic(err)
}
return spt
}
func TestSealAndUnseal() error {
//********************need (sb,ctx,sid,sectorSize,file,seed,ticket,challenge)****************//
sdir, err := homedir.Expand("~/tmp")
if err != nil {
return err
}
err = os.MkdirAll(sdir, 0775) //nolint:gosec
if err != nil {
return xerrors.Errorf("creating sectorbuilder dir: %w", err)
}
tsdir, err := ioutil.TempDir(sdir, "bench")
if err != nil {
return err
}
// defer func() {
// if err := os.RemoveAll(tsdir); err != nil {
// log.Warn("remove all: ", err)
// }
// }()
// TODO: pretty sure this isnt even needed?
if err := os.MkdirAll(tsdir, 0775); err != nil {
return err
}
sbfs := &basicfs.Provider{
Root: tsdir,
}
sb ,err := ffiwrapper.New(sbfs)
if err != nil{
return err
}
ctx := context.TODO()
sectorSize := abi.SectorSize(8*1024*1024)
sid := storage.SectorRef{
ID: abi.SectorID{
Miner: 1000,
Number: 0,
},
ProofType: spt(sectorSize),
}
file := rand.New(rand.NewSource(1587))
seed := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}
trand := blake2b.Sum256([]byte("ticket-preimage"))
ticket := abi.SealRandomness(trand[:])
var challenge [32]byte
rand.Read(challenge[:])
//ADD PIECES
var existingPieceSizes []abi.UnpaddedPieceSize
var pieces []abi.PieceInfo
var sealedSectors []saproof2.SectorInfo
piece, err := AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
if err != nil {
return err
}
existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
pieces = append(pieces, piece)
piece, err = AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
if err != nil {
return err
}
existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
pieces = append(pieces, piece)
//SEAL
cids, proof1, c1o, err := Sealed(sb, ctx, sid, seed, ticket, pieces)
if err != nil {
return err
}
sealedSectors = append(sealedSectors, saproof2.SectorInfo{
SealedCID: cids.Sealed,
SectorNumber: sid.ID.Number,
SealProof: sid.ProofType,
})
proof2, err := GenProofForC2(sb , ctx, sid, c1o)
if err != nil {
return err
}
ok, err := CheckPieceAndDataRoot(sid, cids.Unsealed, pieces)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("commd and pieces info don't match")
}
//verify proof
svi1 := saproof2.SealVerifyInfo{
SectorID: sid.ID,
SealedCID: cids.Sealed,
SealProof: sid.ProofType,
Proof: proof1,
DealIDs: nil,
Randomness: ticket,
InteractiveRandomness: seed,
UnsealedCID: cids.Unsealed,
}
svi2 := saproof2.SealVerifyInfo{
SectorID: sid.ID,
SealedCID: cids.Sealed,
SealProof: sid.ProofType,
Proof: proof2,
DealIDs: nil,
Randomness: ticket,
InteractiveRandomness: seed,
UnsealedCID: cids.Unsealed,
}
ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi1)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
}
ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi2)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
}
proof, _, err := GenProofForWindowPost(sb, ctx, sid.ID.Miner, sealedSectors, challenge[:])
wpvi := saproof2.WindowPoStVerifyInfo{
Randomness: challenge[:],
Proofs: proof,
ChallengedSectors: sealedSectors,
Prover: sid.ID.Miner,
}
ok, err = ffiwrapper.ProofVerifier.VerifyWindowPoSt(ctx, wpvi)
if err != nil {
return err
}
if !ok {
log.Error("window post verification failed")
}
return nil
}
\ No newline at end of file
seal/test_seal.go 0 → 100644
View file @ bd2eb1aa
package seal
import(
"context"
"fmt"
"io/ioutil"
"os"
"math/rand"
"time"
"golang.org/x/xerrors"
"github.com/mitchellh/go-homedir"
"github.com/minio/blake2b-simd"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/specs-storage/storage"
saproof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
"fil_integrate/build"
"fil_integrate/extern/sector-storage/ffiwrapper"
"fil_integrate/extern/sector-storage/ffiwrapper/basicfs"
)
func TestAggregateWindowPoSt(
sectorSize abi.SectorSize,
numSectors int,
numAggregate int,
) error {
sdir, err := homedir.Expand("~/tmp")
if err != nil {
return err
}
err = os.MkdirAll(sdir, 0775) //nolint:gosec
if err != nil {
return xerrors.Errorf("creating sectorbuilder dir: %w", err)
}
tsdir, err := ioutil.TempDir(sdir, "bench")
if err != nil {
return err
}
// defer func() {
// if err := os.RemoveAll(tsdir); err != nil {
// log.Warn("remove all: ", err)
// }
// }()
// TODO: pretty sure this isnt even needed?
if err := os.MkdirAll(tsdir, 0775); err != nil {
return err
}
sbfs := &basicfs.Provider{
Root: tsdir,
}
sb ,err := ffiwrapper.New(sbfs)
if err != nil{
return err
}
ctx := context.TODO()
file := rand.New(rand.NewSource(1587))
trand := blake2b.Sum256([]byte("ticket-preimage"))
ticket := abi.SealRandomness(trand[:])
var challenge [32]byte
rand.Read(challenge[:])
var randomnesses []abi.PoStRandomness
var sealedSectorsinfo [][]saproof2.SectorInfo
var sectorCount []uint
var proofs []proof5.PoStProof
sealProofType := spt(sectorSize)
start := time.Now()
for i := 0; i < numAggregate; i++{
for j := 0; j < numSectors; j++{
var pieces []abi.PieceInfo
sid := storage.SectorRef{
ID: abi.SectorID{
Miner: 1000,
Number: abi.SectorNumber(i*numSectors+j),
},
ProofType: sealProofType,
}
piece, err := AddPiece(sb, ctx, sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), file)
if err != nil {
return err
}
pieces = append(pieces, piece)
// log.Infof("[%d] Running replication(1)...", sid.ID.Number)
pc1out, err := sb.SealPreCommit1(ctx, sid, ticket, pieces)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
// log.Infof("[%d] Running replication(2)...", sid.ID.Number)
cids, err := sb.SealPreCommit2(ctx, sid, pc1out)
if err != nil {
return xerrors.Errorf("commit: %w", err)
}
comm_r := cids.Sealed
PutCommRIntoDir(sb, ctx, sid.ID, comm_r)
}
}
log.Infof("Sealed [%d] Sectors Done", numSectors*numAggregate)
sealed := time.Now()
for i := 0; i < numAggregate; i++{
var sealedSectors []saproof2.SectorInfo
for j := 0; j < numSectors; j++{
sectorID := abi.SectorID{
Miner: 1000,
Number: abi.SectorNumber(i*numSectors+j),
}
comm_r, err := GetCommRFromDir(sb, ctx, sectorID)
if err != nil {
return err
}
sealedSectors = append(sealedSectors, saproof2.SectorInfo{
SealedCID: comm_r,
SectorNumber: sectorID.Number,
SealProof: sealProofType,
})
}
sealedSectorsinfo = append(sealedSectorsinfo, sealedSectors)
}
log.Infof("Read [%d] Commitment Rplication Done", numSectors*numAggregate)
loadCommr := time.Now()
for i := 0; i < numAggregate; i++{
log.Infof("[%d] Generating Window-Post", i)
proof, _, err := GenProofForWindowPoSt(sb, ctx, 1000, sealedSectorsinfo[i], challenge[:])
if err != nil {
return err
}
proofs = append(proofs, proof...)
randomnesses = append(randomnesses, challenge[:])
sectorCount = append(sectorCount, uint(numSectors))
}
log.Infof("Generate [%d] Window-PoSt Done", numAggregate)
genWindowPoSt := time.Now()
aggregateProof, err := AggregateWindowPoStProofs(sb, ctx, DefaultAggregationType(), randomnesses, proofs, sectorCount)
if err != nil {
return err
}
aggregateProofs := time.Now()
PoStType, _ := sealProofType.RegisteredWindowPoStProof()
ok, err := VerifyAggregateWindowPostProofs(
ctx,
PoStType,
DefaultAggregationType(),
abi.ActorID(1000),
aggregateProof,
randomnesses,
sealedSectorsinfo,
)
if err != nil {
return err
}
if ok {
fmt.Println("Aggregate proof is true")
} else{
fmt.Println("Aggregate proof is false")
}
verifyProofs := time.Now()
fmt.Printf("Seal %d sectors using %s\n", numSectors*numAggregate, sealed.Sub(start))
fmt.Printf("Read %d comm_r using %s\n", numAggregate*numSectors, loadCommr.Sub(sealed))
fmt.Printf("Generate %d window-post using %s\n", numAggregate, genWindowPoSt.Sub(loadCommr))
fmt.Printf("Aggregate %d window-post Proofs using %s\n", numAggregate, aggregateProofs.Sub(genWindowPoSt))
fmt.Printf("Verify Proofs using %s\n", verifyProofs.Sub(aggregateProofs))
return nil
}
func TestSealAndUnseal() error {
//********************need (sb,ctx,sid,sectorSize,file,seed,ticket,challenge)****************//
sdir, err := homedir.Expand("~/tmp")
if err != nil {
return err
}
err = os.MkdirAll(sdir, 0775) //nolint:gosec
if err != nil {
return xerrors.Errorf("creating sectorbuilder dir: %w", err)
}
tsdir, err := ioutil.TempDir(sdir, "bench")
if err != nil {
return err
}
// defer func() {
// if err := os.RemoveAll(tsdir); err != nil {
// log.Warn("remove all: ", err)
// }
// }()
// TODO: pretty sure this isnt even needed?
if err := os.MkdirAll(tsdir, 0775); err != nil {
return err
}
sbfs := &basicfs.Provider{
Root: tsdir,
}
sb ,err := ffiwrapper.New(sbfs)
if err != nil{
return err
}
ctx := context.TODO()
sectorSize := abi.SectorSize(8*1024*1024)
sid := storage.SectorRef{
ID: abi.SectorID{
Miner: 1000,
Number: 0,
},
ProofType: spt(sectorSize),
}
file := rand.New(rand.NewSource(1587))
seed := []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}
trand := blake2b.Sum256([]byte("ticket-preimage"))
ticket := abi.SealRandomness(trand[:])
var challenge [32]byte
rand.Read(challenge[:])
//ADD PIECES
var existingPieceSizes []abi.UnpaddedPieceSize
var pieces []abi.PieceInfo
var sealedSectors []saproof2.SectorInfo
piece, err := AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
if err != nil {
return err
}
existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
pieces = append(pieces, piece)
piece, err = AddPiece(sb, ctx, sid, existingPieceSizes, abi.PaddedPieceSize(sectorSize/2).Unpadded(), file)
if err != nil {
return err
}
existingPieceSizes = append(existingPieceSizes, piece.Size.Unpadded())
pieces = append(pieces, piece)
//SEAL
cids, proof1, c1o, err := Sealed(sb, ctx, sid, seed, ticket, pieces)
if err != nil {
return err
}
sealedSectors = append(sealedSectors, saproof2.SectorInfo{
SealedCID: cids.Sealed,
SectorNumber: sid.ID.Number,
SealProof: sid.ProofType,
})
proof2, err := GenProofForC2(sb , ctx, sid, c1o)
if err != nil {
return err
}
ok, err := CheckPieceAndDataRoot(sid, cids.Unsealed, pieces)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("commd and pieces info don't match")
}
//verify proof
svi1 := saproof2.SealVerifyInfo{
SectorID: sid.ID,
SealedCID: cids.Sealed,
SealProof: sid.ProofType,
Proof: proof1,
DealIDs: nil,
Randomness: ticket,
InteractiveRandomness: seed,
UnsealedCID: cids.Unsealed,
}
svi2 := saproof2.SealVerifyInfo{
SectorID: sid.ID,
SealedCID: cids.Sealed,
SealProof: sid.ProofType,
Proof: proof2,
DealIDs: nil,
Randomness: ticket,
InteractiveRandomness: seed,
UnsealedCID: cids.Unsealed,
}
ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi1)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
}
ok, err = ffiwrapper.ProofVerifier.VerifySeal(svi2)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("porep proof for sector %d was invalid", sid.ID.Number)
}
proof, _, err := GenProofForWindowPoSt(sb, ctx, sid.ID.Miner, sealedSectors, challenge[:])
wpvi := saproof2.WindowPoStVerifyInfo{
Randomness: challenge[:],
Proofs: proof,
ChallengedSectors: sealedSectors,
Prover: sid.ID.Miner,
}
ok, err = ffiwrapper.ProofVerifier.VerifyWindowPoSt(ctx, wpvi)
if err != nil {
return err
}
if !ok {
log.Error("window post verification failed")
}
return nil
}
func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
spt, err := build.SealProofTypeFromSectorSize(ssize, NewestNetworkVersion)
if err != nil {
panic(err)
}
return spt
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment