add actor-user interface

actor/user/testuser.go

 package user
 
-import(
+import (
 	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/minio/md5-simd"
+	"github.com/mitchellh/go-homedir"
+	"golang.org/x/xerrors"
+
+	"fil_integrate/build/state-types/abi"
+	"fil_integrate/build/storage"
+	"fil_integrate/seal"
 )
 
+type Range struct {
+	offset uint64
+	size uint64
+}
+
+type pieceRead struct {
+	piece storage.Piece
+	idx int
+	pieceRange Range
+}
+
 func TestUser() error {
-	
-}
\ No newline at end of file
+	sectorSize := storage.SectorSize32MiB
+	sdir, err := homedir.Expand("~/tmp/bench")
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(sdir, 0775) //nolint:gosec
+	if err != nil {
+		return xerrors.Errorf("creating sectorbuilder dir: %w", err)
+	}
+
+	tsdir, err := ioutil.TempDir(sdir, "bench")
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := os.RemoveAll(tsdir); err != nil {
+			log.Warn("remove all: ", err)
+		}
+	}()
+
+	// TODO: pretty sure this isnt even needed?
+	if err := os.MkdirAll(tsdir, 0775); err != nil {
+		return err
+	}
+	sp := &seal.Encoder{
+		Root: tsdir,
+	}
+	ctx := context.TODO()
+	u := New(sp)
+
+	b := []byte("random data")
+	var numFile = 4
+	var PiecesRange []pieceRead
+	var range2Read []Range = []Range{Range{0,0}, Range{1024, sectorSize}, Range{1024, 3*sectorSize+1024}, Range{2*sectorSize+2048, sectorSize+4096}}
+	for i := 0; i < numFile; i++ {
+		filename := filepath.Join(tsdir, fmt.Sprintf("input-%d.dat", i))
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		dataSize := uint64(r.Int63n(int64(sectorSize / 8))) * 32 + 4 * sectorSize
+		b, err = seal.GenerateRandomData(filename, dataSize, b)
+		if err != nil {
+			return err
+		}
+
+		in, err := os.OpenFile(filename, os.O_RDONLY, 0644)
+		if err != nil {
+			return err
+		}
+		defer in.Close()
+
+		finalPiece, _, err := sp.EncodeDataToPieces(ctx, abi.SectorSize(sectorSize), in)
+		if err != nil {
+			return err
+		}
+		for _, r := range(range2Read) {
+			PiecesRange = append(PiecesRange, pieceRead{
+				piece: finalPiece,
+				idx: i,
+				pieceRange: r,
+			})
+		}
+		PiecesRange = append(PiecesRange, pieceRead{
+			piece: finalPiece,
+			idx: i,
+			pieceRange: Range{
+				offset: dataSize - 1024,
+				size: 1024,
+			},
+		})
+	}
+
+	for _, r := range(PiecesRange) {
+		input := filepath.Join(tsdir, fmt.Sprintf("input-%d.dat", r.idx))
+		output := filepath.Join(tsdir, "output")
+
+		if _, err = os.Stat(output); !os.IsNotExist(err) {
+			os.Remove(output)
+		}
+		f, err := os.OpenFile(output, os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return err
+		}
+		err = u.ReadPieceRange(ctx, f, r.piece, r.pieceRange.offset, r.pieceRange.size)
+		if err != nil {
+			return err
+		}
+		f.Close()
+
+		ok, err := checkDecodedFile(input, output, r.pieceRange.offset, r.pieceRange.size)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			fmt.Println("decode pieces failed")
+		} else {
+			fmt.Println("decode pieces success")
+		}
+	}
+	return nil
+}
+
+func checkDecodedFile(file1 string, file2 string, offset uint64, size uint64) (bool, error) {
+	in, err := os.Open(file1)
+	if err != nil {
+		return false, err
+	}
+	defer in.Close()
+	var f1 io.Reader = in
+	if offset != 0 || size != 0 {
+		if _, err := in.Seek(int64(offset), io.SeekStart); err != nil {
+			return false, xerrors.Errorf("seek to trailer start: %w", err)
+		}
+		f1 = io.LimitReader(in, int64(size))
+	}
+
+	f2, err := os.Open(file2)
+	if err != nil {
+		return false, err
+	}
+	defer f2.Close()
+
+	inBuf := make([]byte, 2<<20)
+	outBuf := make([]byte, 2<<20)
+
+	server1 := md5simd.NewServer()
+	defer server1.Close()
+	server2 := md5simd.NewServer()
+	defer server2.Close()
+
+	h1 := server1.NewHash()
+	defer h1.Close()
+	h2 := server2.NewHash()
+	defer h2.Close()
+
+	for {
+		_, inerr := f1.Read(inBuf[:])
+		if err != nil && err != io.EOF {
+			return false, err
+		}
+
+		_, outerr := f2.Read(outBuf[:])
+		if err != nil && err != io.EOF {
+			return false, err
+		}
+
+		h1.Write(inBuf)
+		h2.Write(outBuf)
+
+		if inerr == io.EOF && outerr == io.EOF {
+			hash1 := h1.Sum(nil)
+			hash2 := h2.Sum(nil)
+			if string(hash1) != string(hash2) {
+				return false, xerrors.Errorf("the output can't match input file")
+			}
+			break
+		}
+	}
+	return true, nil
+}

actor/user/type.go

 package user
 
-import(
-	"io"
+import (
 	"context"
+	"io"
 
-	"fil_integrate/build/state-types/abi"
 	"fil_integrate/build/storage"
 )
 
 type UserAPI interface {
 	EncodeDataToPieces(ctx context.Context, file storage.Data) (storage.Piece, []storage.Piece, error)
-	ReadPieceRange(ctx context.Context, out io.Writer, piece storage.Piece, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error
-}
\ No newline at end of file
+	ReadPieceRange(ctx context.Context, out io.Writer, piece storage.Piece, offset uint64, size uint64) error
+}

actor/user/user.go

 package user
 
-import(
+import (
 	"context"
+	"io"
 
+	logging "github.com/ipfs/go-log/v2"
+	"golang.org/x/xerrors"
+
+	"fil_integrate/build/state-types/abi"
 	"fil_integrate/build/storage"
 	"fil_integrate/seal"
 )
 
-type User struct{
+var log = logging.Logger("user")
+
+type User struct {
 	sectorSize abi.SectorSize
-	encoder seal.PieceEncoder
+	encoder    seal.PieceEncoder
 }
 
 var _ UserAPI = &User{}
 
+func New(encoder seal.PieceEncoder) *User {
+	u := &User{
+		sectorSize: abi.SectorSize(storage.SectorSize32MiB), 
+		encoder: encoder, 
+	}
+	return u
+}
+
 func (u *User) EncodeDataToPieces(ctx context.Context, file storage.Data) (storage.Piece, []storage.Piece, error) {
 	finalPiece, pieces, err := u.encoder.EncodeDataToPieces(ctx, u.sectorSize, file)
 	// map(file) -> finalPiece ...
@@ -22,16 +37,17 @@ func (u *User) EncodeDataToPieces(ctx context.Context, file storage.Data) (stora
 }
 
 func (u *User) ReadPieceRange(
-	ctx context.Context, 
-	out io.Writer, 
-	piece storage.Piece, 
-	offset uint64, 
+	ctx context.Context,
+	out io.Writer,
+	piece storage.Piece,
+	offset uint64,
 	size uint64,
 ) error {
+	log.Infof("Reading Piece [%d:%d]", offset, offset + size)
 	UnpaddedSectorSize := abi.PaddedPieceSize(u.sectorSize).Unpadded()
 	DataLen := uint32(UnpaddedSectorSize) - seal.TagLen
 
-	data, err := u.getPiece(ctx, piece.Hash)
+	data, err := u.getPiece(ctx, piece.Commitment)
 	if err != nil {
 		return err
 	}
@@ -45,26 +61,27 @@ func (u *User) ReadPieceRange(
 		piecesHash = append(data.PieceHash, piecesHash...)
 	}
 	buf := data.Data[:]
-	maxSize := uint64(len(piecesHash)) * uint64(DataLen) + uint64(len(buf))
+	maxSize := uint64(len(piecesHash))*uint64(DataLen) + uint64(len(buf))
 
-	if offset == 0 && size == 0{
+	if offset == 0 && size == 0 {
 		size = maxSize
 	}
-	if size + offset > maxSize {
-		return xerrors.Errorf("Piece Size is Out of Range [offset: %w, size:%w, max_size:%w]", offset, size, max_size)
+	if size+offset > maxSize {
+		return xerrors.Errorf("Piece Size is Out of Range [offset: %w, size:%w, max_size:%w]", offset, size, maxSize)
 	}
 
 	piecesHash = piecesHash[offset/uint64(DataLen):]
-	rangePiece := RangePiece{
+	rangePiece := &RangePiece{
 		offset: offset,
-		size: size,
-		lenth: uint64(DataLen),
+		size:   size,
+		lenth:  uint64(DataLen),
 	}
-	for{
+	for {
 		rstart, rsize := rangePiece.nextRange()
 		if rsize == 0 {
 			break
 		}
+		var wbuf []byte
 		if len(piecesHash) != 0 {
 			data, err := u.getPiece(ctx, piecesHash[0])
 			if err != nil {
@@ -85,31 +102,29 @@ func (u *User) ReadPieceRange(
 }
 
 func (u *User) getPiece(ctx context.Context, pieceHash storage.Hash) (*storage.DecodedData, error) {
-	// GET from chian/provider
+	// todo: GET from chian/provider
 	// buf, err := GetPieceFromProvider(pieceHash)
-	unpaddedSectorSize := abi.PaddedPieceSize(u.sectorSize).Unpadded()
-	buf := make([]byte, unpaddedSectorSize)
-
-	read, err := in.Read(buf[:])
-	if err != nil && err != io.EOF {
-		return storage.DecodedData{}, err
-	}
-
-	var data *storage.DecodedData = &storage.DecodedData{}
-	err = data.Deserialize(buf[:read])
+	data, err := u.encoder.DecodePiece(ctx, u.sectorSize, pieceHash)
 	return data, err
 }
 
 type RangePiece struct {
 	offset uint64
-	size uint64
-	lenth uint64
+	size   uint64
+	lenth  uint64
 }
 
-func (rp RangePiece) nextRange() (uint64, uint64) {
+func (rp *RangePiece) nextRange() (uint64, uint64) {
 	start := rp.offset % rp.lenth
-	size := min(rp.size, rp.lenth-rp.start)
+	size := min(rp.size, rp.lenth-start)
 	rp.offset = rp.offset + size
 	rp.size = rp.size - size
 	return start, size
-}
\ No newline at end of file
+}
+
+func min(x, y uint64) uint64 {
+	if x < y {
+		return x
+	}
+	return y
+}

build/storage/storage.go

@@ -12,6 +12,7 @@ import (
 
 	"fil_integrate/build/storiface"
 )
+const SectorSize32MiB uint64 = 32*1024*1024
 
 type Data = io.Reader
 

cmd/actor-test/main.go

+package main
+
+import (
+	"os"
+
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+
+	"fil_integrate/actor/user"
+)
+
+var log = logging.Logger("bench")
+
+func main() {
+	logging.SetLogLevel("*", "INFO")
+
+	log.Info("Starting test")
+
+	app := &cli.App{
+		Name:    "test",
+		Usage:   "test  the actor(user, provider, keeper)",
+		Version: "1.0.1",
+		Commands: []*cli.Command{
+			testUser,
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Warnf("%+v", err)
+		return
+	}
+}
+
+var testUser = &cli.Command{
+	Name:  "test-all",
+	Usage: "Test Seal the sectors and generate window post",
+	// Flags: []cli.Flag{
+	// 	&cli.StringFlag{
+	// 		Name:  "sector-size",
+	// 		Value: "8MiB",
+	// 		Usage: "size of the sectors in bytes",
+	// 	},
+	// 	&cli.IntFlag{
+	// 		Name:  "num-agg",
+	// 		Value: 8,
+	// 		Usage: "How many window-post proofs used to aggregate",
+	// 	},
+	// },
+	Action: func(c *cli.Context) error {
+		err := user.TestUser()
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+}
\ No newline at end of file

seal/encoder.go

@@ -152,22 +152,28 @@ func (sp *Encoder) EncodeData(
 	return pieces, nil
 }
 
-func DecodePiece(
+func (sp *Encoder) DecodePiece(
 	ctx context.Context,
 	sectorSize abi.SectorSize,
-	in io.Reader,
-) (storage.DecodedData, error) {
+	pieceHash storage.Hash,
+) (*storage.DecodedData, error) {
+	filename := filepath.Join(sp.Root, "pieces", fmt.Sprintf("%x.dat", pieceHash[:]))
+	in, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer in.Close()
 	unpaddedSectorSize := abi.PaddedPieceSize(sectorSize).Unpadded()
 	buf := make([]byte, unpaddedSectorSize)
 
 	read, err := in.Read(buf[:])
 	if err != nil && err != io.EOF {
-		return storage.DecodedData{}, err
+		return nil, err
 	}
 
 	var data *storage.DecodedData = &storage.DecodedData{}
 	err = data.Deserialize(buf[:read])
-	return *data, err
+	return data, err
 }
 
 func min(x, y uint32) uint32 {

seal/test.go

@@ -81,7 +81,7 @@ func TestSealAndUnseal() error {
 	// var sectors []storage.SectorRef
 
 	file := filepath.Join(tsdir, "input-0.dat")
-	generateRandomData(file, uint64(abi.PaddedPieceSize(sectorSize).Unpadded()), []byte("sectorSize"))
+	GenerateRandomData(file, uint64(abi.PaddedPieceSize(sectorSize).Unpadded()), []byte("sectorSize"))
 	in, err := os.Open(file)
 	if err != nil {
 		return err
@@ -230,7 +230,7 @@ func TestSplitDataInToPieces(sectorSize abi.SectorSize, dataSize uint64) error {
 	for i := 0; i < numFile; i++ {
 		filename := filepath.Join(tsdir, fmt.Sprintf("input-%d.dat", i))
 		start := time.Now()
-		b, err = generateRandomData(filename, dataSize, b)
+		b, err = GenerateRandomData(filename, dataSize, b)
 		if err != nil {
 			return err
 		}
@@ -309,7 +309,7 @@ func TestSplitDataInToPieces(sectorSize abi.SectorSize, dataSize uint64) error {
 		}
 		defer out.Close()
 
-		err = decodePiecesToData(sb, ctx, tsdir, sectorSize, finalPiece.Commitment, out)
+		err = decodePiecesToData(sb, sp, ctx, tsdir, sectorSize, finalPiece.Commitment, out)
 		if err != nil {
 			return err
 		}
@@ -378,7 +378,7 @@ func TestSealAndWindowPoSt(sectorSize abi.SectorSize, numAggregate int) error {
 		filename := filepath.Join(tsdir, fmt.Sprintf("input-%d.dat", i))
 		r := rand.New(rand.NewSource(time.Now().UnixNano()))
 		Datasize := (r.Intn(1024*1024) + 1024*1024) * 32
-		b, err = generateRandomData(filename, uint64(Datasize), b)
+		b, err = GenerateRandomData(filename, uint64(Datasize), b)
 		if err != nil {
 			return err
 		}
@@ -533,7 +533,7 @@ func TestSealAndWindowPoSt(sectorSize abi.SectorSize, numAggregate int) error {
 		}
 		defer out.Close()
 
-		err = decodePiecesToData(sb, ctx, tsdir, sectorSize, finalPieces[i], out)
+		err = decodePiecesToData(sb, sp, ctx, tsdir, sectorSize, finalPieces[i], out)
 		if err != nil {
 			return err
 		}
@@ -565,7 +565,7 @@ func Insert(sortedPieces []storage.Piece, pieces []storage.Piece, finalPiece sto
 	return append(res, sortedPieces[i+1:]...)
 }
 
-func generateRandomData(filename string, dataSize uint64, b []byte) ([]byte, error) {
+func GenerateRandomData(filename string, dataSize uint64, b []byte) ([]byte, error) {
 	if _, err := os.Stat(filename); !os.IsNotExist(err) {
 		os.Remove(filename)
 	}
@@ -590,26 +590,24 @@ func generateRandomData(filename string, dataSize uint64, b []byte) ([]byte, err
 	return b, nil
 }
 
-func decodePiecesToData(sb *Sealer, ctx context.Context, tsdir string, sectorSize abi.SectorSize, finalHash storage.Hash, out io.Writer) error {
+func decodePiecesToData(sb *Sealer, sp *Encoder, ctx context.Context, tsdir string, sectorSize abi.SectorSize, finalHash storage.Hash, out io.Writer) error {
 	// var piecesHash []storage.Hash
-	file, err := unseal(sb, ctx, finalHash)
+	err := unseal(sb, ctx, finalHash)
 	if err != nil {
 		return err
 	}
-	data, err := DecodePiece(ctx, sectorSize, file)
-	file.Close()
+	data, err := sp.DecodePiece(ctx, sectorSize, finalHash)
 	if err != nil {
 		return err
 	}
 	piecesHash := data.PieceHash
 
 	for data.HasPre {
-		file, err = unseal(sb, ctx, data.PreHash)
+		err = unseal(sb, ctx, data.PreHash)
 		if err != nil {
 			return err
 		}
-		data, err = DecodePiece(ctx, sectorSize, file)
-		file.Close()
+		data, err = sp.DecodePiece(ctx, sectorSize, data.PreHash)
 		if err != nil {
 			return err
 		}
@@ -618,12 +616,11 @@ func decodePiecesToData(sb *Sealer, ctx context.Context, tsdir string, sectorSiz
 	buf := data.Data[:]
 
 	for _, pieceHash := range piecesHash {
-		file, err = unseal(sb, ctx, pieceHash)
+		err = unseal(sb, ctx, pieceHash)
 		if err != nil {
 			return err
 		}
-		data, err := DecodePiece(ctx, sectorSize, file)
-		file.Close()
+		data, err := sp.DecodePiece(ctx, sectorSize, pieceHash)
 		if err != nil {
 			return err
 		}
@@ -640,25 +637,21 @@ func decodePiecesToData(sb *Sealer, ctx context.Context, tsdir string, sectorSiz
 	return nil
 }
 
-func unseal(sb *Sealer, ctx context.Context, fileHash storage.Hash) (*os.File, error) {
+func unseal(sb *Sealer, ctx context.Context, fileHash storage.Hash) error {
 	rangeSector, ok := hashMap[fileHash]
 	filename := filepath.Join(sb.sectors.GetRoot(), "pieces", fmt.Sprintf("%x.dat", fileHash[:]))
 	if ok {
 		file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644)
 		if err != nil {
-			return nil, err
+			return err
 		}
+		defer file.Close()
 		err = sb.UnsealedRange(ctx, file, rangeSector.Sector, rangeSector.Unsealed, rangeSector.Offset, rangeSector.Size)
 		if err != nil {
-			return nil, err
+			return err
 		}
-		file.Close()
-	}
-	file, err := os.Open(filename)
-	if err != nil {
-		return nil, err
 	}
-	return file, nil
+	return nil
 }
 
 func checkDecodedFile(root string, i int) (bool, error) {

seal/type.go

@@ -21,6 +21,7 @@ type PieceEncoder interface {
 	// Split and encode data into pieces
 	// Pieces structure is [ Tag | MetaData | HashData ] or [ Tag | PreHash | HashData]
 	EncodeDataToPieces(ctx context.Context, sectorSize abi.SectorSize, file storage.Data) (storage.Piece, []storage.Piece, error)
+	DecodePiece(ctx context.Context, sectorSize abi.SectorSize, pieceHash storage.Hash) (*storage.DecodedData, error)
 }
 
 //interface