add some necessary struct and functions

build/fr32/fr32.go

+package fr32
+
+import (
+	"math/bits"
+	"runtime"
+	"sync"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+var MTTresh = uint64(32 << 20)
+
+func mtChunkCount(usz abi.PaddedPieceSize) uint64 {
+	threads := (uint64(usz)) / MTTresh
+	if threads > uint64(runtime.NumCPU()) {
+		threads = 1 << (bits.Len32(uint32(runtime.NumCPU())))
+	}
+	if threads == 0 {
+		return 1
+	}
+	if threads > 32 {
+		return 32 // avoid too large buffers
+	}
+	return threads
+}
+
+func mt(in, out []byte, padLen int, op func(unpadded, padded []byte)) {
+	threads := mtChunkCount(abi.PaddedPieceSize(padLen))
+	threadBytes := abi.PaddedPieceSize(padLen / int(threads))
+
+	var wg sync.WaitGroup
+	wg.Add(int(threads))
+
+	for i := 0; i < int(threads); i++ {
+		go func(thread int) {
+			defer wg.Done()
+
+			start := threadBytes * abi.PaddedPieceSize(thread)
+			end := start + threadBytes
+
+			op(in[start.Unpadded():end.Unpadded()], out[start:end])
+		}(i)
+	}
+	wg.Wait()
+}
+
+func Pad(in, out []byte) {
+	// Assumes len(in)%127==0 and len(out)%128==0
+	if len(out) > int(MTTresh) {
+		mt(in, out, len(out), pad)
+		return
+	}
+
+	pad(in, out)
+}
+
+func pad(in, out []byte) {
+	chunks := len(out) / 128
+	for chunk := 0; chunk < chunks; chunk++ {
+		inOff := chunk * 127
+		outOff := chunk * 128
+
+		copy(out[outOff:outOff+31], in[inOff:inOff+31])
+
+		t := in[inOff+31] >> 6
+		out[outOff+31] = in[inOff+31] & 0x3f
+		var v byte
+
+		for i := 32; i < 64; i++ {
+			v = in[inOff+i]
+			out[outOff+i] = (v << 2) | t
+			t = v >> 6
+		}
+
+		t = v >> 4
+		out[outOff+63] &= 0x3f
+
+		for i := 64; i < 96; i++ {
+			v = in[inOff+i]
+			out[outOff+i] = (v << 4) | t
+			t = v >> 4
+		}
+
+		t = v >> 2
+		out[outOff+95] &= 0x3f
+
+		for i := 96; i < 127; i++ {
+			v = in[inOff+i]
+			out[outOff+i] = (v << 6) | t
+			t = v >> 2
+		}
+
+		out[outOff+127] = t & 0x3f
+	}
+}
+
+func Unpad(in []byte, out []byte) {
+	// Assumes len(in)%128==0 and len(out)%127==0
+	if len(in) > int(MTTresh) {
+		mt(out, in, len(in), unpad)
+		return
+	}
+
+	unpad(out, in)
+}
+
+func unpad(out, in []byte) {
+	chunks := len(in) / 128
+	for chunk := 0; chunk < chunks; chunk++ {
+		inOffNext := chunk*128 + 1
+		outOff := chunk * 127
+
+		at := in[chunk*128]
+
+		for i := 0; i < 32; i++ {
+			next := in[i+inOffNext]
+
+			out[outOff+i] = at
+			//out[i] |= next << 8
+
+			at = next
+		}
+
+		out[outOff+31] |= at << 6
+
+		for i := 32; i < 64; i++ {
+			next := in[i+inOffNext]
+
+			out[outOff+i] = at >> 2
+			out[outOff+i] |= next << 6
+
+			at = next
+		}
+
+		out[outOff+63] ^= (at << 6) ^ (at << 4)
+
+		for i := 64; i < 96; i++ {
+			next := in[i+inOffNext]
+
+			out[outOff+i] = at >> 4
+			out[outOff+i] |= next << 4
+
+			at = next
+		}
+
+		out[outOff+95] ^= (at << 4) ^ (at << 2)
+
+		for i := 96; i < 127; i++ {
+			next := in[i+inOffNext]
+
+			out[outOff+i] = at >> 6
+			out[outOff+i] |= next << 2
+
+			at = next
+		}
+	}
+}

build/fr32/readers.go

+package fr32
+
+import (
+	"io"
+	"math/bits"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+type unpadReader struct {
+	src io.Reader
+
+	left uint64
+	work []byte
+}
+
+func NewUnpadReader(src io.Reader, sz abi.PaddedPieceSize) (io.Reader, error) {
+	if err := sz.Validate(); err != nil {
+		return nil, xerrors.Errorf("bad piece size: %w", err)
+	}
+
+	buf := make([]byte, MTTresh*mtChunkCount(sz))
+
+	return &unpadReader{
+		src: src,
+
+		left: uint64(sz),
+		work: buf,
+	}, nil
+}
+
+func (r *unpadReader) Read(out []byte) (int, error) {
+	if r.left == 0 {
+		return 0, io.EOF
+	}
+
+	chunks := len(out) / 127
+
+	outTwoPow := 1 << (63 - bits.LeadingZeros64(uint64(chunks*128)))
+
+	if err := abi.PaddedPieceSize(outTwoPow).Validate(); err != nil {
+		return 0, xerrors.Errorf("output must be of valid padded piece size: %w", err)
+	}
+
+	todo := abi.PaddedPieceSize(outTwoPow)
+	if r.left < uint64(todo) {
+		todo = abi.PaddedPieceSize(1 << (63 - bits.LeadingZeros64(r.left)))
+	}
+
+	r.left -= uint64(todo)
+
+	n, err := r.src.Read(r.work[:todo])
+	if err != nil && err != io.EOF {
+		return n, err
+	}
+
+	if n != int(todo) {
+		return 0, xerrors.Errorf("didn't read enough: %w", err)
+	}
+
+	Unpad(r.work[:todo], out[:todo.Unpadded()])
+
+	return int(todo.Unpadded()), err
+}
+
+type padWriter struct {
+	dst io.Writer
+
+	stash []byte
+	work  []byte
+}
+
+func NewPadWriter(dst io.Writer) io.WriteCloser {
+	return &padWriter{
+		dst: dst,
+	}
+}
+
+func (w *padWriter) Write(p []byte) (int, error) {
+	in := p
+
+	if len(p)+len(w.stash) < 127 {
+		w.stash = append(w.stash, p...)
+		return len(p), nil
+	}
+
+	if len(w.stash) != 0 {
+		in = append(w.stash, in...)
+	}
+
+	for {
+		pieces := subPieces(abi.UnpaddedPieceSize(len(in)))
+		biggest := pieces[len(pieces)-1]
+
+		if abi.PaddedPieceSize(cap(w.work)) < biggest.Padded() {
+			w.work = make([]byte, 0, biggest.Padded())
+		}
+
+		Pad(in[:int(biggest)], w.work[:int(biggest.Padded())])
+
+		n, err := w.dst.Write(w.work[:int(biggest.Padded())])
+		if err != nil {
+			return int(abi.PaddedPieceSize(n).Unpadded()), err
+		}
+
+		in = in[biggest:]
+
+		if len(in) < 127 {
+			if cap(w.stash) < len(in) {
+				w.stash = make([]byte, 0, len(in))
+			}
+			w.stash = w.stash[:len(in)]
+			copy(w.stash, in)
+
+			return len(p), nil
+		}
+	}
+}
+
+func (w *padWriter) Close() error {
+	if len(w.stash) > 0 {
+		return xerrors.Errorf("still have %d unprocessed bytes", len(w.stash))
+	}
+
+	// allow gc
+	w.stash = nil
+	w.work = nil
+	w.dst = nil
+
+	return nil
+}

build/fr32/readers_test.go

+package fr32_test
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"fil_integrate/extern/sector-storage/fr32"
+)
+
+func TestUnpadReader(t *testing.T) {
+	ps := abi.PaddedPieceSize(64 << 20).Unpadded()
+
+	raw := bytes.Repeat([]byte{0x77}, int(ps))
+
+	padOut := make([]byte, ps.Padded())
+	fr32.Pad(raw, padOut)
+
+	r, err := fr32.NewUnpadReader(bytes.NewReader(padOut), ps.Padded())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// using bufio reader to make sure reads are big enough for the padreader - it can't handle small reads right now
+	readered, err := ioutil.ReadAll(bufio.NewReaderSize(r, 512))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	require.Equal(t, raw, readered)
+}

build/fr32/utils.go

+package fr32
+
+import (
+	"math/bits"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+func subPieces(in abi.UnpaddedPieceSize) []abi.UnpaddedPieceSize {
+	// Convert to in-sector bytes for easier math:
+	//
+	// (we convert to sector bytes as they are nice round binary numbers)
+
+	w := uint64(in.Padded())
+
+	out := make([]abi.UnpaddedPieceSize, bits.OnesCount64(w))
+	for i := range out {
+		// Extract the next lowest non-zero bit
+		next := bits.TrailingZeros64(w)
+		psize := uint64(1) << next
+		// e.g: if the number is 0b010100, psize will be 0b000100
+
+		// set that bit to 0 by XORing it, so the next iteration looks at the
+		// next bit
+		w ^= psize
+
+		// Add the piece size to the list of pieces we need to create
+		out[i] = abi.PaddedPieceSize(psize).Unpadded()
+	}
+	return out
+}

build/proof/proof.go

+package proof
+
+import(
+        "github.com/ipfs/go-cid"
+        "github.com/filecoin-project/go-state-types/abi"
+)
+
+type SectorInfo struct {
+     SealProof    abi.RegisteredSealProof // RegisteredProof used when sealing - needs to be mapped to PoSt registered proof when used to verify a PoSt
+     SectorNumber abi.SectorNumber
+     SealedCID    cid.Cid // CommR
+}
+
+type SealVerifyInfo struct {
+     SealProof             abi.RegisteredSealProof
+     SectorID              abi.SectorID
+     DealIDs               []abi.DealID
+     Randomness            abi.SealRandomness
+     InteractiveRandomness abi.InteractiveSealRandomness
+     Proof                 []byte
+
+     // Safe because we get those from the miner actor
+     SealedCID   cid.Cid `checked:"true"` // CommR
+     UnsealedCID cid.Cid `checked:"true"` // CommD
+}
+
+type AggregateSealVerifyInfo struct {
+        Number                abi.SectorNumber
+        Randomness            abi.SealRandomness
+        InteractiveRandomness abi.InteractiveSealRandomness
+
+        // Safe because we get those from the miner actor
+        SealedCID   cid.Cid `checked:"true"` // CommR
+        UnsealedCID cid.Cid `checked:"true"` // CommD
+}
+
+type AggregateSealVerifyProofAndInfos struct {
+        Miner          abi.ActorID
+        SealProof      abi.RegisteredSealProof
+        AggregateProof abi.RegisteredAggregationProof
+        Proof          []byte
+        Infos          []AggregateSealVerifyInfo
+}
+
+type PoStProof struct {
+     PoStProof  abi.RegisteredPoStProof
+     ProofBytes []byte
+}
+
+type WinningPoStVerifyInfo struct {
+     Randomness        abi.PoStRandomness
+     Proofs            []PoStProof
+     ChallengedSectors []SectorInfo
+     Prover            abi.ActorID // used to derive 32-byte prover ID
+}
+
+type WindowPoStVerifyInfo struct {
+     Randomness        abi.PoStRandomness
+     Proofs            []PoStProof
+     ChallengedSectors []SectorInfo
+     Prover            abi.ActorID // used to derive 32-byte prover ID
+}
+
+type AggregateWindowPostInfos struct{
+    PoStType          abi.RegisteredPoStProof
+    AggregateType     abi.RegisteredAggregationProof
+    Miner             abi.ActorID
+    AggregationProof  []byte
+    ChallengedSectors []SectorInfo
+    Arr               []uint
+    Randomnesses      []abi.PoStRandomness
+    SectorCount       []uint
+}
\ No newline at end of file

build/storage/storage.go

+package storage
+
+import (
+	"io"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+)
+
+type Data = io.Reader
+
+type SectorRef struct {
+	ID        abi.SectorID
+	ProofType abi.RegisteredSealProof
+}
+
+type PreCommit1Out []byte
+
+type Commit1Out []byte
+
+type Proof []byte
+
+type SectorCids struct {
+	Unsealed cid.Cid
+	Sealed   cid.Cid
+}
\ No newline at end of file

build/storiface/ffi.go

+package storiface
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+var ErrSectorNotFound = errors.New("sector not found")
+
+type UnpaddedByteIndex uint64
+
+func (i UnpaddedByteIndex) Padded() PaddedByteIndex {
+	return PaddedByteIndex(abi.UnpaddedPieceSize(i).Padded())
+}
+
+type PaddedByteIndex uint64
+
+type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error)

build/storiface/filetype.go

+package storiface
+
+import (
+	"fmt"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+const (
+	FTUnsealed SectorFileType = 1 << iota
+	FTSealed
+	FTCache
+
+	FileTypes = iota
+)
+
+var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache}
+
+const (
+	FTNone SectorFileType = 0
+)
+
+const FSOverheadDen = 10
+
+var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads
+	FTUnsealed: FSOverheadDen,
+	FTSealed:   FSOverheadDen,
+	FTCache:    141, // 11 layers + D(2x ssize) + C + R
+}
+
+var FsOverheadFinalized = map[SectorFileType]int{
+	FTUnsealed: FSOverheadDen,
+	FTSealed:   FSOverheadDen,
+	FTCache:    2,
+}
+
+type SectorFileType int
+
+func (t SectorFileType) String() string {
+	switch t {
+	case FTUnsealed:
+		return "unsealed"
+	case FTSealed:
+		return "sealed"
+	case FTCache:
+		return "cache"
+	default:
+		return fmt.Sprintf("<unknown %d>", t)
+	}
+}
+
+func (t SectorFileType) Has(singleType SectorFileType) bool {
+	return t&singleType == singleType
+}
+
+func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) {
+	var need uint64
+	for _, pathType := range PathTypes {
+		if !t.Has(pathType) {
+			continue
+		}
+
+		oh, ok := FSOverheadSeal[pathType]
+		if !ok {
+			return 0, xerrors.Errorf("no seal overhead info for %s", pathType)
+		}
+
+		need += uint64(oh) * uint64(ssize) / FSOverheadDen
+	}
+
+	return need, nil
+}
+
+func (t SectorFileType) All() [FileTypes]bool {
+	var out [FileTypes]bool
+
+	for i := range out {
+		out[i] = t&(1<<i) > 0
+	}
+
+	return out
+}
+
+type SectorPaths struct {
+	ID abi.SectorID
+
+	Unsealed string
+	Sealed   string
+	Cache    string
+}
+
+func ParseSectorID(baseName string) (abi.SectorID, error) {
+	var n abi.SectorNumber
+	var mid abi.ActorID
+	read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n)
+	if err != nil {
+		return abi.SectorID{}, xerrors.Errorf("sscanf sector name ('%s'): %w", baseName, err)
+	}
+
+	if read != 2 {
+		return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read)
+	}
+
+	return abi.SectorID{
+		Miner:  mid,
+		Number: n,
+	}, nil
+}
+
+func SectorName(sid abi.SectorID, numbers ...int32) string {
+	out := fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number)
+	for _, number := range numbers{
+		out = fmt.Sprintf("%s-%d", out, number)
+	}
+	return out
+}
+
+func PathByType(sps SectorPaths, fileType SectorFileType) string {
+	switch fileType {
+	case FTUnsealed:
+		return sps.Unsealed
+	case FTSealed:
+		return sps.Sealed
+	case FTCache:
+		return sps.Cache
+	}
+
+	panic("requested unknown path type")
+}
+
+func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) {
+	switch fileType {
+	case FTUnsealed:
+		sps.Unsealed = p
+	case FTSealed:
+		sps.Sealed = p
+	case FTCache:
+		sps.Cache = p
+	}
+}

build/storiface/storage.go

+package storiface
+
+type PathType string
+
+const (
+	PathStorage PathType = "storage"
+	PathSealing PathType = "sealing"
+)
+
+type AcquireMode string
+
+const (
+	AcquireMove AcquireMode = "move"
+	AcquireCopy AcquireMode = "copy"
+)

seal/type.go

@@ -33,9 +33,9 @@ type SectorVerifier interface{
 }
 
 type SectorProvider interface {
+	GetRoot() (string)
 	// * returns storiface.ErrSectorNotFound if a requested existing sector doesn't exist
 	// * returns an error when allocate is set, and existing isn't, and the sector exists
-	GetRoot() (string)
 	AcquireUnsealed(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error)
 	AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType, numbers ...int32) (storiface.SectorPaths, func(), error)
 }