pump

package
v1.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 4, 2021 License: MIT Imports: 21 Imported by: 3

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func PumpIt

func PumpIt(enumerator Enumerator, collector Collector, drain Drain, failedBlocksWriter FailedBlocksWriter, progressWriter ProgressWriter, worker uint)

Types

type APICollector

type APICollector struct {
	URL string
}

func NewAPICollector

func NewAPICollector(URL string) *APICollector

func (*APICollector) Blocks

func (a *APICollector) Blocks(in <-chan BlockInfo, out chan<- Block) error

type APIDrain

type APIDrain struct {
	// contains filtered or unexported fields
}

func NewAPIDrain

func NewAPIDrain(URL string) *APIDrain

func NewAPIDrainWithShell

func NewAPIDrainWithShell(shell *shell.Shell) *APIDrain

func (*APIDrain) Drain

func (a *APIDrain) Drain(block Block) error

type APIPinEnumerator

type APIPinEnumerator struct {
	URL string
	// contains filtered or unexported fields
}

func NewAPIPinEnumerator

func NewAPIPinEnumerator(URL string, stream bool) *APIPinEnumerator

func (*APIPinEnumerator) CIDs

func (a *APIPinEnumerator) CIDs(out chan<- BlockInfo) error

func (*APIPinEnumerator) TotalCount

func (a *APIPinEnumerator) TotalCount() int

type Block

type Block struct {
	Error error
	CID   cid.Cid
	Data  []byte
}

type BlockInfo

type BlockInfo struct {
	Error error
	CID   cid.Cid
}

type Collector

type Collector interface {
	// Blocks read each CID from the input, retrieve the corresponding
	// block and emit it to the output
	Blocks(in <-chan BlockInfo, out chan<- Block) error
}

A Collector is able to read a block from a source

type CountedDrain

type CountedDrain interface {
	Drain
	SuccessfulBlocksCount() uint64
}

type CounterDrain

type CounterDrain struct {
	// contains filtered or unexported fields
}

func NewCountedDrain

func NewCountedDrain(drain Drain) *CounterDrain

func (*CounterDrain) Drain

func (c *CounterDrain) Drain(block Block) error

func (*CounterDrain) SuccessfulBlocksCount

func (c *CounterDrain) SuccessfulBlocksCount() uint64

type DatastoreCollector

type DatastoreCollector struct {
	// contains filtered or unexported fields
}

func NewBadgerCollector

func NewBadgerCollector(path string) (*DatastoreCollector, error)

func NewDatastoreCollector

func NewDatastoreCollector(dstore ds.Datastore) *DatastoreCollector

func NewFlatFSCollector

func NewFlatFSCollector(path string) (*DatastoreCollector, error)

func NewS3Collector

func NewS3Collector(config s3ds.Config) (*DatastoreCollector, error)

func (*DatastoreCollector) Blocks

func (d *DatastoreCollector) Blocks(in <-chan BlockInfo, out chan<- Block) error

type DatastoreDrain

type DatastoreDrain struct {
	// contains filtered or unexported fields
}

func NewBadgerDrain

func NewBadgerDrain(path string) (*DatastoreDrain, error)

func NewDatastoreDrain

func NewDatastoreDrain(dstore ds.Datastore) *DatastoreDrain

func NewFlatFSDrain

func NewFlatFSDrain(path string) (*DatastoreDrain, error)

func NewS3Drain

func NewS3Drain(config s3ds.Config) (*DatastoreDrain, error)

func (*DatastoreDrain) Drain

func (d *DatastoreDrain) Drain(block Block) error

type DatastoreEnumerator

type DatastoreEnumerator struct {
	// contains filtered or unexported fields
}

func NewBadgerEnumerator

func NewBadgerEnumerator(path string) (*DatastoreEnumerator, error)

func NewDatastoreEnumerator

func NewDatastoreEnumerator(dstore ds.Datastore) *DatastoreEnumerator

func NewFlatFSEnumerator

func NewFlatFSEnumerator(path string) (*DatastoreEnumerator, error)

func NewS3Enumerator

func NewS3Enumerator(config s3ds.Config) (*DatastoreEnumerator, error)

func (*DatastoreEnumerator) CIDs

func (d *DatastoreEnumerator) CIDs(out chan<- BlockInfo) error

func (*DatastoreEnumerator) TotalCount

func (*DatastoreEnumerator) TotalCount() int

type Drain

type Drain interface {
	Drain(block Block) error
}

A Drain is able to write a block to a destination

type Enumerator

type Enumerator interface {
	// TotalCount return the total number of existing blocks,
	// or -1 if unknown/unsupported.
	TotalCount() int

	// CIDs emit in the given channel each CID existing in the source
	CIDs(out chan<- BlockInfo) error
}

An Enumerator is able to enumerate the blocks from a source

type FailedBlocksWriter

type FailedBlocksWriter interface {
	Write(c cid.Cid) (int, error)
	Flush() error
	Count() uint
}

func NewFileEnumeratorWriter

func NewFileEnumeratorWriter(path string) (enumWriter FailedBlocksWriter, close func() error, err error)

func NewNullableFileEnumeratorWriter

func NewNullableFileEnumeratorWriter() FailedBlocksWriter

type FileEnumerator

type FileEnumerator struct {
	// contains filtered or unexported fields
}

func NewFileEnumerator

func NewFileEnumerator(file io.ReadSeeker) (*FileEnumerator, error)

func (*FileEnumerator) CIDs

func (f *FileEnumerator) CIDs(out chan<- BlockInfo) error

func (*FileEnumerator) TotalCount

func (f *FileEnumerator) TotalCount() int

type FileEnumeratorWriter

type FileEnumeratorWriter struct {
	// contains filtered or unexported fields
}

func (*FileEnumeratorWriter) Count

func (f *FileEnumeratorWriter) Count() uint

func (*FileEnumeratorWriter) Flush

func (f *FileEnumeratorWriter) Flush() error

func (*FileEnumeratorWriter) Write

func (f *FileEnumeratorWriter) Write(c cid.Cid) (int, error)

type MockCollector

type MockCollector struct {
	// contains filtered or unexported fields
}

func NewMockCollector

func NewMockCollector(source *sync.Map) *MockCollector

func (*MockCollector) Blocks

func (m *MockCollector) Blocks(in <-chan BlockInfo, out chan<- Block) error

type MockEnumerator

type MockEnumerator struct {
	// contains filtered or unexported fields
}

func (*MockEnumerator) CIDs

func (m *MockEnumerator) CIDs(out chan<- BlockInfo) error

func (*MockEnumerator) TotalCount

func (m *MockEnumerator) TotalCount() int

type NullProgressWriter

type NullProgressWriter struct {
}

func NewNullProgressWriter

func NewNullProgressWriter() *NullProgressWriter

func (*NullProgressWriter) Finish

func (p *NullProgressWriter) Finish()

func (*NullProgressWriter) Increment

func (p *NullProgressWriter) Increment() int

func (*NullProgressWriter) Prefix

func (p *NullProgressWriter) Prefix(elem string)

func (*NullProgressWriter) SetTotal

func (p *NullProgressWriter) SetTotal(total int)

type NullableFileEnumeratorWriter

type NullableFileEnumeratorWriter struct {
	// contains filtered or unexported fields
}

func (*NullableFileEnumeratorWriter) Count

func (*NullableFileEnumeratorWriter) Flush

func (*NullableFileEnumeratorWriter) Write

func (f *NullableFileEnumeratorWriter) Write(c cid.Cid) (int, error)

type ProgressBarWriter

type ProgressBarWriter struct {
	// contains filtered or unexported fields
}

func NewProgressWriter

func NewProgressWriter() *ProgressBarWriter

func (*ProgressBarWriter) Finish

func (p *ProgressBarWriter) Finish()

func (*ProgressBarWriter) Increment

func (p *ProgressBarWriter) Increment() int

func (*ProgressBarWriter) Prefix

func (p *ProgressBarWriter) Prefix(elem string)

func (*ProgressBarWriter) SetTotal

func (p *ProgressBarWriter) SetTotal(total int)

type ProgressWriter

type ProgressWriter interface {
	Increment() int
	SetTotal(total int)
	Prefix(elem string)
	Finish()
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL