Versions in this module Expand all Collapse all v0 v0.0.4 Mar 31, 2022 v0.0.3 Mar 31, 2022 v0.0.2 Mar 31, 2022 Changes in this version + const DefaultColumnBufferSize + const DefaultColumnIndexSizeLimit + const DefaultCreatedBy + const DefaultDataPageStatistics + const DefaultDataPageVersion + const DefaultPageBufferSize + const DefaultSkipBloomFilters + const DefaultSkipPageIndex + const MaxColumnDepth + const MaxColumnIndex + const MaxDefinitionLevel + const MaxRepetitionLevel + var Brotli = brotli.Codec + var ByteStreamSplit bytestreamsplit.Encoding + var DeltaBinaryPacked delta.BinaryPackedEncoding + var DeltaByteArray delta.ByteArrayEncoding + var DeltaLengthByteArray delta.LengthByteArrayEncoding + var ErrCorrupted = errors.New("corrupted parquet page") + var ErrMissingRootColumn = errors.New("parquet file is missing a root column") + var ErrRowGroupSchemaMismatch = errors.New("cannot write row groups with mismatching schemas") + var ErrRowGroupSchemaMissing = errors.New("cannot write rows to a row group which has no schema") + var ErrRowGroupSortingColumnsMismatch = errors.New("cannot write row groups with mismatching sorting columns") + var ErrSeekOutOfRange = errors.New("seek to row index out of page range") + var Gzip = gzip.Codec + var Lz4Raw = lz4.Codec + var Plain plain.Encoding + var PlainDictionary plain.DictionaryEncoding + var RLE rle.Encoding + var RLEDictionary rle.DictionaryEncoding + var Snappy snappy.Codec + var Uncompressed uncompressed.Codec + var Zstd = zstd.Codec + func CopyPages(dst PageWriter, src PageReader) (numValues int64, err error) + func CopyRows(dst RowWriter, src RowReader) (int64, error) + func CopyValues(dst ValueWriter, src ValueReader) (int64, error) + func Equal(v1, v2 Value) bool + func LookupCompressionCodec(codec format.CompressionCodec) compress.Codec + func LookupEncoding(enc format.Encoding) encoding.Encoding + func Print(w io.Writer, name string, node Node) error + func PrintIndent(w io.Writer, name string, node Node, pattern, newline string) error + type BloomFilter interface + Check func(value Value) (bool, error) + Size func() int64 + type BloomFilterColumn interface + Hash func() bloom.Hash + NewFilter func(numValues int64, bitsPerValue uint) bloom.MutableFilter + Path func() []string + func SplitBlockFilter(path ...string) BloomFilterColumn + type Buffer struct + func NewBuffer(options ...RowGroupOption) *Buffer + func (buf *Buffer) Column(i int) ColumnChunk + func (buf *Buffer) Len() int + func (buf *Buffer) Less(i, j int) bool + func (buf *Buffer) NumColumns() int + func (buf *Buffer) NumRows() int64 + func (buf *Buffer) Reset() + func (buf *Buffer) Rows() Rows + func (buf *Buffer) Schema() *Schema + func (buf *Buffer) Size() int64 + func (buf *Buffer) SortingColumns() []SortingColumn + func (buf *Buffer) Swap(i, j int) + func (buf *Buffer) Write(row interface{}) error + func (buf *Buffer) WriteRow(row Row) error + func (buf *Buffer) WriteRowGroup(rowGroup RowGroup) (int64, error) + type BufferedPage interface + Clone func() BufferedPage + DefinitionLevels func() []int8 + RepetitionLevels func() []int8 + Slice func(i, j int64) BufferedPage + WriteTo func(encoding.Encoder) error + type Column struct + func (c *Column) ChildByIndex(index int) Node + func (c *Column) ChildByName(name string) Node + func (c *Column) ChildNames() []string + func (c *Column) Children() []*Column + func (c *Column) Column(name string) *Column + func (c *Column) Columns() []*Column + func (c *Column) Compression() []compress.Codec + func (c *Column) Depth() int + func (c *Column) Encoding() []encoding.Encoding + func (c *Column) GoType() reflect.Type + func (c *Column) Index() int + func (c *Column) MaxDefinitionLevel() int + func (c *Column) MaxRepetitionLevel() int + func (c *Column) Name() string + func (c *Column) NumChildren() int + func (c *Column) Optional() bool + func (c *Column) Pages() Pages + func (c *Column) Path() []string + func (c *Column) Repeated() bool + func (c *Column) Required() bool + func (c *Column) String() string + func (c *Column) Type() Type + func (c *Column) ValueByIndex(base reflect.Value, index int) reflect.Value + func (c *Column) ValueByName(base reflect.Value, name string) reflect.Value + type ColumnBuffer interface + Cap func() int + Clone func() ColumnBuffer + Dictionary func() Dictionary + Len func() int + Less func(i, j int) bool + Page func() BufferedPage + Reset func() + Size func() int64 + Swap func(i, j int) + type ColumnChunk interface + BloomFilter func() BloomFilter + Column func() int + ColumnIndex func() ColumnIndex + NumValues func() int64 + OffsetIndex func() OffsetIndex + Pages func() Pages + Type func() Type + type ColumnIndex interface + IsAscending func() bool + IsDescending func() bool + MaxValue func(int) Value + MinValue func(int) Value + NullCount func(int) int64 + NullPage func(int) bool + NumPages func() int + type ColumnIndexer interface + ColumnIndex func() format.ColumnIndex + IndexPage func(numValues, numNulls int64, min, max Value) + Reset func() + type ColumnReader interface + Column func() int + Reset func(decoder encoding.Decoder) + Type func() Type + type CompressedPage interface + CRC func() uint32 + PageData func() io.Reader + PageHeader func() PageHeader + PageSize func() int64 + type Conversion interface + Column func(int) int + Convert func(dst, src Row) (Row, error) + Schema func() *Schema + func Convert(to, from Node) (conv Conversion, err error) + type ConvertError struct + From Node + Path []string + Reason string + To Node + func (e *ConvertError) Error() string + type DataPageHeader interface + DefinitionLevelEncoding func() format.Encoding + IsCompressed func(format.CompressionCodec) bool + MaxValue func() []byte + MinValue func() []byte + NullCount func() int64 + RepetitionLevelEncoding func() format.Encoding + type DataPageHeaderV1 struct + func (v1 DataPageHeaderV1) DefinitionLevelEncoding() format.Encoding + func (v1 DataPageHeaderV1) Encoding() format.Encoding + func (v1 DataPageHeaderV1) IsCompressed(codec format.CompressionCodec) bool + func (v1 DataPageHeaderV1) MaxValue() []byte + func (v1 DataPageHeaderV1) MinValue() []byte + func (v1 DataPageHeaderV1) NullCount() int64 + func (v1 DataPageHeaderV1) NumValues() int64 + func (v1 DataPageHeaderV1) PageType() format.PageType + func (v1 DataPageHeaderV1) RepetitionLevelEncoding() format.Encoding + func (v1 DataPageHeaderV1) String() string + type DataPageHeaderV2 struct + func (v2 DataPageHeaderV2) DefinitionLevelEncoding() format.Encoding + func (v2 DataPageHeaderV2) DefinitionLevelsByteLength() int64 + func (v2 DataPageHeaderV2) Encoding() format.Encoding + func (v2 DataPageHeaderV2) IsCompressed(codec format.CompressionCodec) bool + func (v2 DataPageHeaderV2) MaxValue() []byte + func (v2 DataPageHeaderV2) MinValue() []byte + func (v2 DataPageHeaderV2) NullCount() int64 + func (v2 DataPageHeaderV2) NumNulls() int64 + func (v2 DataPageHeaderV2) NumRows() int64 + func (v2 DataPageHeaderV2) NumValues() int64 + func (v2 DataPageHeaderV2) PageType() format.PageType + func (v2 DataPageHeaderV2) RepetitionLevelEncoding() format.Encoding + func (v2 DataPageHeaderV2) RepetitionLevelsByteLength() int64 + func (v2 DataPageHeaderV2) String() string + type Dictionary interface + Bounds func(indexed []int32) (min, max Value) + Index func(index int32) Value + Insert func(indexes []int32, values []Value) + Len func() int + Lookup func(indexes []int32, values []Value) + Page func() BufferedPage + Reset func() + Type func() Type + type DictionaryPageHeader struct + func (dict DictionaryPageHeader) Encoding() format.Encoding + func (dict DictionaryPageHeader) IsSorted() bool + func (dict DictionaryPageHeader) NumValues() int64 + func (dict DictionaryPageHeader) PageType() format.PageType + func (dict DictionaryPageHeader) String() string + type File struct + func OpenFile(r io.ReaderAt, size int64, options ...FileOption) (*File, error) + func (f *File) ColumnIndexes() []format.ColumnIndex + func (f *File) Lookup(key string) (value string, ok bool) + func (f *File) NumRowGroups() int + func (f *File) OffsetIndexes() []format.OffsetIndex + func (f *File) ReadAt(b []byte, off int64) (int, error) + func (f *File) ReadPageIndex() ([]format.ColumnIndex, []format.OffsetIndex, error) + func (f *File) Root() *Column + func (f *File) RowGroup(i int) RowGroup + func (f *File) Size() int64 + type FileConfig struct + SkipBloomFilters bool + SkipPageIndex bool + func DefaultFileConfig() *FileConfig + func NewFileConfig(options ...FileOption) (*FileConfig, error) + func (c *FileConfig) Apply(options ...FileOption) + func (c *FileConfig) ConfigureFile(config *FileConfig) + func (c *FileConfig) Validate() error + type FileOption interface + ConfigureFile func(*FileConfig) + func SkipPageIndex(skip bool) FileOption + type Group map[string]Node + func (g Group) ChildByName(name string) Node + func (g Group) ChildNames() []string + func (g Group) Compression() []compress.Codec + func (g Group) Encoding() []encoding.Encoding + func (g Group) GoType() reflect.Type + func (g Group) NumChildren() int + func (g Group) Optional() bool + func (g Group) Repeated() bool + func (g Group) Required() bool + func (g Group) String() string + func (g Group) Type() Type + func (g Group) ValueByName(base reflect.Value, name string) reflect.Value + type IndexedNode interface + ChildByIndex func(index int) Node + ValueByIndex func(base reflect.Value, index int) reflect.Value + type Kind int8 + const Boolean + const ByteArray + const Double + const FixedLenByteArray + const Float + const Int32 + const Int64 + const Int96 + func (k Kind) String() string + func (k Kind) Value(v []byte) Value + type Node interface + ChildByName func(name string) Node + ChildNames func() []string + Compression func() []compress.Codec + Encoding func() []encoding.Encoding + GoType func() reflect.Type + NumChildren func() int + Optional func() bool + Repeated func() bool + Required func() bool + String func() string + Type func() Type + ValueByName func(base reflect.Value, name string) reflect.Value + func BSON() Node + func Compressed(node Node, codecs ...compress.Codec) Node + func Date() Node + func Decimal(scale, precision int, typ Type) Node + func Encoded(node Node, encodings ...encoding.Encoding) Node + func Enum() Node + func Int(bitWidth int) Node + func JSON() Node + func Leaf(typ Type) Node + func List(of Node) Node + func Map(key, value Node) Node + func Optional(node Node) Node + func Repeated(node Node) Node + func Required(node Node) Node + func String() Node + func Time(unit TimeUnit) Node + func Timestamp(unit TimeUnit) Node + func UUID() Node + func Uint(bitWidth int) Node + type OffsetIndex interface + CompressedPageSize func(int) int64 + FirstRowIndex func(int) int64 + NumPages func() int + Offset func(int) int64 + type Page interface + Bounds func() (min, max Value) + Buffer func() BufferedPage + Column func() int + Dictionary func() Dictionary + NumNulls func() int64 + NumRows func() int64 + NumValues func() int64 + Size func() int64 + Values func() ValueReader + type PageBufferPool interface + GetPageBuffer func() io.ReadWriter + PutPageBuffer func(io.ReadWriter) + func NewFileBufferPool(tempdir, pattern string) PageBufferPool + func NewPageBufferPool() PageBufferPool + type PageHeader interface + Encoding func() format.Encoding + NumValues func() int64 + PageType func() format.PageType + type PageReader interface + ReadPage func() (Page, error) + type PageWriter interface + WritePage func(Page) (int64, error) + type Pages interface + type Reader struct + func NewReader(input io.ReaderAt, options ...ReaderOption) *Reader + func (r *Reader) NumRows() int64 + func (r *Reader) Read(row interface{}) (err error) + func (r *Reader) ReadRow(row Row) (Row, error) + func (r *Reader) Reset() + func (r *Reader) Schema() *Schema + func (r *Reader) SeekToRow(rowIndex int64) error + type ReaderConfig struct + Schema *Schema + func DefaultReaderConfig() *ReaderConfig + func NewReaderConfig(options ...ReaderOption) (*ReaderConfig, error) + func (c *ReaderConfig) Apply(options ...ReaderOption) + func (c *ReaderConfig) ConfigureReader(config *ReaderConfig) + func (c *ReaderConfig) Validate() error + type ReaderOption interface + ConfigureReader func(*ReaderConfig) + type RequiredReader interface + ReadRequired func(data []T) (int, error) + type RequiredWriter interface + WriteRequired func(data []T) (int, error) + type Row []Value + func (row Row) Equal(other Row) bool + type RowGroup interface + Column func(int) ColumnChunk + NumColumns func() int + NumRows func() int64 + Rows func() Rows + Schema func() *Schema + SortingColumns func() []SortingColumn + func ConvertRowGroup(rowGroup RowGroup, conv Conversion) RowGroup + func MergeRowGroups(rowGroups []RowGroup, options ...RowGroupOption) (RowGroup, error) + type RowGroupConfig struct + ColumnBufferSize int + Schema *Schema + SortingColumns []SortingColumn + func DefaultRowGroupConfig() *RowGroupConfig + func NewRowGroupConfig(options ...RowGroupOption) (*RowGroupConfig, error) + func (c *RowGroupConfig) Apply(options ...RowGroupOption) + func (c *RowGroupConfig) ConfigureRowGroup(config *RowGroupConfig) + func (c *RowGroupConfig) Validate() error + type RowGroupOption interface + ConfigureRowGroup func(*RowGroupConfig) + func ColumnBufferSize(size int) RowGroupOption + func SortingColumns(sortingColumns ...SortingColumn) RowGroupOption + type RowGroupReader interface + ReadRowGroup func() (RowGroup, error) + type RowGroupWriter interface + WriteRowGroup func(RowGroup) (int64, error) + type RowReadSeeker interface + type RowReader interface + ReadRow func(Row) (Row, error) + type RowReaderAt interface + ReadRowAt func(Row, int64) (Row, error) + type RowReaderFrom interface + ReadRowsFrom func(RowReader) (int64, error) + type RowReaderWithSchema interface + Schema func() *Schema + func ConvertRowReader(rows RowReader, conv Conversion) RowReaderWithSchema + type RowSeeker interface + SeekToRow func(int64) error + type RowWriter interface + WriteRow func(Row) error + type RowWriterAt interface + WriteRowAt func(Row, int64) error + type RowWriterTo interface + WriteRowsTo func(RowWriter) (int64, error) + type RowWriterWithSchema interface + Schema func() *Schema + type Rows interface + func NewRowGroupRowReader(rowGroup RowGroup) Rows + type Schema struct + func NewSchema(name string, root Node) *Schema + func SchemaOf(model interface{}) *Schema + func (s *Schema) ChildByName(name string) Node + func (s *Schema) ChildNames() []string + func (s *Schema) Compression() []compress.Codec + func (s *Schema) ConfigureReader(config *ReaderConfig) + func (s *Schema) ConfigureRowGroup(config *RowGroupConfig) + func (s *Schema) ConfigureWriter(config *WriterConfig) + func (s *Schema) Deconstruct(row Row, value interface{}) Row + func (s *Schema) Encoding() []encoding.Encoding + func (s *Schema) GoType() reflect.Type + func (s *Schema) Name() string + func (s *Schema) NumChildren() int + func (s *Schema) Optional() bool + func (s *Schema) Reconstruct(value interface{}, row Row) error + func (s *Schema) Repeated() bool + func (s *Schema) Required() bool + func (s *Schema) String() string + func (s *Schema) Type() Type + func (s *Schema) ValueByName(base reflect.Value, name string) reflect.Value + type SortConfig struct + Descending bool + MaxDefinitionLevel int + MaxRepetitionLevel int + NullsFirst bool + func (c *SortConfig) Apply(options ...SortOption) + func (c *SortConfig) ConfigureSort(config *SortConfig) + type SortFunc func(a, b []Value) int + func SortFuncOf(t Type, options ...SortOption) SortFunc + type SortOption interface + ConfigureSort func(*SortConfig) + func SortDescending(descending bool) SortOption + func SortMaxDefinitionLevel(level int) SortOption + func SortMaxRepetitionLevel(level int) SortOption + func SortNullsFirst(nullsFirst bool) SortOption + type SortingColumn interface + Descending func() bool + NullsFirst func() bool + Path func() []string + func Ascending(path ...string) SortingColumn + func Descending(path ...string) SortingColumn + func NullsFirst(sortingColumn SortingColumn) SortingColumn + type TimeUnit interface + Duration func() time.Duration + TimeUnit func() format.TimeUnit + var Microsecond TimeUnit = µsecond{} + var Millisecond TimeUnit = &millisecond{} + var Nanosecond TimeUnit = &nanosecond{} + type Type interface + ColumnOrder func() *format.ColumnOrder + Compare func(a, b Value) int + ConvertedType func() *deprecated.ConvertedType + Kind func() Kind + Length func() int + LogicalType func() *format.LogicalType + NewColumnBuffer func(columnIndex, bufferSize int) ColumnBuffer + NewColumnIndexer func(sizeLimit int) ColumnIndexer + NewColumnReader func(columnIndex, bufferSize int) ColumnReader + NewDictionary func(columnIndex, bufferSize int) Dictionary + PhysicalType func() *format.Type + ReadDictionary func(columnIndex, numValues int, decoder encoding.Decoder) (Dictionary, error) + String func() string + var BooleanType Type = primitiveType[bool]{} + var ByteArrayType Type = byteArrayType{} + var DoubleType Type = primitiveType[float64]{} + var FloatType Type = primitiveType[float32]{} + var Int32Type Type = primitiveType[int32]{} + var Int64Type Type = primitiveType[int64]{} + var Int96Type Type = primitiveType[deprecated.Int96]{} + func FixedLenByteArrayType(length int) Type + type Value struct + func ValueOf(v interface{}) Value + func (v Value) AppendBytes(b []byte) []byte + func (v Value) Boolean() bool + func (v Value) ByteArray() []byte + func (v Value) Bytes() []byte + func (v Value) Clone() Value + func (v Value) Column() int + func (v Value) DefinitionLevel() int + func (v Value) Double() float64 + func (v Value) Float() float32 + func (v Value) Format(w fmt.State, r rune) + func (v Value) GoString() string + func (v Value) Int32() int32 + func (v Value) Int64() int64 + func (v Value) Int96() deprecated.Int96 + func (v Value) IsNull() bool + func (v Value) Kind() Kind + func (v Value) Level(repetitionLevel, definitionLevel, columnIndex int) Value + func (v Value) RepetitionLevel() int + func (v Value) String() string + type ValueReader interface + ReadValues func([]Value) (int, error) + type ValueReaderFrom interface + ReadValuesFrom func(ValueReader) (int64, error) + type ValueWriter interface + WriteValues func([]Value) (int, error) + type ValueWriterTo interface + WriteValuesTo func(ValueWriter) (int64, error) + type WrappedNode interface + Unwrap func() Node + type Writer struct + func NewWriter(output io.Writer, options ...WriterOption) *Writer + func (w *Writer) Close() error + func (w *Writer) Flush() error + func (w *Writer) ReadRowsFrom(rows RowReader) (written int64, err error) + func (w *Writer) Reset(output io.Writer) + func (w *Writer) Schema() *Schema + func (w *Writer) Write(row interface{}) error + func (w *Writer) WriteRow(row Row) error + func (w *Writer) WriteRowGroup(rowGroup RowGroup) (int64, error) + type WriterConfig struct + BloomFilters []BloomFilterColumn + ColumnIndexSizeLimit int + ColumnPageBuffers PageBufferPool + CreatedBy string + DataPageStatistics bool + DataPageVersion int + KeyValueMetadata map[string]string + PageBufferPool PageBufferPool + PageBufferSize int + Schema *Schema + func DefaultWriterConfig() *WriterConfig + func NewWriterConfig(options ...WriterOption) (*WriterConfig, error) + func (c *WriterConfig) Apply(options ...WriterOption) + func (c *WriterConfig) ConfigureWriter(config *WriterConfig) + func (c *WriterConfig) Validate() error + type WriterOption interface + ConfigureWriter func(*WriterConfig) + func BloomFilters(filters ...BloomFilterColumn) WriterOption + func ColumnIndexSizeLimit(sizeLimit int) WriterOption + func ColumnPageBuffers(buffers PageBufferPool) WriterOption + func CreatedBy(createdBy string) WriterOption + func DataPageStatistics(enabled bool) WriterOption + func DataPageVersion(version int) WriterOption + func KeyValueMetadata(key, value string) WriterOption + func PageBufferSize(size int) WriterOption