Class ParquetDestination<TInput>

Inherited Members
object.Equals(object)
object.Equals(object, object)
object.GetHashCode()
object.GetType()
object.MemberwiseClone()
object.ReferenceEquals(object, object)
object.ToString()
Namespace: ETLBox.Parquet
Assembly: ETLBox.Parquet.dll
Syntax
    public class ParquetDestination<TInput> : DataFlowStreamDestination<TInput>, ILoggableTask, IDataFlowLogging, IDataFlowStreamDestination<TInput>, IDataFlowStreamDestination, IDataFlowBatchDestination<TInput>, IDataFlowDestination<TInput>, IDataFlowBatchDestination, IDataFlowDestination, IDataFlowComponent
Type Parameters
NameDescription
TInput

Constructors

ParquetDestination()

Declaration
    public ParquetDestination()

ParquetDestination(string, ResourceType)

Declaration
    public ParquetDestination(string uri, ResourceType resourceType)
Parameters
TypeNameDescription
stringuri

The source parquet file name or uri

ResourceTyperesourceType

Specifies if data is loaded from a file, a web endpoint or other storage types (e.g. Azure Blob Storage)

ParquetDestination(string)

Declaration
    public ParquetDestination(string uri)
Parameters
TypeNameDescription
stringuri

The source parquet file name or uri

Properties

BatchSize

The batch size defines how many records needs to be in the Input buffer before data is written into the destination. The default batch size is 1000.

Declaration
    public int BatchSize { get; set; }
Property Value
TypeDescription
int

HttpContentType

The content type used when sending the http request content.

Declaration
    public override string HttpContentType { get; set; }
Property Value
TypeDescription
string
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.HttpContentType

ParquetColumns

List of mappings between columns names in parquet file and property name in object.

Declaration
    public ICollection<ParquetColumn> ParquetColumns { get; set; }
Property Value
TypeDescription
System.Collections.Generic.ICollection<T><ParquetColumn>

ParquetOptions

Options used when creating the ParquetWriter

Declaration
    public ParquetOptions ParquetOptions { get; set; }
Property Value
TypeDescription
ParquetOptions

SupportedDataTypes

Currently the ParquetDestination only supports 'basics' data types. Access this property to get all Clr types that are supported.

Declaration
    public Type[] SupportedDataTypes { get; }
Property Value
TypeDescription
System.Type[]

TaskName

A name to identify the task or component. Every component or task comes with a default name that can be overwritten.

Declaration
    public override string TaskName { get; set; }
Property Value
TypeDescription
string
Overrides

UseBufferBlock

Declaration
    protected override bool UseBufferBlock { get; set; }
Property Value
TypeDescription
bool
Overrides
ETLBox.DataFlow.DataFlowDestination<TInput>.UseBufferBlock

Methods

CheckParameter()

Declaration
    protected override void CheckParameter()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.CheckParameter()

CloseStream()

Declaration
    protected override void CloseStream()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.CloseStream()

FindParquetNameMapping(string)

Declaration
    public string FindParquetNameMapping(string propertyName)
Parameters
TypeNameDescription
stringpropertyName
Returns
TypeDescription
string

FindPropNameMapping(string)

Declaration
    public string FindPropNameMapping(string columnName)
Parameters
TypeNameDescription
stringcolumnName
Returns
TypeDescription
string

InitStream()

Declaration
    protected override void InitStream()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.InitStream()

PrepareParameter()

Declaration
    protected override void PrepareParameter()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.PrepareParameter()

WriteBatch(TInput[])

Declaration
    protected override void WriteBatch(TInput[] data)
Parameters
TypeNameDescription
TInput[]data
Overrides
ETLBox.DataFlow.DataFlowDestination<TInput>.WriteBatch(TInput[])

WriteIntoStream(TInput)

Declaration
    protected override void WriteIntoStream(TInput data)
Parameters
TypeNameDescription
TInputdata
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.WriteIntoStream(TInput)

Implements