ParquetDestination<TInput>
Class ParquetDestination<TInput>
Implements
Inherited Members
Namespace: ETLBox.Parquet
Assembly: ETLBox.Parquet.dll
Syntax
public class ParquetDestination<TInput> : DataFlowStreamDestination<TInput>, IDataFlowStreamDestination<TInput>, IDataFlowStreamDestination, IDataFlowBatchDestination<TInput>, IDataFlowDestination<TInput>, IDataFlowBatchDestination, IDataFlowDestination, IDataFlowComponent, ILoggableTask
Type Parameters
Name | Description |
---|---|
TInput |
Constructors
ParquetDestination()
Declaration
public ParquetDestination()
ParquetDestination(string)
Declaration
public ParquetDestination(string uri)
Parameters
Type | Name | Description |
---|---|---|
string | uri | The source parquet file name or uri |
ParquetDestination(string, ResourceType)
Declaration
public ParquetDestination(string uri, ResourceType resourceType)
Parameters
Type | Name | Description |
---|---|---|
string | uri | The source parquet file name or uri |
ResourceType | resourceType | Specifies if data is loaded from a file, a web endpoint or other storage types (e.g. Azure Blob Storage) |
Properties
AddCustomMetaData
Each parquet writer allows to have a set of custom meta data - use this function to set the custom meta data for every uri (only called multiple times when using HasNextUri / GetNextUri.
Declaration
public Func<StreamMetaData, Dictionary<string, string>> AddCustomMetaData { get; set; }
Property Value
Type | Description |
---|---|
Func<StreamMetaData, Dictionary<string, string>> |
BatchSize
The batch size defines how many records needs to be in the Input buffer before data is written into the destination. The default batch size is 1000.
Declaration
public int BatchSize { get; set; }
Property Value
Type | Description |
---|---|
int |
HttpContentType
The content type used when sending the http request content.
Declaration
public override string HttpContentType { get; set; }
Property Value
Type | Description |
---|---|
string |
Overrides
ParquetColumns
List of mappings between columns names in parquet file and property name in object.
Declaration
public ICollection<ParquetColumn> ParquetColumns { get; set; }
Property Value
Type | Description |
---|---|
ICollection<ParquetColumn> |
ParquetOptions
Options used when creating the ParquetWriter
Declaration
public ParquetOptions ParquetOptions { get; set; }
Property Value
Type | Description |
---|---|
ParquetOptions |
SupportedDataTypes
Currently the ParquetDestination only supports 'basics' data types. Access this property to get all Clr types that are supported.
Declaration
public Type[] SupportedDataTypes { get; }
Property Value
Type | Description |
---|---|
Type[] |
UseBufferBlock
Declaration
protected override bool UseBufferBlock { get; set; }
Property Value
Type | Description |
---|---|
bool |
Overrides
Methods
CheckParameter()
Declaration
protected override void CheckParameter()
Overrides
CloseStream()
Declaration
protected override void CloseStream()
Overrides
FindParquetNameMapping(string)
Declaration
public string FindParquetNameMapping(string propertyName)
Parameters
Type | Name | Description |
---|---|---|
string | propertyName |
Returns
Type | Description |
---|---|
string |
FindPropNameMapping(string)
Declaration
public string FindPropNameMapping(string columnName)
Parameters
Type | Name | Description |
---|---|---|
string | columnName |
Returns
Type | Description |
---|---|
string |
InitStream()
Declaration
protected override void InitStream()
Overrides
PrepareParameterForCheck()
Declaration
protected override void PrepareParameterForCheck()
Overrides
Reset()
Declaration
protected override void Reset()
Overrides
WriteBatch(TInput[])
Declaration
protected override void WriteBatch(TInput[] data)
Parameters
Type | Name | Description |
---|---|---|
TInput[] | data |
Overrides
WriteIntoStream(TInput)
Declaration
protected override void WriteIntoStream(TInput data)
Parameters
Type | Name | Description |
---|---|---|
TInput | data |