Class ParquetDestination<TInput>
Inheritance
object
DataFlowDestination<TInput>
DataFlowStreamDestination<TInput>
ParquetDestination<TInput>
Implements
IDataFlowStreamDestination<TInput>
IDataFlowBatchDestination<TInput>
IDataFlowDestination<TInput>
Inherited Members
object.Equals(object)
object.Equals(object, object)
object.GetHashCode()
object.GetType()
object.MemberwiseClone()
object.ReferenceEquals(object, object)
object.ToString()
Namespace: ETLBox.Parquet
Assembly: ETLBox.Parquet.dll
Syntax
public class ParquetDestination<TInput> : DataFlowStreamDestination<TInput>, ILoggableTask, IDataFlowLogging, IDataFlowStreamDestination<TInput>, IDataFlowStreamDestination, IDataFlowBatchDestination<TInput>, IDataFlowDestination<TInput>, IDataFlowBatchDestination, IDataFlowDestination, IDataFlowComponent
Type Parameters
| Name | Description |
|---|---|
| TInput |
Constructors
ParquetDestination()
Declaration
public ParquetDestination()
ParquetDestination(string, ResourceType)
Declaration
public ParquetDestination(string uri, ResourceType resourceType)
Parameters
| Type | Name | Description |
|---|---|---|
| string | uri | The source parquet file name or uri |
| ResourceType | resourceType | Specifies if data is loaded from a file, a web endpoint or other storage types (e.g. Azure Blob Storage) |
ParquetDestination(string)
Declaration
public ParquetDestination(string uri)
Parameters
| Type | Name | Description |
|---|---|---|
| string | uri | The source parquet file name or uri |
Properties
BatchSize
The batch size defines how many records needs to be in the Input buffer before data is written into the destination. The default batch size is 1000.
Declaration
public int BatchSize { get; set; }
Property Value
| Type | Description |
|---|---|
| int |
HttpContentType
The content type used when sending the http request content.
Declaration
public override string HttpContentType { get; set; }
Property Value
| Type | Description |
|---|---|
| string |
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.HttpContentType
ParquetColumns
List of mappings between columns names in parquet file and property name in object.
Declaration
public ICollection<ParquetColumn> ParquetColumns { get; set; }
Property Value
| Type | Description |
|---|---|
| System.Collections.Generic.ICollection<T><ParquetColumn> |
ParquetOptions
Options used when creating the ParquetWriter
Declaration
public ParquetOptions ParquetOptions { get; set; }
Property Value
| Type | Description |
|---|---|
| ParquetOptions |
SupportedDataTypes
Currently the ParquetDestination only supports 'basics' data types. Access this property to get all Clr types that are supported.
Declaration
public Type[] SupportedDataTypes { get; }
Property Value
| Type | Description |
|---|---|
| System.Type[] |
TaskName
A name to identify the task or component. Every component or task comes with a default name that can be overwritten.
Declaration
public override string TaskName { get; set; }
Property Value
| Type | Description |
|---|---|
| string |
Overrides
UseBufferBlock
Declaration
protected override bool UseBufferBlock { get; set; }
Property Value
| Type | Description |
|---|---|
| bool |
Overrides
ETLBox.DataFlow.DataFlowDestination<TInput>.UseBufferBlock
Methods
CheckParameter()
Declaration
protected override void CheckParameter()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.CheckParameter()
CloseStream()
Declaration
protected override void CloseStream()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.CloseStream()
FindParquetNameMapping(string)
Declaration
public string FindParquetNameMapping(string propertyName)
Parameters
| Type | Name | Description |
|---|---|---|
| string | propertyName |
Returns
| Type | Description |
|---|---|
| string |
FindPropNameMapping(string)
Declaration
public string FindPropNameMapping(string columnName)
Parameters
| Type | Name | Description |
|---|---|---|
| string | columnName |
Returns
| Type | Description |
|---|---|
| string |
InitStream()
Declaration
protected override void InitStream()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.InitStream()
PrepareParameter()
Declaration
protected override void PrepareParameter()
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.PrepareParameter()
WriteBatch(TInput[])
Declaration
protected override void WriteBatch(TInput[] data)
Parameters
| Type | Name | Description |
|---|---|---|
| TInput[] | data |
Overrides
ETLBox.DataFlow.DataFlowDestination<TInput>.WriteBatch(TInput[])
WriteIntoStream(TInput)
Declaration
protected override void WriteIntoStream(TInput data)
Parameters
| Type | Name | Description |
|---|---|---|
| TInput | data |
Overrides
ETLBox.DataFlow.DataFlowStreamDestination<TInput>.WriteIntoStream(TInput)