Class DataFlowStreamSource<TOutput>
Inheritance
object
DataFlowStreamSource<TOutput>
Inherited Members
object.Equals(object)
object.Equals(object, object)
object.GetHashCode()
object.GetType()
object.MemberwiseClone()
object.ReferenceEquals(object, object)
object.ToString()
Namespace: ETLBox.DataFlow
Assembly: ETLBox.dll
Syntax
public abstract class DataFlowStreamSource<TOutput> : DataFlowExecutableSource<TOutput>, ILoggableTask, IDataFlowLogging, IDataFlowStreamSource<TOutput>, IDataFlowExecutableSource<TOutput>, IDataFlowSource<TOutput>, IDataFlowSource, IDataFlowComponent, IDataFlowExecutableSource, IDataFlowStreamSource
Type Parameters
Properties
AzureBlobStorage
Use this to pass your AzureBlobStorage connection information.
There are different methods to connect with Azure Blob Storage:
Option 1: provide an existing BlockBlobClient
Option 2: create BlockBlobClient based on the Uri and optional credential tokens - the Uri needs to be the blob url
Option 3: create BlockBlobClient with provided ConnectionString and ContainerName, the Uri is then the name of the blob
Declaration
public AzureBlobStorageConfiguration AzureBlobStorage { get; set; }
Property Value
CollectUnparsedData
Declaration
public virtual bool CollectUnparsedData { get; set; }
Property Value
CreateStreamReader
By default, stream are created internally based on Uri & ResourceType.
If you already have an existing stream, use this function to returns a StreamReader based on this stream.
Declaration
public Func<string, StreamReader> CreateStreamReader { get; set; }
Property Value
| Type | Description |
|---|
| System.Func<T, TResult><string, System.IO.StreamReader> | |
Encoding
Encoding used to read data from the source file or web request.
Declaration
public Encoding Encoding { get; set; }
Property Value
| Type | Description |
|---|
| System.Text.Encoding | |
GetNextUri
This function returns the next url that is used for reading data. It will be called until HasNextUri returns false.
The incoming StreamMetaData holds information about the current progress and other meta data from the response, like unparsed
json data that contains references to the next page of the response.
This property can be used if you want to read multiple files or if you want to paginate through web responses.
Declaration
public Func<StreamMetaData, string> GetNextUri { get; set; }
Property Value
HasNextUri
This func determines if another request is started to read additional data from the next uri.
StreamMetaData has information about the current progress and other meta data from the response.
Declaration
public Func<StreamMetaData, bool> HasNextUri { get; set; }
Property Value
HttpClient
The System.Net.Http.HttpClient uses for the request. Use this client if you want to
add or change the http request data, e.g. you can add your authorization information here.
Declaration
public HttpClient HttpClient { get; set; }
Property Value
| Type | Description |
|---|
| System.Net.Http.HttpClient | |
HttpRequestMessage
The System.Net.Http.HttpRequestMessage used for the request in the HttpClient. Use this property
to add additional header data type or to change the http method (e.g. for POST requests).
Declaration
public HttpRequestMessage HttpRequestMessage { get; set; }
Property Value
| Type | Description |
|---|
| System.Net.Http.HttpRequestMessage | |
HttpResponseMessage
The System.Net.HttpResponseMessage returned from the current request.
Declaration
public HttpResponseMessage HttpResponseMessage { get; }
Property Value
| Type | Description |
|---|
| System.Net.Http.HttpResponseMessage | |
Declaration
protected StreamMetaData NewMetaDataObject { get; }
Property Value
RequestUri
The Uri used for the current request - this will only be different to the Uri
if you use the GetNextUri and HasNextUri functions.
Declaration
public string RequestUri { get; protected set; }
Property Value
ResourceType
Specifies the resource type. By default requests are made with HttpClient.
Specify ResourceType.File if you want to read from a json file.
Declaration
public ResourceType ResourceType { get; set; }
Property Value
RowModificationAction
Allows you to modify a record directly after it is read from the source and before it is send to the next component.
While processing of data in other components can happen asynchronously, the execution of this action is synchronous
and always occurs after each record was read and before a new record is read.
Declaration
public Action<TOutput, StreamMetaData> RowModificationAction { get; set; }
Property Value
SkipRows
Number of rows to skip before starting reading the header and csv data
Declaration
public int SkipRows { get; set; }
Property Value
StreamReader
The stream reader used for the current request.
Declaration
public StreamReader StreamReader { get; protected set; }
Property Value
| Type | Description |
|---|
| System.IO.StreamReader | |
UnparsedData
Contains all data from the stream that couldn't be processed.
Declaration
public string UnparsedData { get; }
Property Value
UnparsedDataBuilder
Declaration
protected StringBuilder UnparsedDataBuilder { get; set; }
Property Value
| Type | Description |
|---|
| System.Text.StringBuilder | |
Uri
Declaration
public string Uri { get; set; }
Property Value
Methods
CheckParameter()
Declaration
protected override void CheckParameter()
Overrides
CleanUpOnFaulted(Exception)
Declaration
protected override void CleanUpOnFaulted(Exception e)
Parameters
| Type | Name | Description |
|---|
| System.Exception | e | |
Overrides
CleanUpOnSuccess()
Declaration
protected override void CleanUpOnSuccess()
Overrides
CloseReader()
Declaration
protected abstract void CloseReader()
InitReader()
Declaration
protected abstract void InitReader()
OnExecutionDoAsyncWork()
Declaration
protected override void OnExecutionDoAsyncWork()
Overrides
ETLBox.DataFlow.DataFlowExecutableSource<TOutput>.OnExecutionDoAsyncWork()
OnExecutionDoSynchronousWork()
Declaration
protected override void OnExecutionDoSynchronousWork()
Overrides
ETLBox.DataFlow.DataFlowExecutableSource<TOutput>.OnExecutionDoSynchronousWork()
PrepareParameter()
Declaration
protected override void PrepareParameter()
Overrides
ReadAllRecords()
Declaration
protected abstract void ReadAllRecords()
Implements