ETLBox ETLBox ETLBox Buy Try Buy

Unit Testing

This recipe demonstrates how unit tests can be written for data flows. The approaches are starting points. ETLBox is a .NET library, so any test framework and supporting package can be used.

Unit test injecting source data

namespace ETLBoxTests.DataFlow {
    public class UnitTestsInjectDifferentSourceData {

        [Fact]
        public void RowTransformerTest() {
            //Arrange
            RowTransformer transformer = new RowTransformer();
            DataTransferObject testInput = new DataTransferObject() {
                Id = 1,
                Value = "Test"
            };

            //Act
            var result = transformer.ChangeRow(testInput);

            //Assert
            Assert.Equal(1, result.Id);
            Assert.Equal("Adjusted Row (Test)", result.Value);
        }

        [Fact]
        public void NetworkRunnerTest() {
            //Arrange
            NetworkRunner runner = new NetworkRunner();
            var testData = new List<DataTransferObject>() {
                    new DataTransferObject { Id = 1, Value = "TestDataRow 1" },
                    new DataTransferObject { Id = 2, Value = "TestDataRow 2" }
                };

            //Act
            runner.ExecuteFlow(testData);

            //Assert
            Assert.Collection(runner.Result,
                row => { Assert.Equal(1, row.Id); Assert.Equal("Adjusted Row (TestDataRow 1)", row.Value); },
                row => { Assert.Equal(2, row.Id); Assert.Equal("Adjusted Row (TestDataRow 2)", row.Value); }
            );
            Assert.Equal(System.Threading.Tasks.TaskStatus.RanToCompletion, runner.Flow.Destinations.First().Completion.Status);
        }
    }
}

namespace ETLBox.DataFlowTests.Demo {
    public class DataTransferObject {
        public int Id { get; set; }
        public string Value { get; set; }
    }

    public class NetworkRunner {
        public ICollection<DataTransferObject> SourceData { get; private set; }
        public ICollection<DataTransferObject> Result => Destination?.Data;
        public Network Flow { get; set; }

        MemorySource<DataTransferObject> Source = new MemorySource<DataTransferObject>();
        MemoryDestination<DataTransferObject> Destination = new MemoryDestination<DataTransferObject>();
        RowTransformation<DataTransferObject> Transformation = new RowTransformation<DataTransferObject>();
        RowTransformer Transformer = new RowTransformer();

        private Network CreateDataflow() {
            Source.Data = SourceData;
            Transformation.TransformationFunc = Transformer.ChangeRow;
            Source.LinkTo(Transformation).LinkTo(Destination);

            return new Network(Source);
        }

        public void ExecuteFlow(ICollection<DataTransferObject> sourceData) {
            SourceData = sourceData;
            Flow = CreateDataflow();
            Flow.Execute();
        }
    }

    public class RowTransformer {
        public DataTransferObject ChangeRow(DataTransferObject dto) {
            dto.Value = "Adjusted Row (" + dto.Value + ")";
            return dto;
        }
    }

    public static class App {
        public static int Start(string[] args) {
            NetworkRunner runner = new NetworkRunner();
            var prodData = new List<DataTransferObject>() {
                    new DataTransferObject { Id = 1, Value = "ProductionDataRow 1" },
                    new DataTransferObject { Id = 2, Value = "ProductionDataRow 2" }
                };
            runner.ExecuteFlow(prodData);
            return runner.Result.Count > 0 ? 1 : 0;
        }
    }
}

Unit tests replacing source and destination

namespace ETLBoxTests.DataFlow
{
    public class UnitTestsReplacingSourceAndDestinationExample {

        [Fact]
        public void RunTestReplacingSourceAndDestination() {
            //Arrange
            NetworkRunner runner = new NetworkRunner();
            var newSource = new MemorySource<DataTransferObject>();
            newSource.Data = new List<DataTransferObject>() {
                new DataTransferObject() { Id = 1, Value = "Test1" },
                new DataTransferObject() { Id = 2, Value = "Test2" },
                new DataTransferObject() { Id = 3, Value = "Test3" }
            };
            var newDestination = new MemoryDestination<DataTransferObject>();

            runner.SetupDataFlow();
            runner.Source = newSource;
            runner.Destination = newDestination;

            //Act
            runner.ExecuteFlow();

            //Assert
            Assert.Equal(3, newDestination.Data.Count);
        }
    }
}

namespace ETLBox.DataFlowTests.ReplacingSourceAndDestinationExample
{
    public class DataTransferObject
    {
        public int Id { get; set; }
        public string Value { get; set; }
    }

    public class NetworkRunner
    {
        public IDataFlowExecutableSource<DataTransferObject> Source;
        public IDataFlowDestination<DataTransferObject> Destination;
        public IDataFlowTransformation<DataTransferObject, DataTransferObject> Transformation;

        public void SetupDataFlow() {
            var dbSource = new DbSource<DataTransferObject>();
                dbSource.TableName = "SourceTable";
            Source = dbSource;

            var dbDest = new DbDestination<DataTransferObject>();
            dbDest.TableName = "DestTable";
            Destination = dbDest;

            var trans = new RowTransformation<DataTransferObject>();
            trans.TransformationFunc = ChangeRow;
            Transformation = trans;


        }

        private DataTransferObject ChangeRow(DataTransferObject dto) {
            dto.Value = "Adjusted Row (" + dto.Value + ")";
            return dto;
        }

        public void ExecuteFlow() {
            Source.LinkTo(Transformation).LinkTo(Destination);
            Network.Execute(Source);
        }
    }

    public static class App
    {
        public static int Start(string[] args) {
            NetworkRunner runner = new NetworkRunner();
            runner.SetupDataFlow();
            runner.ExecuteFlow();
            return 1;
        }
    }
}

Replacing nodes in an already linked flow

NetworkHelper swaps sources, destinations, and error destinations in a flow that is already linked. The production class that called LinkTo stays unchanged. The test supplies real components such as MemorySource, MemoryDestination, CustomSource, or CustomDestination, then runs Network.Execute on the replacement source.

Call the helper before Network.Execute. A replacement has to be a real data flow component. An interface-only mock is not a node in the network. Link predicates stay on the link, so a filtered destination still receives only the rows that matched.

Pick the node by index, or match exactly one component, for example by TaskName:

var flow = new ProductionFlow();
flow.Link();

var testSource = new MemorySource<Row>();
testSource.Data = new List<Row> {
    new Row { Id = 1, Value = "A" },
    new Row { Id = 2, Value = "B" }
};
var testDestination = new MemoryDestination<Row>();

NetworkHelper.ReplaceSource(flow.Source, sources => sources[0], testSource);
NetworkHelper.ReplaceDestination(testSource, destinations => destinations[0], testDestination);
Network.Execute(testSource);

Assert.Equal(new[] { "Adjusted A", "Adjusted B" }, testDestination.Data.Select(row => row.Value));

ProductionFlow in this example already links a DbSource named DbSource through a row transformation to a DbDestination named DbDestination. The same swap can select those ends by name:

NetworkHelper.ReplaceSource(flow.Source,
    source => ((IDataFlowComponent)source).TaskName == "DbSource", testSource);
NetworkHelper.ReplaceDestination(testSource,
    destination => ((IDataFlowComponent)destination).TaskName == "DbDestination", testDestination);

ReplaceErrorDestination works the same way for a destination linked with LinkErrorTo. To assert a write without a database, use a CustomDestination and let Moq verify the action inside it. Moq replaces the side effect, while the destination itself remains an ETLBox component.

Unit tests using Moq for Source

namespace ETLBoxTests.DataFlow
{
    public class UnitTestsUsingMoqForSource {

        [Fact]
        public void RunTestUsingMoq() {
            //Arrange
            NetworkRunner runner = new NetworkRunner();
            var newSource = new MemorySource<DataTransferObject>();
            newSource.Data = new List<DataTransferObject>() {
                new DataTransferObject() { Id = 1, Value = "Test1" },
                new DataTransferObject() { Id = 2, Value = "Test2" },
                new DataTransferObject() { Id = 3, Value = "Test3" }
            };


            //Setting up moq source
            var moqSource = new Mock<IDataFlowExecutableSource<DataTransferObject>>();
            newSource.InitBufferObjects();
            moqSource
                .Setup(s => s.SourceBlock)
                .Returns(newSource.SourceBlock);
            moqSource
                .Setup(s => s.ExecuteAsync())
                .Callback(() => newSource.ExecuteAsync());
            moqSource
                .Setup(s => s.LinkTo(It.IsAny<IDataFlowDestination<DataTransferObject>>()))
                .Callback<IDataFlowDestination<DataTransferObject>>(d => newSource.LinkTo(d));

            //Use moq with transformation and destinations is currently not supported!
            //Use the interfaces instead
            runner.SetupDataFlow();

            var newDestination = new MemoryDestination<DataTransferObject>();

            //Injecting moq objects
            runner.Source = moqSource.Object;
            runner.Destination = newDestination;

            //Act
            runner.ExecuteFlow();

            //Assert
            Assert.Equal(3, newDestination.Data.Count);
        }
    }
}

namespace ETLBox.DataFlowTests.MoqDemo
{
    public class DataTransferObject
    {
        public int Id { get; set; }
        public string Value { get; set; }
    }

    public class NetworkRunner
    {
        public IDataFlowExecutableSource<DataTransferObject> Source;
        public IDataFlowDestination<DataTransferObject> Destination;
        public IDataFlowTransformation<DataTransferObject, DataTransferObject> Transformation;

        public void SetupDataFlow() {
            var dbSource = new DbSource<DataTransferObject>();
                dbSource.TableName = "SourceTable";
            Source = dbSource;

            var dbDest = new DbDestination<DataTransferObject>();
            dbDest.TableName = "DestTable";
            Destination = dbDest;

            var trans = new RowTransformation<DataTransferObject>();
            trans.TransformationFunc = ChangeRow;
            Transformation = trans;


        }

        private DataTransferObject ChangeRow(DataTransferObject dto) {
            dto.Value = "Adjusted Row (" + dto.Value + ")";
            return dto;
        }

        public void ExecuteFlow() {
            Source.LinkTo(Transformation);
            Transformation.LinkTo(Destination);
            //Do not use mocked components here, they won't be recognized by the network!
            Network.Execute(Transformation);
        }
    }

    public static class App
    {
        public static int Start(string[] args) {
            NetworkRunner runner = new NetworkRunner();
            runner.SetupDataFlow();
            runner.ExecuteFlow();
            return 1;
        }
    }
}
Edit this page on GitHub
Prev
Database Performance Tester