ETLBox
Chunk Transformation
This article contains example code that shows the usage of the ChunkTransformation component.
ChunkTransformation splits each row into one or more text pieces and sends each piece downstream as its own row. Point it at the text with TextSelector or the [ChunkText] attribute, or pass an existing IngestionDocument with DocumentSelector or [ChunkDocument]. The default output is Chunk<T>, with Source, Text, Context, and Index. Context stays empty unless the chunker sets it, as HeaderChunker does with the heading path.
Chunk a typed object
This example reads a product CSV, builds one string from name, category, price, and description, and splits that string. MaxTokensPerChunk and OverlapTokens are small here so the short texts produce more than one chunk.
public class Product
{
public int Id { get; set; }
public string Name { get; set; }
public string Category { get; set; }
public decimal Price { get; set; }
public string Description { get; set; }
}
string file = "products.csv";
File.WriteAllLines(file, new[] {
"Id,Name,Category,Price,Description",
"101,Trail Bottle,Outdoor,24.90,\"Leakproof bottle for hiking. Fits in a side pocket.\"",
"102,Desk Lamp,Home,39.50,\"Warm light for late work. Adjustable arm and stable base.\"",
"103,City Backpack,Travel,79.00,\"Compact backpack for laptops. Padded straps and a rain cover.\""
});
var source = new CsvSource<Product>(file);
var chunk = new ChunkTransformation<Product> {
TextSelector = product => $"{product.Name}. Category: {product.Category}. Price: {product.Price.ToString(CultureInfo.InvariantCulture)}. {product.Description}",
MaxTokensPerChunk = 12,
OverlapTokens = 2
};
var dest = new MemoryDestination<Chunk<Product>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"Id:{row.Source.Id} Index:{row.Index} Text:{row.Text}");
//Outputs
//Id:101 Index:0 Text:Trail Bottle. Category: Outdoor. Price: 24.
//Id:101 Index:1 Text:24.90. Leakproof bottle for hiking. Fits in
//Id:101 Index:2 Text: Fits in a side pocket.
//Id:102 Index:0 Text:Desk Lamp. Category: Home. Price: 39.
//Id:102 Index:1 Text:39.50. Warm light for late work. Adjustable arm
//Id:102 Index:2 Text: Adjustable arm and stable base.
//Id:103 Index:0 Text:City Backpack. Category: Travel. Price: 79.
//Id:103 Index:1 Text:79.00. Compact backpack for laptops. Padded straps
//Id:103 Index:2 Text:added straps and a rain cover.Chunk with an attribute
[ChunkText] replaces TextSelector. The attribute belongs on one readable string.
public class CatalogItem
{
public int Id { get; set; }
public string Name { get; set; }
[ChunkText]
public string Description { get; set; }
}
var source = new MemorySource<CatalogItem>();
source.DataAsList.Add(new CatalogItem { Id = 201, Name = "Wool Socks", Description = "Warm socks for winter walks. Reinforced heel and breathable wool." });
source.DataAsList.Add(new CatalogItem { Id = 202, Name = "Coffee Grinder", Description = "Compact grinder for fresh coffee. Stepless settings and a clear lid." });
source.DataAsList.Add(new CatalogItem { Id = 203, Name = "Notebook Set", Description = "Three lined notebooks for project notes. Soft covers and numbered pages." });
//No TextSelector needed - the attribute defines the text property
var chunk = new ChunkTransformation<CatalogItem> {
MaxTokensPerChunk = 8,
OverlapTokens = 1
};
var dest = new MemoryDestination<Chunk<CatalogItem>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"Id:{row.Source.Id} Name:{row.Source.Name} Index:{row.Index} Text:{row.Text}");
//Outputs
//Id:201 Name:Wool Socks Index:0 Text:Warm socks for winter walks. Reinforced
//Id:201 Name:Wool Socks Index:1 Text:forced heel and breathable wool.
//Id:202 Name:Coffee Grinder Index:0 Text:Compact grinder for fresh coffee. Stepless
//Id:202 Name:Coffee Grinder Index:1 Text:pless settings and a clear lid.
//Id:203 Name:Notebook Set Index:0 Text:Three lined notebooks for project notes. Soft
//Id:203 Name:Notebook Set Index:1 Text: Soft covers and numbered pages.Chunk a dynamic object
For ExpandoObject, the non-generic ChunkTransformation reads the property Text.
var source = new MemorySource();
AddProduct(source, 301, "Running Shoes", "Sport", 89.00m, "Light shoes for daily runs. Cushioned sole and breathable mesh.");
AddProduct(source, 302, "Cast Iron Pan", "Kitchen", 45.00m, "Seasoned pan for searing and baking. Holds heat and cleans easily.");
AddProduct(source, 303, "Travel Adapter", "Electronics", 18.50m, "Compact adapter for international trips. Two USB ports and a foldable plug.");
//Reads property "Text"
var chunk = new ChunkTransformation {
MaxTokensPerChunk = 10,
OverlapTokens = 0
};
var dest = new MemoryDestination<Chunk<ExpandoObject>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (dynamic result in dest.Data)
Console.WriteLine($"Index:{result.Index} Text:{result.Text}");
//Outputs
//Index:0 Text:Running Shoes. Category: Sport. Price:
//Index:1 Text:89.00. Light shoes for daily runs.
//Index:2 Text: Cushioned sole and breathable mesh.
//Index:0 Text:Cast Iron Pan. Category: Kitchen. Price:
//Index:1 Text: 45.00. Seasoned pan for se
//Index:2 Text:aring and baking. Holds heat and cleans easily.
//Index:0 Text:Travel Adapter. Category: Electronics. Price:
//Index:1 Text:18.50. Compact adapter for international trips.
//Index:2 Text: Two USB ports and a foldable plug.
void AddProduct(MemorySource source, int id, string name, string category, decimal price, string description)
{
dynamic row = new ExpandoObject();
row.Id = id;
row.Name = name;
row.Text = $"{name}. Category: {category}. Price: {price.ToString(CultureInfo.InvariantCulture)}. {description}";
source.DataAsList.Add(row);
}Map each chunk to your own type
CreateOutput builds the downstream row. Use ChunkTransformation<TInput, TOutput> when that row is a different type than Chunk<TInput>.
public class Product
{
public int Id { get; set; }
public string Name { get; set; }
public string Category { get; set; }
public decimal Price { get; set; }
public string Description { get; set; }
}
public class ProductChunk
{
public int ProductId { get; set; }
public string Name { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; }
}
var source = new MemorySource<Product>();
source.DataAsList.Add(new Product { Id = 401, Name = "Wool Blanket", Category = "Home", Price = 59.00m, Description = "Soft blanket for cold evenings and weekend travel." });
source.DataAsList.Add(new Product { Id = 402, Name = "Chef Knife", Category = "Kitchen", Price = 64.00m, Description = "Balanced knife for vegetables, bread, and daily prep." });
source.DataAsList.Add(new Product { Id = 403, Name = "Floor Fan", Category = "Home", Price = 42.00m, Description = "Quiet fan for warm offices with three speed settings." });
var chunk = new ChunkTransformation<Product, ProductChunk> {
TextSelector = product => $"{product.Name}. {product.Description}",
CreateOutput = chunk => new ProductChunk {
ProductId = chunk.Source.Id,
Name = chunk.Source.Name,
ChunkIndex = chunk.Index,
ChunkText = chunk.Text
},
MaxTokensPerChunk = 8,
OverlapTokens = 0
};
var dest = new MemoryDestination<ProductChunk>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"ProductId:{row.ProductId} Name:{row.Name} ChunkIndex:{row.ChunkIndex} ChunkText:{row.ChunkText}");
//Outputs
//ProductId:401 Name:Wool Blanket ChunkIndex:0 ChunkText:Wool Blanket. Soft blanket for cold
//ProductId:401 Name:Wool Blanket ChunkIndex:1 ChunkText: evenings and weekend travel.
//ProductId:402 Name:Chef Knife ChunkIndex:0 ChunkText:Chef Knife. Balanced knife for vegetables,
//ProductId:402 Name:Chef Knife ChunkIndex:1 ChunkText: bread, and daily prep.
//ProductId:403 Name:Floor Fan ChunkIndex:0 ChunkText:Floor Fan. Quiet fan for warm offices
//ProductId:403 Name:Floor Fan ChunkIndex:1 ChunkText: with three speed settings.Chunk a document with an attribute
[ChunkDocument] replaces DocumentSelector. The attribute belongs on one readable IngestionDocument.
public class Manual
{
public string Name { get; set; }
[ChunkDocument]
public IngestionDocument Document { get; set; }
}
var source = new MemorySource<Manual>();
source.DataAsList.Add(CreateManual("Notebook", "Lined pages for meeting notes and project planning."));
source.DataAsList.Add(CreateManual("Pencil Set", "Soft pencils for sketches, lists, and margin notes."));
source.DataAsList.Add(CreateManual("Ruler", "Clear ruler for notebooks, maps, and desk work."));
//No DocumentSelector needed - the attribute defines the document property
var chunk = new ChunkTransformation<Manual> {
MaxTokensPerChunk = 20,
OverlapTokens = 0
};
var dest = new MemoryDestination<Chunk<Manual>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"Name:{row.Source.Name} Index:{row.Index} Text:{row.Text}");
//Outputs
//Name:Notebook Index:0 Text:Lined pages for meeting notes and project planning.
//Name:Pencil Set Index:0 Text:Soft pencils for sketches, lists, and margin notes.
//Name:Ruler Index:0 Text:Clear ruler for notebooks, maps, and desk work.
Manual CreateManual(string name, string description)
{
var document = new IngestionDocument(name);
var section = new IngestionDocumentSection(name);
section.Elements.Add(new IngestionDocumentParagraph(description));
document.Sections.Add(section);
return new Manual { Name = name, Document = document };
}Chunk an existing document
DocumentSelector passes an IngestionDocument to the chunker unchanged.
public class ProductDocument
{
public int Id { get; set; }
public string Name { get; set; }
public decimal Price { get; set; }
public IngestionDocument Document { get; set; }
}
var source = new MemorySource<ProductDocument>();
source.DataAsList.Add(CreateDocument(501, "Hiking Poles", 49.00m, "Adjustable poles for steep trails and long day hikes."));
source.DataAsList.Add(CreateDocument(502, "Tea Kettle", 32.00m, "Steel kettle for camping and a quick cup at home."));
source.DataAsList.Add(CreateDocument(503, "Desk Organizer", 27.00m, "Small organizer for cables, pens, and daily notes."));
//The existing document is passed to the chunker unchanged
var chunk = new ChunkTransformation<ProductDocument> {
DocumentSelector = product => product.Document,
MaxTokensPerChunk = 8,
OverlapTokens = 1
};
var dest = new MemoryDestination<Chunk<ProductDocument>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"Id:{row.Source.Id} Name:{row.Source.Name} Index:{row.Index} Text:{row.Text}");
//Outputs
//Id:501 Name:Hiking Poles Index:0 Text:Hiking Poles costs 49.
//Id:501 Name:Hiking Poles Index:1 Text:.00. Adjustable poles for steep trails
//Id:501 Name:Hiking Poles Index:2 Text: trails and long day hikes.
//Id:502 Name:Tea Kettle Index:0 Text:Tea Kettle costs 32.00
//Id:502 Name:Tea Kettle Index:1 Text:00. Steel kettle for camping and a
//Id:502 Name:Tea Kettle Index:2 Text: a quick cup at home.
//Id:503 Name:Desk Organizer Index:0 Text:Desk Organizer costs 27.00.
//Id:503 Name:Desk Organizer Index:1 Text:. Small organizer for cables, pens,
//Id:503 Name:Desk Organizer Index:2 Text:, and daily notes.
ProductDocument CreateDocument(int id, string name, decimal price, string description)
{
var document = new IngestionDocument($"product-{id}");
var section = new IngestionDocumentSection(name);
section.Elements.Add(new IngestionDocumentParagraph($"{name} costs {price.ToString(CultureInfo.InvariantCulture)}. {description}"));
document.Sections.Add(section);
return new ProductDocument { Id = id, Name = name, Price = price, Document = document };
}Keep each section in one chunk
Assign Chunker to replace the default token splitter. SectionChunker keeps each document section together. The limits belong on IngestionChunkerOptions.
public class ProductDocument
{
public int Id { get; set; }
public string Name { get; set; }
public decimal Price { get; set; }
public IngestionDocument Document { get; set; }
}
var source = new MemorySource<ProductDocument>();
source.DataAsList.Add(CreateDocument(601, "Notebook", 12.00m, "Lined pages for meeting notes and project planning."));
source.DataAsList.Add(CreateDocument(602, "Pencil Set", 8.50m, "Soft pencils for sketches, lists, and margin notes."));
source.DataAsList.Add(CreateDocument(603, "Ruler", 4.20m, "Clear ruler for notebooks, maps, and desk work."));
var tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");
//SectionChunker keeps each document section together
var chunk = new ChunkTransformation<ProductDocument> {
DocumentSelector = product => product.Document,
Tokenizer = tokenizer,
Chunker = new SectionChunker(new IngestionChunkerOptions(tokenizer) {
MaxTokensPerChunk = 200,
OverlapTokens = 0
})
};
var dest = new MemoryDestination<Chunk<ProductDocument>>();
source.LinkTo(chunk);
chunk.LinkTo(dest);
Network.Execute(source);
foreach (var row in dest.Data)
Console.WriteLine($"Id:{row.Source.Id} Section:{row.Source.Name} Text:{row.Text}");
//Outputs
//Id:601 Section:Notebook Text:Notebook costs 12.00. Lined pages for meeting notes and project planning.
//Id:602 Section:Pencil Set Text:Pencil Set costs 8.50. Soft pencils for sketches, lists, and margin notes.
//Id:603 Section:Ruler Text:Ruler costs 4.20. Clear ruler for notebooks, maps, and desk work.
ProductDocument CreateDocument(int id, string name, decimal price, string description)
{
var document = new IngestionDocument($"product-{id}");
var section = new IngestionDocumentSection(name);
section.Elements.Add(new IngestionDocumentParagraph($"{name} costs {price.ToString(CultureInfo.InvariantCulture)}. {description}"));
document.Sections.Add(section);
return new ProductDocument { Id = id, Name = name, Price = price, Document = document };
}