From c493dc1a2785e14e9c630c383c9fbd88dde81329 Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Sat, 7 Mar 2026 12:17:16 -0800 Subject: [PATCH] feat: add TestBox tests and docs for batch processing (#1911) Replace RocketUnit tests with TestBox BDD specs for findEach() and findInBatches(). Add shared test infrastructure (populate.cfm, model CFCs). Co-Authored-By: Claude Opus 4.6 --- CHANGELOG.md | 7 + docs/src/SUMMARY.md | 1 + .../batch-processing.md | 134 +++++++++++++++ tests/_assets/models/Author.cfc | 9 + tests/_assets/models/Post.cfc | 8 + tests/populate.cfm | 75 +++++++++ tests/specs/models/BatchProcessingSpec.cfc | 158 ++++++++++++++++++ 7 files changed, 392 insertions(+) create mode 100644 docs/src/database-interaction-through-models/batch-processing.md create mode 100644 tests/_assets/models/Author.cfc create mode 100644 tests/_assets/models/Post.cfc create mode 100644 tests/specs/models/BatchProcessingSpec.cfc diff --git a/CHANGELOG.md b/CHANGELOG.md index fb96224ebd..eda2a6e7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,13 @@ All historical references to "CFWheels" in this changelog have been preserved fo ---- +## [Unreleased] + +### Added +- Batch processing with `findEach()` and `findInBatches()` for memory-efficient record iteration + +---- + # [3.0.0](https://github.com/wheels-dev/wheels/releases/tag/v3.0.0) => 2026-01-10 diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 2496690e4c..2b7bfc3332 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -177,6 +177,7 @@ * [Transactions](database-interaction-through-models/transactions.md) * [Dirty Records](database-interaction-through-models/dirty-records.md) * [Soft Delete](database-interaction-through-models/soft-delete.md) +* [Batch Processing](database-interaction-through-models/batch-processing.md) * [Automatic Time Stamps](database-interaction-through-models/automatic-time-stamps.md) * [Database Migrations](database-interaction-through-models/database-migrations/README.md) * [Migrations in Production](database-interaction-through-models/database-migrations/migrations-in-production.md) diff --git a/docs/src/database-interaction-through-models/batch-processing.md b/docs/src/database-interaction-through-models/batch-processing.md new file mode 100644 index 0000000000..6474f7cdb4 --- /dev/null +++ b/docs/src/database-interaction-through-models/batch-processing.md @@ -0,0 +1,134 @@ +# Batch Processing + +When working with large datasets, loading all records into memory at once can cause performance problems or even crash your application. Wheels provides two batch processing methods that iterate through records in manageable chunks: `findEach()` and `findInBatches()`. + +## findEach() + +Processes records **one at a time** via a callback function. Internally, records are fetched in batches (default 1000) but your callback receives individual model objects. + +```cfm +model("User").findEach( + order = "id", + callback = function(user) { + user.sendReminderEmail(); + } +); +``` + +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `callback` | function | *required* | Closure receiving each record as a model object (or struct) | +| `batchSize` | numeric | 1000 | Number of records fetched per internal query | +| `where` | string | `""` | SQL WHERE clause to filter records | +| `order` | string | primary key ASC | Sort order (defaults to primary key for consistent pagination) | +| `include` | string | `""` | Associated models to include | +| `select` | string | `""` | Columns to select | +| `returnAs` | string | `"object"` | Set to `"struct"` to receive structs instead of objects | +| `includeSoftDeletes` | boolean | `false` | Include soft-deleted records | + +### Examples + +**Filter with a WHERE clause:** + +```cfm +model("Order").findEach( + where = "status = 'pending'", + batchSize = 500, + callback = function(order) { + order.update(status = "processing"); + } +); +``` + +**With associated models:** + +```cfm +model("User").findEach( + include = "profile", + callback = function(user) { + writeOutput(user.profile.bio); + } +); +``` + +## findInBatches() + +Processes records **in groups**. Your callback receives the entire batch as a query result set (or array of objects/structs). + +```cfm +model("User").findInBatches( + order = "id", + batchSize = 500, + callback = function(users) { + // `users` is a query with up to 500 rows + processBatch(users); + } +); +``` + +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `callback` | function | *required* | Closure receiving each batch | +| `batchSize` | numeric | 500 | Number of records per batch | +| `where` | string | `""` | SQL WHERE clause to filter records | +| `order` | string | primary key ASC | Sort order | +| `include` | string | `""` | Associated models to include | +| `select` | string | `""` | Columns to select | +| `returnAs` | string | `"query"` | Set to `"objects"` or `"structs"` for arrays | +| `includeSoftDeletes` | boolean | `false` | Include soft-deleted records | + +### Examples + +**Export data in chunks:** + +```cfm +model("Transaction").findInBatches( + where = "year(createdAt) = 2025", + order = "createdAt", + batchSize = 1000, + callback = function(batch) { + writeToCSV(batch); + } +); +``` + +**Receive objects instead of a query:** + +```cfm +model("Product").findInBatches( + batchSize = 100, + returnAs = "objects", + callback = function(products) { + for (var product in products) { + product.recalculatePrice(); + product.save(); + } + } +); +``` + +## Using with Scopes + +Both methods work with query scopes when called through a scope chain: + +```cfm +model("User").active().findEach( + batchSize = 500, + callback = function(user) { + user.sendNewsletter(); + } +); +``` + +## Choosing Between findEach and findInBatches + +| Use Case | Method | +|----------|--------| +| Need to process or update individual records | `findEach()` | +| Bulk operations on groups of records (CSV export, API calls) | `findInBatches()` | +| Need model callbacks (save, update) on each record | `findEach()` | +| Need to pass batches to external services | `findInBatches()` | diff --git a/tests/_assets/models/Author.cfc b/tests/_assets/models/Author.cfc new file mode 100644 index 0000000000..ba7bf6a27d --- /dev/null +++ b/tests/_assets/models/Author.cfc @@ -0,0 +1,9 @@ +component extends="Model" { + + function config() { + table("c_o_r_e_authors"); + hasMany("posts"); + hasOne("profile"); + } + +} diff --git a/tests/_assets/models/Post.cfc b/tests/_assets/models/Post.cfc new file mode 100644 index 0000000000..4ec82a4682 --- /dev/null +++ b/tests/_assets/models/Post.cfc @@ -0,0 +1,8 @@ +component extends="Model" { + + function config() { + table("c_o_r_e_posts"); + belongsTo("author"); + } + +} diff --git a/tests/populate.cfm b/tests/populate.cfm index e69de29bb2..d6c8df6f19 100644 --- a/tests/populate.cfm +++ b/tests/populate.cfm @@ -0,0 +1,75 @@ + + +// Get database info +local.db_info = $dbinfo(datasource = application.wheels.dataSourceName, type = "version"); +local.db = LCase(Replace(local.db_info.database_productname, " ", "", "all")); + +// Set DB-specific types +local.identityColumnType = "int NOT NULL IDENTITY"; +if (local.db IS "mysql" or local.db IS "mariadb") { + local.identityColumnType = "int NOT NULL AUTO_INCREMENT"; +} else if (local.db IS "postgresql") { + local.identityColumnType = "SERIAL NOT NULL"; +} +local.storageEngine = (local.db IS "mysql" or local.db IS "mariadb") ? "ENGINE=InnoDB" : ""; + + + + + DROP TABLE IF EXISTS c_o_r_e_posts + + + + DROP TABLE IF EXISTS c_o_r_e_authors + + + + + +CREATE TABLE c_o_r_e_authors ( + id #local.identityColumnType#, + firstname varchar(100) NOT NULL, + lastname varchar(100) NOT NULL, + PRIMARY KEY(id) +) #local.storageEngine# + + + +CREATE TABLE c_o_r_e_posts ( + id #local.identityColumnType#, + authorid int NULL, + title varchar(250) NOT NULL, + body text NOT NULL, + createdat datetime NOT NULL, + updatedat datetime NOT NULL, + deletedat datetime NULL, + views int DEFAULT 0 NOT NULL, + averagerating float NULL, + status varchar(20) DEFAULT 'draft' NOT NULL, + PRIMARY KEY(id) +) #local.storageEngine# + + + + +model("author").create(firstName = "Per", lastName = "Djurner"); +model("author").create(firstName = "Tony", lastName = "Petruzzi"); +model("author").create(firstName = "Chris", lastName = "Peters"); +model("author").create(firstName = "Peter", lastName = "Amiri"); +model("author").create(firstName = "James", lastName = "Gibson"); +model("author").create(firstName = "Raul", lastName = "Riera"); +model("author").create(firstName = "Andy", lastName = "Bellenie"); +model("author").create(firstName = "Adam", lastName = "Chapman"); +model("author").create(firstName = "Tom", lastName = "King"); +model("author").create(firstName = "David", lastName = "Belanger"); + +// Create posts with various statuses +local.per = model("author").findOne(where = "firstName = 'Per'"); +local.per.createPost(title = "First post", body = "Body 1", views = 5, status = "published"); +local.per.createPost(title = "Second post", body = "Body 2", views = 5, status = "published"); +local.per.createPost(title = "Third post", body = "Body 3", views = 0, averageRating = "3.2", status = "archived"); + +local.tony = model("author").findOne(where = "firstName = 'Tony'"); +local.tony.createPost(title = "Fourth post", body = "Body 4", views = 3, averageRating = "3.6", status = "draft"); +local.tony.createPost(title = "Fifth post", body = "Body 5", views = 2, averageRating = "3.6", status = "draft"); + diff --git a/tests/specs/models/BatchProcessingSpec.cfc b/tests/specs/models/BatchProcessingSpec.cfc new file mode 100644 index 0000000000..e4166d1b33 --- /dev/null +++ b/tests/specs/models/BatchProcessingSpec.cfc @@ -0,0 +1,158 @@ +component extends="wheels.WheelsTest" { + + function run() { + + describe("findEach()", () => { + + it("iterates all records", () => { + var result = {count = 0}; + model("author").findEach( + order = "id", + callback = function(record) { + result.count++; + } + ); + var total = model("author").count(); + expect(result.count).toBe(total); + }) + + it("respects custom batchSize", () => { + var result = {count = 0}; + model("author").findEach( + order = "id", + batchSize = 3, + callback = function(record) { + result.count++; + } + ); + expect(result.count).toBe(model("author").count()); + }) + + it("passes model objects to the callback", () => { + var result = {receivedObject = false}; + model("author").findEach( + order = "id", + batchSize = 2, + callback = function(record) { + if (IsObject(record) AND StructKeyExists(record, "firstName")) { + result.receivedObject = true; + } + } + ); + expect(result.receivedObject).toBeTrue(); + }) + + it("filters with a where clause", () => { + var result = {count = 0}; + model("author").findEach( + where = "lastName = 'Djurner'", + order = "id", + callback = function(record) { + result.count++; + } + ); + expect(result.count).toBe(1); + }) + + it("handles no matching records", () => { + var result = {count = 0}; + model("author").findEach( + where = "lastName = 'NonExistent'", + order = "id", + callback = function(record) { + result.count++; + } + ); + expect(result.count).toBe(0); + }) + + }) + + describe("findInBatches()", () => { + + it("processes all records across batches", () => { + var result = {totalRecords = 0}; + model("author").findInBatches( + order = "id", + batchSize = 3, + callback = function(records) { + result.totalRecords += records.recordcount; + } + ); + expect(result.totalRecords).toBe(model("author").count()); + }) + + it("creates the correct number of batches", () => { + var result = {batchCount = 0}; + var authorCount = model("author").count(); + model("author").findInBatches( + order = "id", + batchSize = 3, + callback = function(records) { + result.batchCount++; + } + ); + expect(result.batchCount).toBe(Ceiling(authorCount / 3)); + }) + + it("returns query objects by default", () => { + var result = {receivedQuery = false}; + model("author").findInBatches( + order = "id", + batchSize = 5, + callback = function(records) { + if (IsQuery(records)) { + result.receivedQuery = true; + } + } + ); + expect(result.receivedQuery).toBeTrue(); + }) + + it("produces correct batch sizes", () => { + var result = {sizes = []}; + model("author").findInBatches( + order = "id", + batchSize = 4, + callback = function(records) { + ArrayAppend(result.sizes, records.recordcount); + } + ); + expect(result.sizes[1]).toBe(4); + expect(result.sizes[2]).toBe(4); + var authorCount = model("author").count(); + var remainder = authorCount MOD 4; + if (remainder GT 0) { + expect(result.sizes[ArrayLen(result.sizes)]).toBe(remainder); + } + }) + + it("filters with a where clause", () => { + var result = {totalRecords = 0}; + model("author").findInBatches( + where = "lastName = 'Djurner'", + order = "id", + batchSize = 10, + callback = function(records) { + result.totalRecords += records.recordcount; + } + ); + expect(result.totalRecords).toBe(1); + }) + + it("handles no matching records", () => { + var result = {batchCount = 0}; + model("author").findInBatches( + where = "lastName = 'NonExistent'", + order = "id", + callback = function(records) { + result.batchCount++; + } + ); + expect(result.batchCount).toBe(0); + }) + + }) + + } +}