From 13be19648c7ba3cdadf47ef459a8632e38b39c3b Mon Sep 17 00:00:00 2001 From: David Omrai Date: Mon, 20 Jul 2026 12:36:50 +0200 Subject: [PATCH 1/8] feat: update dataset id and limit/offset desc for clarity --- components/apify/apify.app.mjs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index 75ae9935a556f..cde2b55c0cb0d 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -62,7 +62,7 @@ export default { datasetId: { type: "string", label: "Dataset ID", - description: "The ID of the dataset to retrieve items within", + description: "Select a dataset, or enter a Dataset ID or `username~dataset-name`", async options({ page }) { const { items } = await this.listDatasets({ offset: LIMIT * page, @@ -119,15 +119,15 @@ export default { limit: { type: "integer", label: "Limit", - description: "The maximum number of items to return", - default: LIMIT, + description: "The maximum number of items to return. Leave empty to return all items", + min: 1, optional: true, }, offset: { type: "integer", label: "Offset", - description: "The number records to skip before returning results", - default: 0, + description: "The number of records to skip before returning results. Leave empty to start from the first item", + min: 0, optional: true, }, }, From 8deb36dd45b18ef3c19b02f69c20b87542b82191 Mon Sep 17 00:00:00 2001 From: David Omrai Date: Mon, 20 Jul 2026 13:43:34 +0200 Subject: [PATCH 2/8] feat: bump version to 0.0.7 and refactor dataset item retrieval logic for improved clarity and efficiency --- .../get-dataset-items/get-dataset-items.mjs | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs index c705dd0d72ab1..36b242a67c9e2 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -5,7 +5,7 @@ export default { key: "apify-get-dataset-items", name: "Get Dataset Items", description: "Returns data stored in a dataset. [See the documentation](https://docs.apify.com/api/v2/dataset-items-get)", - version: "0.0.6", + version: "0.0.7", annotations: { destructiveHint: false, openWorldHint: true, @@ -52,32 +52,36 @@ export default { }, }, async run({ $ }) { - const params = { - limit: LIMIT, - offset: this.offset, - clean: this.clean, - fields: this.fields, - omit: this.omit, - }; + const { + datasetId, clean, fields, omit, limit, + } = this; + const offset = this.offset ?? 0; const results = []; - let total; + let currentOffset = offset; - do { + while (limit === undefined || results.length < limit) { + const pageSize = limit === undefined + ? LIMIT + : Math.min(LIMIT, limit - results.length); const { items } = await this.apify.listDatasetItems({ - datasetId: this.datasetId, - params, + datasetId, + params: { + offset: currentOffset, + limit: pageSize, + clean, + fields, + omit, + }, }); + if (!items?.length) { + break; + } results.push(...items); - if (results.length >= this.limit) { + currentOffset += items.length; + if (items.length < pageSize) { break; } - total = items?.length; - params.offset += LIMIT; - } while (total); - - if (results.length > this.limit) { - results.length = this.limit; } if (results.length > 0) { From 1b76cc9847f3a72df7a2b7568c8c942065edb81d Mon Sep 17 00:00:00 2001 From: David Omrai Date: Mon, 20 Jul 2026 13:55:14 +0200 Subject: [PATCH 3/8] test: add test for get dataset items action to validate pagination and parameter handling --- .../get-dataset-items.test.mjs | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 components/apify/actions/get-dataset-items/get-dataset-items.test.mjs diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs new file mode 100644 index 0000000000000..097d5b1ff428e --- /dev/null +++ b/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs @@ -0,0 +1,191 @@ +import getDatasetItems from "./get-dataset-items.mjs"; +import apify from "../../apify.app.mjs"; +import { LIMIT } from "../../common/constants.mjs"; + +// Hand-rolled spy: jest.fn is not available under the ESM vm-modules runner. +// Models the real Apify endpoint — honors the offset/limit paging params — so +// the pagination loop is exercised the same way it runs live. +function makeApify(dataset, { maxCalls = 100 } = {}) { + const calls = []; + return { + calls, + listDatasetItems({ + datasetId, params, + }) { + calls.push({ + datasetId, + params, + }); + if (calls.length > maxCalls) { + throw new Error(`listDatasetItems called ${calls.length} times — likely infinite loop`); + } + const { + offset = 0, limit, + } = params; + const end = limit === undefined + ? undefined + : offset + limit; + return Promise.resolve({ + items: dataset.slice(offset, end), + }); + }, + }; +} + +function makeContext(dataset, props, opts) { + const spy = makeApify(dataset, opts); + const exported = {}; + const ctx = { + ...getDatasetItems, + apify: spy, + datasetId: "ds_test", + clean: undefined, + fields: undefined, + omit: undefined, + offset: undefined, + limit: undefined, + ...props, + }; + const $ = { + export: (key, value) => { + exported[key] = value; + }, + }; + return { + ctx, + spy, + exported, + run: () => getDatasetItems.run.call(ctx, { + $, + }), + }; +} + +const dataset = (n) => Array.from({ + length: n, +}, (_, i) => ({ + i, +})); + +test("limit caps the number of returned items across a large dataset", async () => { + const { + run, spy, exported, + } = makeContext(dataset(150), { + limit: 2, + }); + const results = await run(); + expect(results).toHaveLength(2); + // Only the needed page is fetched, sized to the limit. + expect(spy.calls).toHaveLength(1); + expect(spy.calls[0].params.limit).toBe(2); + // Summary reports the count (plural form here). + expect(exported["$summary"]).toMatch(/retrieved 2 items/); +}); + +test("limit larger than the dataset returns everything and stops on the short page", async () => { + const { + run, spy, + } = makeContext(dataset(50), { + limit: 500, + }); + const results = await run(); + expect(results).toHaveLength(50); + expect(spy.calls).toHaveLength(1); +}); + +test("empty dataset returns [] with a single request and no summary", async () => { + const { + run, spy, exported, + } = makeContext(dataset(0), { + limit: undefined, + }); + const results = await run(); + expect(results).toEqual([]); + expect(spy.calls).toHaveLength(1); + expect(exported["$summary"]).toBeUndefined(); +}); + +test("clean, fields and omit are forwarded to listDatasetItems", async () => { + const { + run, spy, + } = makeContext(dataset(10), { + limit: 1, + clean: true, + fields: [ + "a", + ], + omit: [ + "b", + ], + }); + await run(); + expect(spy.calls[0].params).toMatchObject({ + clean: true, + fields: [ + "a", + ], + omit: [ + "b", + ], + }); +}); + +test("empty limit paginates and returns all items, stopping on a short page", async () => { + const { + run, spy, + } = makeContext(dataset(150), { + limit: undefined, + }); + const results = await run(); + expect(results).toHaveLength(150); + // 100 + 50 (short page terminates), no gratuitous extra fetch. + expect(spy.calls).toHaveLength(2); + expect(spy.calls[0].params.limit).toBe(LIMIT); +}); + +test("empty limit terminates on an empty page for an exact multiple (no infinite loop)", async () => { + const { + run, spy, + } = makeContext(dataset(200), { + limit: undefined, + }, { + maxCalls: 5, + }); + const results = await run(); + expect(results).toHaveLength(200); + // 100 + 100 + empty terminal fetch. + expect(spy.calls).toHaveLength(3); +}); + +test("offset is passed through to the first request", async () => { + const { + run, spy, + } = makeContext(dataset(150), { + offset: 25, + limit: 2, + }); + const results = await run(); + expect(results).toHaveLength(2); + expect(spy.calls[0].params.offset).toBe(25); +}); + +test("empty offset defaults to 0", async () => { + const { + run, spy, + } = makeContext(dataset(10), { + offset: undefined, + limit: 5, + }); + await run(); + expect(spy.calls[0].params.offset).toBe(0); +}); + +test("limit propDefinition enforces a minimum of 1 and has no default", () => { + expect(apify.propDefinitions.limit.min).toBe(1); + expect("default" in apify.propDefinitions.limit).toBe(false); +}); + +test("offset propDefinition enforces a minimum of 0 and has no default", () => { + expect(apify.propDefinitions.offset.min).toBe(0); + expect("default" in apify.propDefinitions.offset).toBe(false); +}); From 3993f98f188df0edc4e3089836ca2516235a87f5 Mon Sep 17 00:00:00 2001 From: David Omrai Date: Mon, 20 Jul 2026 14:17:46 +0200 Subject: [PATCH 4/8] fix: normalize dataset id format in get-dataset-items action --- .../get-dataset-items/get-dataset-items.mjs | 5 +++- .../get-dataset-items.test.mjs | 27 +++++++++++++++++++ components/apify/apify.app.mjs | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs index 36b242a67c9e2..4b90a077b7abf 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -53,8 +53,11 @@ export default { }, async run({ $ }) { const { - datasetId, clean, fields, omit, limit, + clean, fields, omit, limit, } = this; + // The Apify API path accepts `username~dataset-name`; accept the friendlier + // `username/dataset-name` form too by normalizing the separator to a tilde. + const datasetId = this.datasetId?.replace("/", "~"); const offset = this.offset ?? 0; const results = []; diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs index 097d5b1ff428e..f175974b81208 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs @@ -105,6 +105,33 @@ test("empty dataset returns [] with a single request and no summary", async () = expect(exported["$summary"]).toBeUndefined(); }); +test("username/dataset-name is normalized to the tilde form the API expects", async () => { + const { + run, spy, + } = makeContext(dataset(10), { + datasetId: "myuser/my-dataset", + limit: 1, + }); + await run(); + expect(spy.calls[0].datasetId).toBe("myuser~my-dataset"); +}); + +test("a raw dataset ID or tilde form is passed through unchanged", async () => { + const idCase = makeContext(dataset(10), { + datasetId: "WkzbQMuFYuamGv3YF", + limit: 1, + }); + await idCase.run(); + expect(idCase.spy.calls[0].datasetId).toBe("WkzbQMuFYuamGv3YF"); + + const tildeCase = makeContext(dataset(10), { + datasetId: "myuser~my-dataset", + limit: 1, + }); + await tildeCase.run(); + expect(tildeCase.spy.calls[0].datasetId).toBe("myuser~my-dataset"); +}); + test("clean, fields and omit are forwarded to listDatasetItems", async () => { const { run, spy, diff --git a/components/apify/apify.app.mjs b/components/apify/apify.app.mjs index cde2b55c0cb0d..4710296436724 100644 --- a/components/apify/apify.app.mjs +++ b/components/apify/apify.app.mjs @@ -62,7 +62,7 @@ export default { datasetId: { type: "string", label: "Dataset ID", - description: "Select a dataset, or enter a Dataset ID or `username~dataset-name`", + description: "Select a dataset, or enter a Dataset ID or `username/dataset-name`", async options({ page }) { const { items } = await this.listDatasets({ offset: LIMIT * page, From 02c34e2cbe80d792b667721aa01d30ad61f25a41 Mon Sep 17 00:00:00 2001 From: David Omrai Date: Tue, 28 Jul 2026 10:51:12 +0200 Subject: [PATCH 5/8] chore: bump version to 0.0.4 for get-dataset-items action --- .../apify_oauth/actions/get-dataset-items/get-dataset-items.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs index 1f977dd20e280..df83c1ed1bed5 100644 --- a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-get-dataset-items", - version: "0.0.2", + version: "0.0.4", name, description, type, From 11fbb039c52b348b9b94471f132f76734f21d478 Mon Sep 17 00:00:00 2001 From: David Omrai Date: Wed, 29 Jul 2026 19:10:39 +0200 Subject: [PATCH 6/8] chore: remove unecessary test file --- .../get-dataset-items.test.mjs | 218 ------------------ 1 file changed, 218 deletions(-) delete mode 100644 components/apify/actions/get-dataset-items/get-dataset-items.test.mjs diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs deleted file mode 100644 index f175974b81208..0000000000000 --- a/components/apify/actions/get-dataset-items/get-dataset-items.test.mjs +++ /dev/null @@ -1,218 +0,0 @@ -import getDatasetItems from "./get-dataset-items.mjs"; -import apify from "../../apify.app.mjs"; -import { LIMIT } from "../../common/constants.mjs"; - -// Hand-rolled spy: jest.fn is not available under the ESM vm-modules runner. -// Models the real Apify endpoint — honors the offset/limit paging params — so -// the pagination loop is exercised the same way it runs live. -function makeApify(dataset, { maxCalls = 100 } = {}) { - const calls = []; - return { - calls, - listDatasetItems({ - datasetId, params, - }) { - calls.push({ - datasetId, - params, - }); - if (calls.length > maxCalls) { - throw new Error(`listDatasetItems called ${calls.length} times — likely infinite loop`); - } - const { - offset = 0, limit, - } = params; - const end = limit === undefined - ? undefined - : offset + limit; - return Promise.resolve({ - items: dataset.slice(offset, end), - }); - }, - }; -} - -function makeContext(dataset, props, opts) { - const spy = makeApify(dataset, opts); - const exported = {}; - const ctx = { - ...getDatasetItems, - apify: spy, - datasetId: "ds_test", - clean: undefined, - fields: undefined, - omit: undefined, - offset: undefined, - limit: undefined, - ...props, - }; - const $ = { - export: (key, value) => { - exported[key] = value; - }, - }; - return { - ctx, - spy, - exported, - run: () => getDatasetItems.run.call(ctx, { - $, - }), - }; -} - -const dataset = (n) => Array.from({ - length: n, -}, (_, i) => ({ - i, -})); - -test("limit caps the number of returned items across a large dataset", async () => { - const { - run, spy, exported, - } = makeContext(dataset(150), { - limit: 2, - }); - const results = await run(); - expect(results).toHaveLength(2); - // Only the needed page is fetched, sized to the limit. - expect(spy.calls).toHaveLength(1); - expect(spy.calls[0].params.limit).toBe(2); - // Summary reports the count (plural form here). - expect(exported["$summary"]).toMatch(/retrieved 2 items/); -}); - -test("limit larger than the dataset returns everything and stops on the short page", async () => { - const { - run, spy, - } = makeContext(dataset(50), { - limit: 500, - }); - const results = await run(); - expect(results).toHaveLength(50); - expect(spy.calls).toHaveLength(1); -}); - -test("empty dataset returns [] with a single request and no summary", async () => { - const { - run, spy, exported, - } = makeContext(dataset(0), { - limit: undefined, - }); - const results = await run(); - expect(results).toEqual([]); - expect(spy.calls).toHaveLength(1); - expect(exported["$summary"]).toBeUndefined(); -}); - -test("username/dataset-name is normalized to the tilde form the API expects", async () => { - const { - run, spy, - } = makeContext(dataset(10), { - datasetId: "myuser/my-dataset", - limit: 1, - }); - await run(); - expect(spy.calls[0].datasetId).toBe("myuser~my-dataset"); -}); - -test("a raw dataset ID or tilde form is passed through unchanged", async () => { - const idCase = makeContext(dataset(10), { - datasetId: "WkzbQMuFYuamGv3YF", - limit: 1, - }); - await idCase.run(); - expect(idCase.spy.calls[0].datasetId).toBe("WkzbQMuFYuamGv3YF"); - - const tildeCase = makeContext(dataset(10), { - datasetId: "myuser~my-dataset", - limit: 1, - }); - await tildeCase.run(); - expect(tildeCase.spy.calls[0].datasetId).toBe("myuser~my-dataset"); -}); - -test("clean, fields and omit are forwarded to listDatasetItems", async () => { - const { - run, spy, - } = makeContext(dataset(10), { - limit: 1, - clean: true, - fields: [ - "a", - ], - omit: [ - "b", - ], - }); - await run(); - expect(spy.calls[0].params).toMatchObject({ - clean: true, - fields: [ - "a", - ], - omit: [ - "b", - ], - }); -}); - -test("empty limit paginates and returns all items, stopping on a short page", async () => { - const { - run, spy, - } = makeContext(dataset(150), { - limit: undefined, - }); - const results = await run(); - expect(results).toHaveLength(150); - // 100 + 50 (short page terminates), no gratuitous extra fetch. - expect(spy.calls).toHaveLength(2); - expect(spy.calls[0].params.limit).toBe(LIMIT); -}); - -test("empty limit terminates on an empty page for an exact multiple (no infinite loop)", async () => { - const { - run, spy, - } = makeContext(dataset(200), { - limit: undefined, - }, { - maxCalls: 5, - }); - const results = await run(); - expect(results).toHaveLength(200); - // 100 + 100 + empty terminal fetch. - expect(spy.calls).toHaveLength(3); -}); - -test("offset is passed through to the first request", async () => { - const { - run, spy, - } = makeContext(dataset(150), { - offset: 25, - limit: 2, - }); - const results = await run(); - expect(results).toHaveLength(2); - expect(spy.calls[0].params.offset).toBe(25); -}); - -test("empty offset defaults to 0", async () => { - const { - run, spy, - } = makeContext(dataset(10), { - offset: undefined, - limit: 5, - }); - await run(); - expect(spy.calls[0].params.offset).toBe(0); -}); - -test("limit propDefinition enforces a minimum of 1 and has no default", () => { - expect(apify.propDefinitions.limit.min).toBe(1); - expect("default" in apify.propDefinitions.limit).toBe(false); -}); - -test("offset propDefinition enforces a minimum of 0 and has no default", () => { - expect(apify.propDefinitions.offset.min).toBe(0); - expect("default" in apify.propDefinitions.offset).toBe(false); -}); From 3e524086520992128b0a3fb14c3b296f2aefe4bf Mon Sep 17 00:00:00 2001 From: David Omrai Date: Tue, 18 Aug 2026 14:31:26 +0200 Subject: [PATCH 7/8] chore: lower get-dataset-items version --- .../apify_oauth/actions/get-dataset-items/get-dataset-items.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs index df83c1ed1bed5..dc35879131a5a 100644 --- a/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify_oauth/actions/get-dataset-items/get-dataset-items.mjs @@ -11,7 +11,7 @@ const props = adjustPropDefinitions(others.props, app); export default { ...others, key: "apify_oauth-get-dataset-items", - version: "0.0.4", + version: "0.0.3", name, description, type, From 02e069ffddef5559506f4e4efc63b58739eca102 Mon Sep 17 00:00:00 2001 From: David Omrai Date: Tue, 18 Aug 2026 14:36:07 +0200 Subject: [PATCH 8/8] chore: remove unecesary comment --- .../apify/actions/get-dataset-items/get-dataset-items.mjs | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/apify/actions/get-dataset-items/get-dataset-items.mjs b/components/apify/actions/get-dataset-items/get-dataset-items.mjs index 4b90a077b7abf..cedd74e94a725 100644 --- a/components/apify/actions/get-dataset-items/get-dataset-items.mjs +++ b/components/apify/actions/get-dataset-items/get-dataset-items.mjs @@ -55,8 +55,6 @@ export default { const { clean, fields, omit, limit, } = this; - // The Apify API path accepts `username~dataset-name`; accept the friendlier - // `username/dataset-name` form too by normalizing the separator to a tilde. const datasetId = this.datasetId?.replace("/", "~"); const offset = this.offset ?? 0;