diff --git a/package.json b/package.json index 0e6c2a5e..70593f99 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:watch": "vitest" }, "dependencies": { + "@apify/actor-memory-expression": "^1.0.1", "@apify/input_schema": "^3.29.2", "@crawlee/core": "4.0.0-beta.145", "@crawlee/fs-storage": "4.0.0-beta.145", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c528b04..22a13533 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: .: dependencies: + '@apify/actor-memory-expression': + specifier: ^1.0.1 + version: 1.0.1 '@apify/input_schema': specifier: ^3.29.2 version: 3.29.2(ajv@8.20.0) @@ -93,12 +96,20 @@ importers: packages: + '@apify/actor-memory-expression@1.0.1': + resolution: {integrity: sha512-8ki7KeFG8p+2KiX3y29ta0giWGtxHOKKTy1pTpJ3P/ZlLhYs+hM7H03e6BiShNGPzb8iR7PfvYiT1SxIXRsmPg==} + engines: {node: '>=22'} + '@apify/consts@2.57.2': resolution: {integrity: sha512-iCAltDfZqEZazrMFoGIp3EEkvd7GTlgs9y74QZ+JTz+zH3GqBvOD2DfW4WYsqI1rfKlLyo5z3S7wipWpkSNNJQ==} '@apify/consts@2.58.1': resolution: {integrity: sha512-pKnoPincN2eqiSANmX2u19FihJATX3f1ucMhxss7t2epUmB/UhSjGM7G4immjC7Q/CXrNmbETBbOGmR0/EH4/Q==} + '@apify/consts@3.0.1': + resolution: {integrity: sha512-ZPmfdb4qt143R+6/jFYVCQDoHGInl24Ntz8ltEAMW7tFHSdiXnQoZtDgydZYT06vw6gOtPa/jEXgSneq8FM+uA==} + engines: {node: '>=22'} + '@apify/datastructures@2.0.6': resolution: {integrity: sha512-hI/Q5cCjXXI/g4OPwX+/xuk+JsOtXRhKNXl2wnq/nKmdrji3nYCc9vGsAEc1iXVbs16xQMZy5uRhoY3XTABovA==} @@ -119,6 +130,10 @@ packages: '@apify/log@2.5.54': resolution: {integrity: sha512-EZRHrKCWnet0+6WaTx14Mpjxrn/PRudmRk6/N1dCTz+jtHN+Pbdvw5FFtIrCUngPhm4+bdI+kKVYUBmvTwrjvg==} + '@apify/log@3.0.1': + resolution: {integrity: sha512-lZbSxqW78S5TqEtDrE9o71MtqUnpQJ96FYaEMk2RtiAPH+/ZEAf76DUaFKP5my9KxozohyQLmmg1isbySXXu0A==} + engines: {node: '>=22'} + '@apify/ps-tree@1.2.0': resolution: {integrity: sha512-VHIswI7rD/R4bToeIDuJ9WJXt+qr5SdhfoZ9RzdjmCs9mgy7l0P4RugQEUCcU+WB4sfImbd4CKwzXcn0uYx1yw==} engines: {node: '>= 0.10'} @@ -133,6 +148,10 @@ packages: '@apify/utilities@2.35.7': resolution: {integrity: sha512-UUJkBCCWpOdTPpGenGtkNnnUbwmgANwxJDZoDPfa27+V23V8UfurE/lGA0LCGDH6DWYYJkQoEAtehxYJHR0U9g==} + '@babel/runtime@7.29.7': + resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} + engines: {node: '>=6.9.0'} + '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} @@ -1149,6 +1168,9 @@ packages: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + complex.js@2.4.3: + resolution: {integrity: sha512-UrQVSUur14tNX6tiP4y8T4w4FeJAX3bi2cIv0pu/DTLFNxoq7z2Yh83Vfzztj6Px3X/lubqQ9IrPp7Bpn6p4MQ==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -1206,6 +1228,9 @@ packages: supports-color: optional: true + decimal.js@10.6.0: + resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} @@ -1321,6 +1346,9 @@ packages: escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escape-latex@1.2.0: + resolution: {integrity: sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -1473,6 +1501,9 @@ packages: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} + fraction.js@5.3.4: + resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + fresh@2.0.0: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} @@ -1614,6 +1645,9 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + javascript-natural-sort@0.7.1: + resolution: {integrity: sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==} + js-yaml@4.3.1: resolution: {integrity: sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==} hasBin: true @@ -1676,6 +1710,11 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + mathjs@15.2.0: + resolution: {integrity: sha512-UAQzSVob9rNLdGpqcFMYmSu9dkuLYy7Lr2hBEQS5SHQdknA9VppJz3cy2KkpMzTODunad6V6cNv+5kOLsePLow==} + engines: {node: '>= 18'} + hasBin: true + media-typer@1.1.1: resolution: {integrity: sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==} engines: {node: '>= 0.8'} @@ -1909,6 +1948,9 @@ packages: resolution: {integrity: sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==} engines: {node: '>=11.0.0'} + seedrandom@3.0.5: + resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -2046,6 +2088,9 @@ packages: through@2.3.8: resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + tiny-emitter@2.1.0: + resolution: {integrity: sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -2116,6 +2161,10 @@ packages: resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==} engines: {node: '>= 18'} + typed-function@4.2.2: + resolution: {integrity: sha512-VwaXim9Gp1bngi/q3do8hgttYn2uC3MoT/gfuMWylnj1IeZBUAyPddHZlo1K05BDoj8DYPpMdiHqH1dDYdJf2A==} + engines: {node: '>= 18'} + typescript-eslint@8.67.0: resolution: {integrity: sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2289,10 +2338,18 @@ packages: snapshots: + '@apify/actor-memory-expression@1.0.1': + dependencies: + '@apify/consts': 3.0.1 + '@apify/log': 3.0.1 + mathjs: 15.2.0 + '@apify/consts@2.57.2': {} '@apify/consts@2.58.1': {} + '@apify/consts@3.0.1': {} + '@apify/datastructures@2.0.6': {} '@apify/input_schema@3.29.2(ajv@8.20.0)': @@ -2325,6 +2382,11 @@ snapshots: '@apify/consts': 2.58.1 ansi-colors: 4.1.3 + '@apify/log@3.0.1': + dependencies: + '@apify/consts': 3.0.1 + ansi-colors: 4.1.3 + '@apify/ps-tree@1.2.0': dependencies: event-stream: 3.3.4 @@ -2341,6 +2403,8 @@ snapshots: '@apify/consts': 2.57.2 '@apify/log': 2.5.50 + '@babel/runtime@7.29.7': {} + '@balena/dockerignore@1.0.2': {} '@borewit/text-codec@0.2.2': {} @@ -3251,6 +3315,8 @@ snapshots: dependencies: delayed-stream: 1.0.0 + complex.js@2.4.3: {} + concat-map@0.0.1: {} content-disposition@1.1.0: {} @@ -3295,6 +3361,8 @@ snapshots: dependencies: ms: 2.1.3 + decimal.js@10.6.0: {} + deep-eql@5.0.2: {} deep-is@0.1.4: {} @@ -3457,6 +3525,8 @@ snapshots: escape-html@1.0.3: {} + escape-latex@1.2.0: {} + escape-string-regexp@4.0.0: {} escodegen@2.1.0: @@ -3660,6 +3730,8 @@ snapshots: forwarded@0.2.0: {} + fraction.js@5.3.4: {} + fresh@2.0.0: {} from@0.1.7: {} @@ -3796,6 +3868,8 @@ snapshots: isexe@2.0.0: {} + javascript-natural-sort@0.7.1: {} + js-yaml@4.3.1: dependencies: argparse: 2.0.1 @@ -3843,6 +3917,18 @@ snapshots: math-intrinsics@1.1.0: {} + mathjs@15.2.0: + dependencies: + '@babel/runtime': 7.29.7 + complex.js: 2.4.3 + decimal.js: 10.6.0 + escape-latex: 1.2.0 + fraction.js: 5.3.4 + javascript-natural-sort: 0.7.1 + seedrandom: 3.0.5 + tiny-emitter: 2.1.0 + typed-function: 4.2.2 + media-typer@1.1.1: {} merge-descriptors@2.0.0: {} @@ -4109,6 +4195,8 @@ snapshots: sax@1.6.1: {} + seedrandom@3.0.5: {} + semver@7.8.5: {} send@1.2.1: @@ -4298,6 +4386,8 @@ snapshots: through@2.3.8: {} + tiny-emitter@2.1.0: {} + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -4357,6 +4447,8 @@ snapshots: media-typer: 1.1.1 mime-types: 3.0.2 + typed-function@4.2.2: {} + typescript-eslint@8.67.0(eslint@9.39.5)(typescript@5.9.3): dependencies: '@typescript-eslint/eslint-plugin': 8.67.0(@typescript-eslint/parser@8.67.0(eslint@9.39.5)(typescript@5.9.3))(eslint@9.39.5)(typescript@5.9.3) diff --git a/requirements/actor-driver.md b/requirements/actor-driver.md index a688894c..ca781647 100644 --- a/requirements/actor-driver.md +++ b/requirements/actor-driver.md @@ -183,6 +183,15 @@ start`, ...) is refused by name, naming both the `CMD` fix and how to clear debu - Run details and the run log are kept in internal records that persist across runtime restarts (`storage.md`). +## Run memory + +- Run memory follows the platform's `.actor/actor.json` rules (`defaultMemoryMbytes`, `minMemoryMbytes`, + `maxMemoryMbytes`). Differences: + - A `defaultMemoryMbytes` that fails to evaluate, and a memory changed by the min/max bounds, are + noted in the run log. + - `defaultMemoryMbytes` is not capped to a plan's maximum memory. + - A memory field its schema forbids fails the build. + ## Resource limits - Every run's container has a hard memory limit and a hard CPU limit. Memory is the run's `memoryMbytes`; @@ -191,7 +200,6 @@ start`, ...) is refused by name, naming both the `CMD` fix and how to clear debu - Limits are applied exactly as requested, even when they exceed the host's own capacity. Such a run is warned about in its own log, naming the requested and the host figures; the limits still apply. When the host's capacity cannot be determined, no warning is produced. -- Disk is not limited. `diskMbytes` is reported but never enforced. ## Run resource telemetry diff --git a/requirements/unsupported.md b/requirements/unsupported.md index 10296783..258a7d6b 100644 --- a/requirements/unsupported.md +++ b/requirements/unsupported.md @@ -26,7 +26,6 @@ real account, not in the runtime. - Billing: a run's charges and costs are reported, never invoiced or paid out - Encrypted secret input fields - Actor-level default run options -- Dynamic and bounded memory from `.actor/actor.json` - Extra named default storages (`storageIds`) - Limited-permission Actors and per-run scoped tokens - Automatic run and build retention @@ -102,6 +101,7 @@ real account, not in the runtime. - Memory steps and bounds (128 MB - 32 GB, powers of two); a run asking for anything else is warned about in its log and started with it anyway +- Run disk limit (`diskMbytes`) - Record, item and input size limits - Concurrent run, rate and per-account quotas - Process, file-descriptor and shared-memory limits diff --git a/sample_actor_playwright/.actor/actor.json b/sample_actor_playwright/.actor/actor.json index ad03ad67..3ab5fc8f 100644 --- a/sample_actor_playwright/.actor/actor.json +++ b/sample_actor_playwright/.actor/actor.json @@ -6,6 +6,7 @@ "description": "Crawls up to maxRequestsPerCrawl same-hostname pages from startUrls with a headful Chrome driven by Playwright, pushing one dataset item per page.", "version": "0.0", "buildTag": "latest", + "defaultMemoryMbytes": 4096, "meta": { "templateId": "ts-crawlee-playwright-chrome" }, diff --git a/sample_actor_playwright_py/.actor/actor.json b/sample_actor_playwright_py/.actor/actor.json index a43052d6..eb070b89 100644 --- a/sample_actor_playwright_py/.actor/actor.json +++ b/sample_actor_playwright_py/.actor/actor.json @@ -6,6 +6,7 @@ "description": "Crawls up to max_requests_per_crawl same-hostname pages from start_urls with a headful Chromium driven by Playwright (Crawlee for Python), pushing one dataset item per page.", "version": "0.0", "buildTag": "latest", + "defaultMemoryMbytes": 4096, "meta": { "templateId": "python-crawlee-playwright" }, diff --git a/sample_actor_py/.actor/actor.json b/sample_actor_py/.actor/actor.json index d0383c7b..63e091a9 100644 --- a/sample_actor_py/.actor/actor.json +++ b/sample_actor_py/.actor/actor.json @@ -5,6 +5,7 @@ "description": "Crawls up to maxPages same-domain pages starting from startUrl, over a request queue, pushing one dataset item per page.", "version": "0.0", "buildTag": "latest", + "defaultMemoryMbytes": "get(input, 'maxPages', 1) * 128", "meta": { "templateId": "python-empty" }, diff --git a/sample_actor_py/.actor/input_schema.json b/sample_actor_py/.actor/input_schema.json index 27d20263..383e001d 100644 --- a/sample_actor_py/.actor/input_schema.json +++ b/sample_actor_py/.actor/input_schema.json @@ -14,7 +14,7 @@ "title": "Max pages", "type": "integer", "description": "The maximum number of pages to crawl, following same-domain links from the start URL.", - "default": 2, + "default": 10, "minimum": 1 } }, diff --git a/sample_actor_ts/.actor/actor.json b/sample_actor_ts/.actor/actor.json index e37155a5..163ec04c 100644 --- a/sample_actor_ts/.actor/actor.json +++ b/sample_actor_ts/.actor/actor.json @@ -6,6 +6,7 @@ "description": "Crawls up to maxPages same-hostname pages starting from startUrl, over a request queue, pushing one dataset item per page.", "version": "0.0", "buildTag": "latest", + "defaultMemoryMbytes": "get(input, 'maxPages', 1) * 128", "meta": { "templateId": "ts-empty" }, diff --git a/sample_actor_ts/.actor/input_schema.json b/sample_actor_ts/.actor/input_schema.json index 9ac2d1c1..e3bc26d8 100644 --- a/sample_actor_ts/.actor/input_schema.json +++ b/sample_actor_ts/.actor/input_schema.json @@ -14,7 +14,7 @@ "title": "Max pages", "type": "integer", "description": "The maximum number of pages to crawl, following same-hostname links from the start URL.", - "default": 2, + "default": 10, "minimum": 1 } }, diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index 4ba84e5e..c559a133 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -75,6 +75,14 @@ valid: Field input.maxPages must be >= 1`); no run is created and no container s - Two local differences: Apify Proxy groups are not checked (any `apifyProxyGroups` selection is accepted), and encrypted secret input fields are not supported. +## Run memory + +With no `-m`/`--memory`, a run gets `defaultMemoryMbytes` from `.actor/actor.json` - a number or a +memory expression over the input, evaluated as the platform does - or 1024 MB. Any memory, explicit +or not, is then clamped to the Actor's `minMemoryMbytes` / `maxMemoryMbytes`, and the run log says so. +An expression that fails falls back to 1024 MB with a warning in the run log. The fields come from the +build, so changing them needs an `apify push`. + ## Iterate without rebuilding (dev folder) After that first `apify push`, the runtime registers the pushed directory as the Actor's **dev diff --git a/src/services/actor-memory.ts b/src/services/actor-memory.ts new file mode 100644 index 00000000..aeedf847 --- /dev/null +++ b/src/services/actor-memory.ts @@ -0,0 +1,103 @@ +/** + * The memory fields of `.actor/actor.json` - `defaultMemoryMbytes`, `minMemoryMbytes`, `maxMemoryMbytes` - + * read at build time and applied at run start, in the platform's order: the caller's `memory`, else the + * Actor's default expression, then clamped to the Actor's bounds. + */ +import { calculateRunDynamicMemory, type ActorRunOptions } from '@apify/actor-memory-expression'; + +import type { ActorMemorySettings, SourceFile } from '../storage/entities.js'; +import { parseActorJson } from './actor-source-files.js'; + +/** The bounds `@apify/json_schemas`' `actor.schema.json` puts on each numeric memory field. */ +const MIN_MEMORY_MBYTES = 128; +const MAX_MEMORY_MBYTES = 32_768; + +export type ActorMemoryResolution = + { outcome: 'resolved'; settings: ActorMemorySettings | undefined } | { outcome: 'failure'; message: string }; + +function isMemoryMbytes(value: unknown): value is number { + return Number.isInteger(value) && (value as number) >= MIN_MEMORY_MBYTES && (value as number) <= MAX_MEMORY_MBYTES; +} + +function invalidField(field: string, expected: string): ActorMemoryResolution { + return { outcome: 'failure', message: `.actor/actor.json has invalid format: "${field}" must be ${expected}.` }; +} + +export function resolveActorMemorySettings(sourceFiles: SourceFile[]): ActorMemoryResolution { + const actorJson = parseActorJson(sourceFiles); + if (actorJson.outcome === 'unparseable') return { outcome: 'failure', message: actorJson.message }; + if (actorJson.outcome === 'absent') return { outcome: 'resolved', settings: undefined }; + const specification = actorJson.specification; + if (typeof specification !== 'object' || specification === null) + return { outcome: 'resolved', settings: undefined }; + + const { defaultMemoryMbytes, minMemoryMbytes, maxMemoryMbytes } = specification as Record; + const bounds = `an integer from ${MIN_MEMORY_MBYTES} to ${MAX_MEMORY_MBYTES}`; + const settings: ActorMemorySettings = {}; + if (minMemoryMbytes !== undefined) { + if (!isMemoryMbytes(minMemoryMbytes)) return invalidField('minMemoryMbytes', bounds); + settings.minMemoryMbytes = minMemoryMbytes; + } + if (maxMemoryMbytes !== undefined) { + if (!isMemoryMbytes(maxMemoryMbytes)) return invalidField('maxMemoryMbytes', bounds); + settings.maxMemoryMbytes = maxMemoryMbytes; + } + if (defaultMemoryMbytes !== undefined) { + if (typeof defaultMemoryMbytes !== 'string' && !isMemoryMbytes(defaultMemoryMbytes)) { + return invalidField('defaultMemoryMbytes', `a memory expression string or ${bounds}`); + } + settings.defaultMemoryMbytes = defaultMemoryMbytes; + } + return { outcome: 'resolved', settings: Object.keys(settings).length > 0 ? settings : undefined }; +} + +export interface RunMemoryContext { + /** The caller's `memory`; takes precedence over `defaultMemoryMbytes` but is still clamped. */ + requestedMemoryMbytes: number | undefined; + /** Used when neither the caller nor the Actor sets a memory, and when the expression fails. */ + fallbackMemoryMbytes: number; + /** What the expression sees as `runOptions`; its `memoryMbytes` is the fallback, as on the platform. */ + runOptions: Omit; + input: { body: Buffer; contentType: string } | undefined; +} + +export interface RunMemory { + memoryMbytes: number; + logLines: string[]; +} + +export async function resolveRunMemory( + settings: ActorMemorySettings | undefined, + context: RunMemoryContext, +): Promise { + const logLines: string[] = []; + let memoryMbytes = context.requestedMemoryMbytes ?? context.fallbackMemoryMbytes; + + const defaultMemoryMbytes = settings?.defaultMemoryMbytes; + if (context.requestedMemoryMbytes === undefined && defaultMemoryMbytes !== undefined) { + try { + const body = context.input?.body.toString('utf8'); + memoryMbytes = await calculateRunDynamicMemory(String(defaultMemoryMbytes), { + runOptions: { ...context.runOptions, memoryMbytes: context.fallbackMemoryMbytes }, + input: body ? (JSON.parse(body) as Record) : {}, + }); + } catch (error) { + // The platform falls back the same way, but silently - locally the Actor's author is the one reading. + logLines.push( + `Warning: could not evaluate "defaultMemoryMbytes" from .actor/actor.json ` + + `(${(error as Error).message}); running with ${context.fallbackMemoryMbytes} MB instead.`, + ); + } + } + + // `min` wins over `max` when the two contradict, as in the platform's own clamp. + const unclamped = memoryMbytes; + const { minMemoryMbytes, maxMemoryMbytes } = settings ?? {}; + if (minMemoryMbytes !== undefined && memoryMbytes < minMemoryMbytes) memoryMbytes = minMemoryMbytes; + else if (maxMemoryMbytes !== undefined && memoryMbytes > maxMemoryMbytes) memoryMbytes = maxMemoryMbytes; + if (memoryMbytes !== unclamped) { + const bound = memoryMbytes === minMemoryMbytes ? 'minMemoryMbytes' : 'maxMemoryMbytes'; + logLines.push(`Memory of ${unclamped} MB adjusted to ${memoryMbytes} MB by "${bound}" in .actor/actor.json.`); + } + return { memoryMbytes, logLines }; +} diff --git a/src/services/builds.ts b/src/services/builds.ts index 98325974..b9ea246b 100644 --- a/src/services/builds.ts +++ b/src/services/builds.ts @@ -10,6 +10,7 @@ import { sourceFileToText } from './actor-source-files.js'; import { qualifyDockerfileImageReferences } from './dockerfile-image-refs.js'; import { resolveDockerfileLocation } from './dockerfile-location.js'; import { resolveInputSchemaLocation } from './input-schema-location.js'; +import { resolveActorMemorySettings } from './actor-memory.js'; import { appendLog, appendRuntimeLog, flushLog, markLogTerminal } from './logs.js'; import { isTerminalJobStatus, transitionJobStatus } from './job-status.js'; @@ -227,6 +228,13 @@ export async function runBuildInBackground( for (const line of inputSchemaResolution.logLines) appendRuntimeLog(record.id, line); const inputSchema = inputSchemaResolution.outcome === 'resolved' ? inputSchemaResolution.schema : undefined; + const memoryResolution = resolveActorMemorySettings(version.sourceFiles); + if (memoryResolution.outcome === 'failure') { + await failBuild(record.id, memoryResolution.message); + return; + } + const memorySettings = memoryResolution.settings; + const sourceFiles: SourceFile[] = qualifyDockerfileImages( dockerfileResolution.outcome === 'default' ? [...version.sourceFiles, dockerfileResolution.extraSourceFile] @@ -283,6 +291,7 @@ export async function runBuildInBackground( ? { imageWorkingDirectory: outcome.imageWorkingDirectory } : {}), ...(inputSchema !== undefined ? { inputSchema } : {}), + ...(memorySettings !== undefined ? { memorySettings } : {}), }); if (succeeded?.status !== 'SUCCEEDED') { await updateActor(actor.id, (current) => { diff --git a/src/services/runs.ts b/src/services/runs.ts index 42362526..23368c06 100644 --- a/src/services/runs.ts +++ b/src/services/runs.ts @@ -22,6 +22,7 @@ import { dedicatedCpusFor, platformIncompatibleMemoryWarning } from '../resource import { CONTAINER_EVENTS_WS_BASE_URL } from '../config.js'; import { formatRuntimeLogLines } from '../runtime-log.js'; import { getRunTelemetry } from './events-channel.js'; +import { resolveRunMemory } from './actor-memory.js'; import { initialChargedEventCounts, resolveRunPricingInfo } from './pricing.js'; import { actorStartChargeMessage, @@ -190,7 +191,19 @@ export async function startRun( await store.setValue('INPUT', options.input.body, { contentType: options.input.contentType }); } - const memoryMbytes = options.memoryMbytes ?? DEFAULT_MEMORY_MBYTES; + const buildTag = options.build ?? DEFAULT_BUILD_TAG; + const timeoutSecs = options.timeoutSecs ?? DEFAULT_TIMEOUT_SECS; + const { memoryMbytes, logLines: memoryLogLines } = await resolveRunMemory(build.memorySettings, { + requestedMemoryMbytes: options.memoryMbytes, + fallbackMemoryMbytes: DEFAULT_MEMORY_MBYTES, + runOptions: { + build: buildTag, + timeoutSecs, + diskMbytes: DEFAULT_MEMORY_MBYTES * DISK_MBYTES_PER_MEMORY_MBYTE, + ...(options.maxTotalChargeUsd !== undefined ? { maxTotalChargeUsd: options.maxTotalChargeUsd } : {}), + }, + input: options.input, + }); // Resolved once: a later pricing change must not reprice a run that already exists. const pricingInfo = resolveRunPricingInfo(actor.pricingInfos); const chargedEventCounts = initialChargedEventCounts(pricingInfo, memoryMbytes); @@ -206,9 +219,9 @@ export async function startRun( defaultKeyValueStoreId: keyValueStore.id, defaultRequestQueueId: requestQueue.id, options: { - build: options.build ?? DEFAULT_BUILD_TAG, + build: buildTag, memoryMbytes, - timeoutSecs: options.timeoutSecs ?? DEFAULT_TIMEOUT_SECS, + timeoutSecs, diskMbytes: memoryMbytes * DISK_MBYTES_PER_MEMORY_MBYTE, ...(options.maxTotalChargeUsd !== undefined ? { maxTotalChargeUsd: options.maxTotalChargeUsd } : {}), }, @@ -231,7 +244,8 @@ export async function startRun( await runs.set(record.id, record); registerDefaultDatasetForCharging(record); - // Both lines are about what the caller asked for, so they are written before the run does anything. + // These lines are about what the caller asked for, so they are written before the run does anything. + for (const line of memoryLogLines) appendRuntimeLog(record.id, line); const memoryWarning = platformIncompatibleMemoryWarning(memoryMbytes); if (memoryWarning) appendRuntimeLog(record.id, memoryWarning); const startCharge = actorStartChargeMessage(record); diff --git a/src/storage/entities.ts b/src/storage/entities.ts index 538fa0fe..61d1d8f9 100644 --- a/src/storage/entities.ts +++ b/src/storage/entities.ts @@ -183,10 +183,20 @@ export interface BuildRecord { * schema of the build it resolved, never another tag's more recently pushed one. Absent when the * Actor declares none - such a run takes its input exactly as the caller sent it. */ inputSchema?: InputSchema; + /** The memory fields this build's `.actor/actor.json` declared, build-specific like `inputSchema`. + * Absent when it declares none. */ + memorySettings?: ActorMemorySettings; exitCode?: number; statusMessage?: string; } +/** `.actor/actor.json`'s run-memory fields; `defaultMemoryMbytes` may be a memory expression. */ +export interface ActorMemorySettings { + defaultMemoryMbytes?: string | number; + minMemoryMbytes?: number; + maxMemoryMbytes?: number; +} + export interface RunRecord { id: string; userId: string; diff --git a/test/e2e/actor-dev-loop.test.ts b/test/e2e/actor-dev-loop.test.ts index 2df095e8..025a7370 100644 --- a/test/e2e/actor-dev-loop.test.ts +++ b/test/e2e/actor-dev-loop.test.ts @@ -151,14 +151,18 @@ describe('full Actor dev loop via apify-cli (requires Docker)', () => { }); expect(JSON.parse(storedInput) as Record).toEqual({ startUrl: 'https://crawlee.dev/', - maxPages: 2, + maxPages: 10, }); const infoOutput = apify(['datasets', 'info', call.storage.defaultDatasetId, '--json'], { cwd: actorDir, env, }); - expect((JSON.parse(infoOutput) as DatasetInfoResult).itemCount).toBe(2); + // A 10-page crawl of crawlee.dev can drain its queue a page short of `maxPages` (seen: 9), so the + // count only has to rule out the old default of 2; the INPUT check above pins the defaults. + const { itemCount } = JSON.parse(infoOutput) as DatasetInfoResult; + expect(itemCount).toBeGreaterThan(2); + expect(itemCount).toBeLessThanOrEqual(10); // `maxPages` has `minimum: 1` - the run is refused before any container starts, and the CLI // surfaces the runtime's validation message. diff --git a/test/e2e/pay-per-event.test.ts b/test/e2e/pay-per-event.test.ts index 5bdc3fa1..93c89dcc 100644 --- a/test/e2e/pay-per-event.test.ts +++ b/test/e2e/pay-per-event.test.ts @@ -155,7 +155,7 @@ describe('pay-per-event pricing and the run cost estimate via apify-cli (require expect(call.run.status).toBe('SUCCEEDED'); const run = getRun(call.run.id, env); expect(run.pricingInfo?.pricingModel).toBe('PAY_PER_EVENT'); - // 1024 MB -> the start event once; the SDK charged the page and crawl events over the endpoint. + // 3 pages -> 512 MB from the sample's defaultMemoryMbytes -> the start event once; the SDK charged the page and crawl events over the endpoint. expect(run.chargedEventCounts).toEqual({ [PAGE_EVENT]: 3, [FINISHED_EVENT]: 1, diff --git a/test/integration/actor-memory.test.ts b/test/integration/actor-memory.test.ts new file mode 100644 index 00000000..62348a3f --- /dev/null +++ b/test/integration/actor-memory.test.ts @@ -0,0 +1,77 @@ +/** + * `.actor/actor.json`'s memory fields, from a real build over pushed source to the run it starts + * (`actor-driver.md`'s "Run memory"). The driver is stubbed at `docker build`; nothing here needs Docker. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { fixedBuildOutcomeDriver, startTestServer, type TestServerHandle } from './helpers/test-server.js'; +import { getRegistries } from '../../src/storage/registries.js'; +import { runBuildInBackground } from '../../src/services/builds.js'; +import type { SourceFile } from '../../src/storage/entities.js'; + +describe('run memory from .actor/actor.json (via real apify-client)', () => { + let server: TestServerHandle; + + beforeEach(async () => { + server = await startTestServer(); + }); + + afterEach(async () => { + await server.close(); + }); + + async function build(actorId: string, userId: string, buildId: string, actorJson: unknown): Promise { + const { actors, builds } = getRegistries(); + const sourceFiles: SourceFile[] = [ + { name: 'main.js', format: 'TEXT', content: 'console.log(1)' }, + { name: '.actor/actor.json', format: 'TEXT', content: JSON.stringify(actorJson) }, + ]; + await builds.set(buildId, { + id: buildId, + userId, + actorId, + versionNumber: '0.0', + buildNumber: '0.0.1', + tag: 'latest', + status: 'READY', + startedAt: new Date().toISOString(), + }); + await runBuildInBackground( + fixedBuildOutcomeDriver({ imageId: 'built-image:latest' }), + (await actors.get(actorId))!, + { versionNumber: '0.0', buildTag: 'latest', sourceType: 'SOURCE_FILES', sourceFiles }, + (await builds.get(buildId))!, + { tag: 'latest', useCache: true }, + ); + } + + it('sizes runs by defaultMemoryMbytes and the min/max bounds, and fails a build that declares them wrong', async () => { + const actor = await server.client.actors().create({ name: 'memory-actor' }); + const { builds } = getRegistries(); + + await build(actor.id, actor.userId, 'badMemoryBuild123', { minMemoryMbytes: 'lots' }); + expect((await builds.get('badMemoryBuild123'))?.status).toBe('FAILED'); + + await build(actor.id, actor.userId, 'goodMemoryBuild12', { + defaultMemoryMbytes: "get(input, 'pages', 1) * 1024", + minMemoryMbytes: 2048, + maxMemoryMbytes: 8192, + }); + expect((await builds.get('goodMemoryBuild12'))?.status).toBe('SUCCEEDED'); + + const fromExpression = await server.client.actor(actor.id).start({ pages: 4 }); + expect(fromExpression.options.memoryMbytes).toBe(4096); + expect(fromExpression.options.diskMbytes).toBe(8192); + + const raisedToMin = await server.client.actor(actor.id).start({}); + expect(raisedToMin.options.memoryMbytes).toBe(2048); + + const cappedAtMax = await server.client.actor(actor.id).start({}, { memory: 32768 }); + expect(cappedAtMax.options.memoryMbytes).toBe(8192); + + // The stub driver cannot run anything; let each run fail before the server goes away. + for (const run of [fromExpression, raisedToMin, cappedAtMax]) { + await server.client.run(run.id).waitForFinish({ waitSecs: 10 }); + } + }); +}); diff --git a/test/unit/actor-memory.test.ts b/test/unit/actor-memory.test.ts new file mode 100644 index 00000000..67a5d1c4 --- /dev/null +++ b/test/unit/actor-memory.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; + +import { + resolveActorMemorySettings, + resolveRunMemory, + type RunMemoryContext, +} from '../../src/services/actor-memory.js'; +import type { SourceFile } from '../../src/storage/entities.js'; + +function actorJson(spec: Record): SourceFile[] { + return [{ name: '.actor/actor.json', format: 'TEXT', content: JSON.stringify(spec) }]; +} + +function context(overrides: Partial = {}): RunMemoryContext { + return { + requestedMemoryMbytes: undefined, + fallbackMemoryMbytes: 1024, + runOptions: { build: 'latest', timeoutSecs: 300, diskMbytes: 2048 }, + input: undefined, + ...overrides, + }; +} + +function jsonInput(value: unknown): RunMemoryContext['input'] { + return { body: Buffer.from(JSON.stringify(value)), contentType: 'application/json' }; +} + +describe('resolveActorMemorySettings', () => { + it('reads the three memory fields, and nothing when none is declared', () => { + expect( + resolveActorMemorySettings( + actorJson({ defaultMemoryMbytes: '{{input.n}} * 512', minMemoryMbytes: 512, maxMemoryMbytes: 8192 }), + ), + ).toEqual({ + outcome: 'resolved', + settings: { defaultMemoryMbytes: '{{input.n}} * 512', minMemoryMbytes: 512, maxMemoryMbytes: 8192 }, + }); + expect(resolveActorMemorySettings(actorJson({ name: 'x' }))).toEqual({ + outcome: 'resolved', + settings: undefined, + }); + expect(resolveActorMemorySettings([])).toEqual({ outcome: 'resolved', settings: undefined }); + }); + + it.each([ + [{ minMemoryMbytes: 64 }, 'minMemoryMbytes'], + [{ maxMemoryMbytes: '4096' }, 'maxMemoryMbytes'], + [{ defaultMemoryMbytes: 1024.5 }, 'defaultMemoryMbytes'], + ])('fails on an out-of-schema value: %j', (spec, field) => { + const result = resolveActorMemorySettings(actorJson(spec)); + expect(result.outcome).toBe('failure'); + if (result.outcome === 'failure') expect(result.message).toContain(`"${field}"`); + }); +}); + +describe('resolveRunMemory', () => { + it('uses the fallback when neither the caller nor the Actor sets a memory', async () => { + expect(await resolveRunMemory(undefined, context())).toEqual({ memoryMbytes: 1024, logLines: [] }); + }); + + it('evaluates defaultMemoryMbytes against the input and run options, rounded to a power of two', async () => { + const settings = { defaultMemoryMbytes: "get(input, 'urls.length', 1) * 1000 + {{runOptions.timeoutSecs}}" }; + const result = await resolveRunMemory(settings, context({ input: jsonInput({ urls: ['a', 'b', 'c', 'd'] }) })); + expect(result.memoryMbytes).toBe(4096); + expect(await resolveRunMemory({ defaultMemoryMbytes: 2048 }, context())).toEqual({ + memoryMbytes: 2048, + logLines: [], + }); + }); + + it("lets the caller's memory win over defaultMemoryMbytes", async () => { + const result = await resolveRunMemory({ defaultMemoryMbytes: 4096 }, context({ requestedMemoryMbytes: 256 })); + expect(result.memoryMbytes).toBe(256); + }); + + it('falls back, with a warning, when the expression cannot be evaluated', async () => { + const result = await resolveRunMemory({ defaultMemoryMbytes: '{{nope}} * 2' }, context()); + expect(result.memoryMbytes).toBe(1024); + expect(result.logLines).toHaveLength(1); + expect(result.logLines[0]).toContain('could not evaluate "defaultMemoryMbytes"'); + }); + + it('clamps to minMemoryMbytes / maxMemoryMbytes, including an explicit request', async () => { + const bounds = { minMemoryMbytes: 4096, maxMemoryMbytes: 8192 }; + expect(await resolveRunMemory(bounds, context())).toEqual({ + memoryMbytes: 4096, + logLines: ['Memory of 1024 MB adjusted to 4096 MB by "minMemoryMbytes" in .actor/actor.json.'], + }); + expect((await resolveRunMemory(bounds, context({ requestedMemoryMbytes: 16384 }))).memoryMbytes).toBe(8192); + expect((await resolveRunMemory({ ...bounds, defaultMemoryMbytes: 128 }, context())).memoryMbytes).toBe(4096); + }); +});