From a5b3509e73eb339d2893f09426fe276bf8a23c96 Mon Sep 17 00:00:00 2001 From: Juan Manuel Rodriguez Defago Date: Wed, 16 Sep 2026 11:03:21 -0300 Subject: [PATCH] fix: parse deployment manifests safely with native YAML Replace line-based manifest extraction with graph-ts YAML parsing to avoid indexing failures from CRLF line endings and malformed metadata. Preserve the original manifest and extract schema links, network, source kind, and minimum start block independently, leaving invalid fields unset with warnings. Guard YAML types and missing keys, validate unsigned block numbers and IPFS references, and preserve zero when calculating the minimum start block. Upgrade graph-ts to 0.38.2 and document the Graph Node 0.37.0 requirement. Validation: Arbitrum and L1 test-configuration codegen/builds, targeted lint, and diff checks passed. Matchstick was skipped as requested. The manifest handler and validation helper are identical to the network subgraph implementation. --- README.md | 8 +- package.json | 2 +- src/mappings/helpers/manifest.ts | 128 ++++++++++++++++++++++++ src/mappings/ipfs.ts | 166 +++++++++++++++++++------------ yarn.lock | 43 ++++---- 5 files changed, 255 insertions(+), 92 deletions(-) create mode 100644 src/mappings/helpers/manifest.ts diff --git a/README.md b/README.md index 64b1a41..854985f 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ Everytime a new release is merged into `master` there will be a new github relea # Deploying the subgraph +The deployment manifest IPFS handler uses the native YAML API from `graph-ts` 0.38.2 +and requires **Graph Node 0.37.0 or newer**. Malformed manifests retain their raw +content; derived fields that cannot be safely extracted remain unset and produce +warnings. Schema links support CIDv0 and CIDv1 in base32 or base58btc, optionally +with an IPFS file path. Unsupported links are skipped without creating a file data source. + The npm scripts are set up to deploy the subgraphs in one command. Mainnet is connected to a hook where it will be deployed automatically when the `master` branch is updated. Therefore, we never have to use npm scripts to directly deploy to `graph-network-mainnet`. @@ -91,4 +97,4 @@ the schema. Copyright © 2020 The Graph Foundation. -Licensed under the [MIT license](./LICENSE). \ No newline at end of file +Licensed under the [MIT license](./LICENSE). diff --git a/package.json b/package.json index f5a1cb9..c69394d 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "@graphprotocol/address-book": "^1.3.0", "@graphprotocol/contracts": "6.2.0", "@graphprotocol/graph-cli": "0.97.0", - "@graphprotocol/graph-ts": "0.36.0", + "@graphprotocol/graph-ts": "0.38.2", "@types/node": "^14.0.13", "@typescript-eslint/eslint-plugin": "^3.3.0", "@typescript-eslint/parser": "^3.3.0", diff --git a/src/mappings/helpers/manifest.ts b/src/mappings/helpers/manifest.ts new file mode 100644 index 0000000..0b35517 --- /dev/null +++ b/src/mappings/helpers/manifest.ts @@ -0,0 +1,128 @@ +import { BigInt, YAMLValue } from '@graphprotocol/graph-ts' + +// Avoid YAMLValue's [] accessor: it asserts when a key is absent. +export function yamlField(value: YAMLValue | null, key: string): YAMLValue | null { + if (value === null || !value.isObject()) return null + return value.toObject().get(YAMLValue.newString(key)) +} + +export function yamlString(value: YAMLValue | null): string | null { + if (value === null || !value.isString()) return null + let text = value.toString().trim() + return text.length > 0 ? text : null +} + +// Block numbers must be unsigned integers. YAML NUMBER also includes floats, +// and YAMLValue.toBigInt() does not protect against invalid numeric strings. +export function manifestStartBlock(value: YAMLValue): BigInt | null { + let text: string + if (value.isNumber()) { + text = value.toNumber() + } else if (value.isString()) { + text = value.toString().trim() + } else { + return null + } + if (text.length == 0 || text.length > 20) return null + for (let i = 0; i < text.length; i++) { + let code = text.charCodeAt(i) + if (code < 48 || code > 57) return null + } + if (text.length == 20 && text > '18446744073709551615') return null + return BigInt.fromString(text) +} + +// Decode the CID encodings normally used in published manifests. Unsupported +// encodings are skipped rather than passed to a host function that can abort. +function decodeCid(text: string): Uint8Array | null { + if (text.length == 0 || text.length > 128) return null + let base58 = text.startsWith('Qm') || text.startsWith('z') + if (base58) { + let encoded = text.startsWith('z') ? text.slice(1) : text + let alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + let bytes = new Array() + for (let i = 0; i < encoded.length; i++) { + let carry = alphabet.indexOf(encoded.charAt(i)) + if (carry < 0) return null + for (let j = 0; j < bytes.length; j++) { + carry += i32(bytes[j]) * 58 + bytes[j] = u8(carry & 255) + carry >>= 8 + } + while (carry > 0) { + bytes.push(u8(carry & 255)) + carry >>= 8 + } + } + for (let i = 0; i < encoded.length && encoded.charAt(i) == '1'; i++) bytes.push(0) + let result = new Uint8Array(bytes.length) + for (let i = 0; i < bytes.length; i++) result[i] = bytes[bytes.length - i - 1] + return result + } + if (!text.startsWith('b') && !text.startsWith('B')) return null + let alphabet = text.startsWith('b') ? 'abcdefghijklmnopqrstuvwxyz234567' : 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' + let result = new Uint8Array((text.length - 1) * 5 / 8) + let bits = 0 + let buffer = 0 + let offset = 0 + for (let i = 1; i < text.length; i++) { + let digit = alphabet.indexOf(text.charAt(i)) + if (digit < 0) return null + buffer = (buffer << 5) | digit + bits += 5 + if (bits >= 8) { + bits -= 8 + result[offset++] = u8(buffer >> bits) + buffer &= (1 << bits) - 1 + } + } + return bits < 5 && buffer == 0 ? result : null +} + +function validCid(text: string): bool { + let bytes = decodeCid(text) + if (bytes === null) return false + if (text.startsWith('Qm')) { + return text.length == 46 && bytes.length == 34 && bytes[0] == 0x12 && bytes[1] == 0x20 + } + // CIDv1 contains four unsigned varints: version, codec, hash code, digest + // length; followed by the digest. Graph Node supports digests up to 64 bytes. + let offset = 0 + for (let field = 0; field < 4; field++) { + let value: u64 = 0 + let terminated = false + for (let i = 0; i < 10 && offset < bytes.length; i++) { + let byte = bytes[offset++] + if (i == 9 && byte > 1) return false + value |= u64(byte & 127) << (i * 7) + if ((byte & 128) == 0) { + if (i > 0 && byte == 0) return false + terminated = true + break + } + } + if (!terminated) return false + if (field == 0 && value != 1) return false + if (field == 3) return value <= 64 && value == u64(bytes.length - offset) + } + return false +} + +export function manifestSchemaPath(value: YAMLValue | null): string | null { + // Published manifests use { '/': '/ipfs/CID' }; also accept string links. + if (value !== null && value.isObject()) value = yamlField(value, '/') + let path = yamlString(value) + if (path === null || path.length > 2048) return null + if (path.startsWith('/ipfs/')) path = path.slice(6) + else if (path.startsWith('ipfs://')) path = path.slice(7) + let segments = path.split('/') + if (!validCid(segments[0])) return null + for (let i = 1; i < segments.length; i++) { + if (segments[i].length == 0 || segments[i] == '.' || segments[i] == '..') return null + } + for (let i = 0; i < path.length; i++) { + let code = path.charCodeAt(i) + if (code <= 32 || code == 127 || path.charAt(i) == '?' || path.charAt(i) == '#' || path.charAt(i) == '%' || path.charAt(i) == '\\') return null + } + return path +} diff --git a/src/mappings/ipfs.ts b/src/mappings/ipfs.ts index b182440..30b5db8 100644 --- a/src/mappings/ipfs.ts +++ b/src/mappings/ipfs.ts @@ -1,4 +1,4 @@ -import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt } from '@graphprotocol/graph-ts' +import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt, yaml, YAMLValue } from '@graphprotocol/graph-ts' import { SubgraphMeta, SubgraphVersionMeta, @@ -10,6 +10,7 @@ import { SubgraphDeploymentSchema as SubgraphDeploymentSchemaTemplate } from '../types/templates' import { jsonToString } from './utils' +import { yamlField, yamlString, manifestStartBlock, manifestSchemaPath } from './helpers/manifest' export function handleSubgraphMetadata(content: Bytes): void { let id = dataSource.context().getString("id") @@ -80,78 +81,111 @@ export function handleSubgraphDeploymentSchema(content: Bytes): void { subgraphDeploymentSchema.save() } -export function handleSubgraphDeploymentManifest(content: Bytes): void { - // Shouldn't need ID since the handler isn't gonna be called more than once, given that it's only on deployment creation. - let subgraphDeploymentManifest = new SubgraphDeploymentManifest(dataSource.stringParam()) - if (content !== null) { - subgraphDeploymentManifest.manifest = content.toString() - - let manifest = subgraphDeploymentManifest.manifest! - // we take the right side of the split, since it's the one which will have the schema ipfs hash - let schemaSplitTry = manifest.split('schema:\n', 2) - if (schemaSplitTry.length == 2) { - let schemaSplit = schemaSplitTry[1] - - let schemaFileSplitTry = schemaSplit.split('/ipfs/', 2) - if (schemaFileSplitTry.length == 2) { - let schemaFileSplit = schemaFileSplitTry[1] +function manifestWarning(id: string, field: string): void { + log.warning('[MANIFEST PARSING FAIL] deployment: {}, invalid or unsupported {}', [id, field]) +} - let schemaIpfsHashTry = schemaFileSplit.split('\n', 2) - if (schemaIpfsHashTry.length == 2) { - let schemaIpfsHash = schemaIpfsHashTry[0] - let schemaId = subgraphDeploymentManifest.id.concat('-').concat(schemaIpfsHash) - subgraphDeploymentManifest.schema = schemaId - subgraphDeploymentManifest.schemaIpfsHash = schemaIpfsHash +function readManifestSchema(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + let path = manifestSchemaPath(yamlField(yamlField(root, 'schema'), 'file')) + if (path === null) { + manifestWarning(manifest.id, 'schema.file') + return + } + let schemaId = manifest.id.concat('-').concat(path) + manifest.schema = schemaId + manifest.schemaIpfsHash = path + let context = new DataSourceContext() + context.setString('id', schemaId) + SubgraphDeploymentSchemaTemplate.createWithContext(path, context) +} - let context = new DataSourceContext() - context.setString('id', schemaId) - SubgraphDeploymentSchemaTemplate.createWithContext(schemaIpfsHash, context) - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaIpfsHashTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaIpfsHashTry.length.toString()]) - } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaFileSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaFileSplitTry.length.toString()]) +function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + // Keep the first usable network, falling back to templates when necessary. + let sections = ['dataSources', 'templates'] + for (let section = 0; section < sections.length; section++) { + let entries = yamlField(root, sections[section]) + if (entries === null || !entries.isArray()) continue + let sources = entries.toArray() + for (let i = 0; i < sources.length; i++) { + let network = yamlString(yamlField(sources[i], 'network')) + if (network !== null && validManifestNetwork(network)) { + manifest.network = network + return } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaSplitTry.length.toString()]) } + } + manifestWarning(manifest.id, 'network') +} - // We get the first occurrence of `network` since subgraphs can only have data sources for the same network - let networkSplitTry = manifest.split('network: ', 2) - if (networkSplitTry.length == 2) { - let networkSplit = networkSplitTry[1] - let networkTry = networkSplit.split('\n', 2) - if (networkTry.length == 2) { - let network = networkTry[0] +function validManifestNetwork(network: string): bool { + if (network.length > 256) return false + for (let i = 0; i < network.length; i++) { + let code = network.charCodeAt(i) + if (code <= 32 || code == 127) return false + } + return true +} - subgraphDeploymentManifest.network = network - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkTry.length.toString()]) - } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkSplitTry.length.toString()]) - } - let substreamsSplitTry = manifest.split('- kind: substreams', 2) - subgraphDeploymentManifest.poweredBySubstreams = substreamsSplitTry.length > 1 +function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + let sources = yamlField(root, 'dataSources') + if (sources === null || !sources.isArray() || sources.toArray().length == 0) { + manifestWarning(manifest.id, 'dataSources') + return + } - // startBlock calculation - let templatesSplit = manifest.split("templates:") - let nonTemplateManifestSplit = templatesSplit[0] // we take the left as we want to remove the templates for the source checks. - let sourcesSplit = nonTemplateManifestSplit.split("source:") // We want to know how many source definitions we have - let startBlockSplit = nonTemplateManifestSplit.split("startBlock: ") // And how many startBlock definitions we have to know if we should set startBlock to 0 - - if (sourcesSplit.length > startBlockSplit.length) { - subgraphDeploymentManifest.startBlock = BigInt.fromI32(0) - } else { - // need to figure the minimum startBlock defined, we skip i = 0 as we know it's not gonna contain a start block num, since it's before the first appearance of "startBlock:" - let min = BigInt.fromI32(0) - for(let i = 1; i < startBlockSplit.length; i++) { - let numString = startBlockSplit[i].split("\n", 1)[0].toString() - let num = BigInt.fromString(numString) - min = min == BigInt.fromI32(0) ? num : min <= num ? min : num - } - subgraphDeploymentManifest.startBlock = min + let dataSources = sources.toArray() + let minimum: BigInt | null = null + let validStartBlocks = true + let validKinds = true + let poweredBySubstreams = false + // Only inspect actual dataSources. Templates, comments and context values + // must not affect the minimum start block or the deployment's source kind. + for (let i = 0; i < dataSources.length; i++) { + let dataSource = dataSources[i] + let kind = yamlString(yamlField(dataSource, 'kind')) + if (kind === null) validKinds = false + else if (kind == 'substreams') poweredBySubstreams = true + + let source = yamlField(dataSource, 'source') + if (source === null || !source.isObject()) { + validStartBlocks = false + continue + } + let startBlockValue = yamlField(source, 'startBlock') + // A missing startBlock defaults to zero. An explicit null or malformed + // value is unknown, so we cannot reliably report a minimum. + let startBlock = startBlockValue === null ? BigInt.fromI32(0) : manifestStartBlock(startBlockValue) + if (startBlock === null) { + validStartBlocks = false + } else if (minimum === null || startBlock < minimum) { + minimum = startBlock } } - subgraphDeploymentManifest.save() + + if (poweredBySubstreams || validKinds) manifest.poweredBySubstreams = poweredBySubstreams + else manifestWarning(manifest.id, 'dataSources.kind') + if (validStartBlocks && minimum !== null) manifest.startBlock = minimum + else manifestWarning(manifest.id, 'dataSources.source.startBlock') +} + +export function handleSubgraphDeploymentManifest(content: Bytes): void { + let manifest = new SubgraphDeploymentManifest(dataSource.stringParam()) + manifest.manifest = content.toString() + // Match the native parser's input limit; retain the raw manifest on failure. + if (content.length > 10000000) { + manifestWarning(manifest.id, 'manifest size') + manifest.save() + return + } + let parsed = yaml.try_fromBytes(content) + if (!parsed.isOk) { + manifestWarning(manifest.id, 'YAML') + } else if (!parsed.value.isObject()) { + manifestWarning(manifest.id, 'manifest root') + } else { + readManifestSchema(manifest, parsed.value) + readManifestNetwork(manifest, parsed.value) + readManifestDataSources(manifest, parsed.value) + } + manifest.save() } diff --git a/yarn.lock b/yarn.lock index 4b66b15..348ff92 100644 --- a/yarn.lock +++ b/yarn.lock @@ -912,12 +912,12 @@ web3-eth-abi "4.4.1" yaml "2.7.0" -"@graphprotocol/graph-ts@0.36.0": - version "0.36.0" - resolved "https://registry.yarnpkg.com/@graphprotocol/graph-ts/-/graph-ts-0.36.0.tgz#68ed937806bc7b2f8fe0ae2e15447255144f6a41" - integrity sha512-yJNQK5YZWEThuawSboQQ+U4Fb2C78KBjmaoeOK7Nn0CFoChmHc+woRvW3yj+IKVSPc7JNHt4JSUHxVDJfUZbTA== +"@graphprotocol/graph-ts@0.38.2": + version "0.38.2" + resolved "https://registry.yarnpkg.com/@graphprotocol/graph-ts/-/graph-ts-0.38.2.tgz#e99bdb6e0e50485084c442d7a02409082a836ff8" + integrity sha512-87KIFSFs2+Te+mnmb7Y+M57oqzlLy20cIyPIRbn9qJfpZFSZHTKtBLT6KQmcsK0YkoWis9Ur3c3M2c9mmaaEHQ== dependencies: - assemblyscript "0.19.10" + assemblyscript "0.27.31" "@graphprotocol/pino-sentry-simple@0.7.1": version "0.7.1" @@ -2198,14 +2198,6 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -assemblyscript@0.19.10: - version "0.19.10" - resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.19.10.tgz#7ede6d99c797a219beb4fa4614c3eab9e6343c8e" - integrity sha512-HavcUBXB3mBTRGJcpvaQjmnmaqKHBGREjSPNsIvnAk2f9dj78y4BkMaSSdvBQYWcDDzsHQjyUC8stICFkD1Odg== - dependencies: - binaryen "101.0.0-nightly.20210723" - long "^4.0.0" - assemblyscript@0.19.23: version "0.19.23" resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.19.23.tgz#16ece69f7f302161e2e736a0f6a474e6db72134c" @@ -2215,6 +2207,14 @@ assemblyscript@0.19.23: long "^5.2.0" source-map-support "^0.5.20" +assemblyscript@0.27.31: + version "0.27.31" + resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.27.31.tgz#07412b1bc42c67f78080dbaddca030ab74d3b9b2" + integrity sha512-Ra8kiGhgJQGZcBxjtMcyVRxOEJZX64kd+XGpjWzjcjgxWJVv+CAQO0aDBk4GQVhjYbOkATarC83mHjAVGtwPBQ== + dependencies: + binaryen "116.0.0-nightly.20240114" + long "^5.2.1" + astral-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" @@ -2295,16 +2295,16 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== -binaryen@101.0.0-nightly.20210723: - version "101.0.0-nightly.20210723" - resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-101.0.0-nightly.20210723.tgz#b6bb7f3501341727681a03866c0856500eec3740" - integrity sha512-eioJNqhHlkguVSbblHOtLqlhtC882SOEPKmNFZaDuz1hzQjolxZ+eu3/kaS10n3sGPONsIZsO7R9fR00UyhEUA== - binaryen@102.0.0-nightly.20211028: version "102.0.0-nightly.20211028" resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-102.0.0-nightly.20211028.tgz#8f1efb0920afd34509e342e37f84313ec936afb2" integrity sha512-GCJBVB5exbxzzvyt8MGDv/MeUjs6gkXDvf4xOIItRBptYl0Tz5sm1o/uG95YK0L0VeG5ajDu3hRtkBP2kzqC5w== +binaryen@116.0.0-nightly.20240114: + version "116.0.0-nightly.20240114" + resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-116.0.0-nightly.20240114.tgz#ad8bfbde77d4cb4715b93997114eefc30f45155b" + integrity sha512-0GZrojJnuhoe+hiwji7QFaL3tBlJoA+KFUN7ouYSDGZLSo9CKM8swQX8n/UcbR0d1VuZKU+nhogNzv423JEu5A== + bintrees@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" @@ -4855,12 +4855,7 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" -long@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" - integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== - -long@^5.2.0: +long@^5.2.0, long@^5.2.1: version "5.3.2" resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==