diff --git a/README.md b/README.md index 64b1a41..854985f 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ Everytime a new release is merged into `master` there will be a new github relea # Deploying the subgraph +The deployment manifest IPFS handler uses the native YAML API from `graph-ts` 0.38.2 +and requires **Graph Node 0.37.0 or newer**. Malformed manifests retain their raw +content; derived fields that cannot be safely extracted remain unset and produce +warnings. Schema links support CIDv0 and CIDv1 in base32 or base58btc, optionally +with an IPFS file path. Unsupported links are skipped without creating a file data source. + The npm scripts are set up to deploy the subgraphs in one command. Mainnet is connected to a hook where it will be deployed automatically when the `master` branch is updated. Therefore, we never have to use npm scripts to directly deploy to `graph-network-mainnet`. @@ -91,4 +97,4 @@ the schema. Copyright © 2020 The Graph Foundation. -Licensed under the [MIT license](./LICENSE). \ No newline at end of file +Licensed under the [MIT license](./LICENSE). diff --git a/package.json b/package.json index f5a1cb9..c69394d 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "@graphprotocol/address-book": "^1.3.0", "@graphprotocol/contracts": "6.2.0", "@graphprotocol/graph-cli": "0.97.0", - "@graphprotocol/graph-ts": "0.36.0", + "@graphprotocol/graph-ts": "0.38.2", "@types/node": "^14.0.13", "@typescript-eslint/eslint-plugin": "^3.3.0", "@typescript-eslint/parser": "^3.3.0", diff --git a/src/mappings/helpers/manifest.ts b/src/mappings/helpers/manifest.ts new file mode 100644 index 0000000..0b35517 --- /dev/null +++ b/src/mappings/helpers/manifest.ts @@ -0,0 +1,128 @@ +import { BigInt, YAMLValue } from '@graphprotocol/graph-ts' + +// Avoid YAMLValue's [] accessor: it asserts when a key is absent. +export function yamlField(value: YAMLValue | null, key: string): YAMLValue | null { + if (value === null || !value.isObject()) return null + return value.toObject().get(YAMLValue.newString(key)) +} + +export function yamlString(value: YAMLValue | null): string | null { + if (value === null || !value.isString()) return null + let text = value.toString().trim() + return text.length > 0 ? text : null +} + +// Block numbers must be unsigned integers. YAML NUMBER also includes floats, +// and YAMLValue.toBigInt() does not protect against invalid numeric strings. +export function manifestStartBlock(value: YAMLValue): BigInt | null { + let text: string + if (value.isNumber()) { + text = value.toNumber() + } else if (value.isString()) { + text = value.toString().trim() + } else { + return null + } + if (text.length == 0 || text.length > 20) return null + for (let i = 0; i < text.length; i++) { + let code = text.charCodeAt(i) + if (code < 48 || code > 57) return null + } + if (text.length == 20 && text > '18446744073709551615') return null + return BigInt.fromString(text) +} + +// Decode the CID encodings normally used in published manifests. Unsupported +// encodings are skipped rather than passed to a host function that can abort. +function decodeCid(text: string): Uint8Array | null { + if (text.length == 0 || text.length > 128) return null + let base58 = text.startsWith('Qm') || text.startsWith('z') + if (base58) { + let encoded = text.startsWith('z') ? text.slice(1) : text + let alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + let bytes = new Array() + for (let i = 0; i < encoded.length; i++) { + let carry = alphabet.indexOf(encoded.charAt(i)) + if (carry < 0) return null + for (let j = 0; j < bytes.length; j++) { + carry += i32(bytes[j]) * 58 + bytes[j] = u8(carry & 255) + carry >>= 8 + } + while (carry > 0) { + bytes.push(u8(carry & 255)) + carry >>= 8 + } + } + for (let i = 0; i < encoded.length && encoded.charAt(i) == '1'; i++) bytes.push(0) + let result = new Uint8Array(bytes.length) + for (let i = 0; i < bytes.length; i++) result[i] = bytes[bytes.length - i - 1] + return result + } + if (!text.startsWith('b') && !text.startsWith('B')) return null + let alphabet = text.startsWith('b') ? 'abcdefghijklmnopqrstuvwxyz234567' : 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' + let result = new Uint8Array((text.length - 1) * 5 / 8) + let bits = 0 + let buffer = 0 + let offset = 0 + for (let i = 1; i < text.length; i++) { + let digit = alphabet.indexOf(text.charAt(i)) + if (digit < 0) return null + buffer = (buffer << 5) | digit + bits += 5 + if (bits >= 8) { + bits -= 8 + result[offset++] = u8(buffer >> bits) + buffer &= (1 << bits) - 1 + } + } + return bits < 5 && buffer == 0 ? result : null +} + +function validCid(text: string): bool { + let bytes = decodeCid(text) + if (bytes === null) return false + if (text.startsWith('Qm')) { + return text.length == 46 && bytes.length == 34 && bytes[0] == 0x12 && bytes[1] == 0x20 + } + // CIDv1 contains four unsigned varints: version, codec, hash code, digest + // length; followed by the digest. Graph Node supports digests up to 64 bytes. + let offset = 0 + for (let field = 0; field < 4; field++) { + let value: u64 = 0 + let terminated = false + for (let i = 0; i < 10 && offset < bytes.length; i++) { + let byte = bytes[offset++] + if (i == 9 && byte > 1) return false + value |= u64(byte & 127) << (i * 7) + if ((byte & 128) == 0) { + if (i > 0 && byte == 0) return false + terminated = true + break + } + } + if (!terminated) return false + if (field == 0 && value != 1) return false + if (field == 3) return value <= 64 && value == u64(bytes.length - offset) + } + return false +} + +export function manifestSchemaPath(value: YAMLValue | null): string | null { + // Published manifests use { '/': '/ipfs/CID' }; also accept string links. + if (value !== null && value.isObject()) value = yamlField(value, '/') + let path = yamlString(value) + if (path === null || path.length > 2048) return null + if (path.startsWith('/ipfs/')) path = path.slice(6) + else if (path.startsWith('ipfs://')) path = path.slice(7) + let segments = path.split('/') + if (!validCid(segments[0])) return null + for (let i = 1; i < segments.length; i++) { + if (segments[i].length == 0 || segments[i] == '.' || segments[i] == '..') return null + } + for (let i = 0; i < path.length; i++) { + let code = path.charCodeAt(i) + if (code <= 32 || code == 127 || path.charAt(i) == '?' || path.charAt(i) == '#' || path.charAt(i) == '%' || path.charAt(i) == '\\') return null + } + return path +} diff --git a/src/mappings/ipfs.ts b/src/mappings/ipfs.ts index b182440..30b5db8 100644 --- a/src/mappings/ipfs.ts +++ b/src/mappings/ipfs.ts @@ -1,4 +1,4 @@ -import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt } from '@graphprotocol/graph-ts' +import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt, yaml, YAMLValue } from '@graphprotocol/graph-ts' import { SubgraphMeta, SubgraphVersionMeta, @@ -10,6 +10,7 @@ import { SubgraphDeploymentSchema as SubgraphDeploymentSchemaTemplate } from '../types/templates' import { jsonToString } from './utils' +import { yamlField, yamlString, manifestStartBlock, manifestSchemaPath } from './helpers/manifest' export function handleSubgraphMetadata(content: Bytes): void { let id = dataSource.context().getString("id") @@ -80,78 +81,111 @@ export function handleSubgraphDeploymentSchema(content: Bytes): void { subgraphDeploymentSchema.save() } -export function handleSubgraphDeploymentManifest(content: Bytes): void { - // Shouldn't need ID since the handler isn't gonna be called more than once, given that it's only on deployment creation. - let subgraphDeploymentManifest = new SubgraphDeploymentManifest(dataSource.stringParam()) - if (content !== null) { - subgraphDeploymentManifest.manifest = content.toString() - - let manifest = subgraphDeploymentManifest.manifest! - // we take the right side of the split, since it's the one which will have the schema ipfs hash - let schemaSplitTry = manifest.split('schema:\n', 2) - if (schemaSplitTry.length == 2) { - let schemaSplit = schemaSplitTry[1] - - let schemaFileSplitTry = schemaSplit.split('/ipfs/', 2) - if (schemaFileSplitTry.length == 2) { - let schemaFileSplit = schemaFileSplitTry[1] +function manifestWarning(id: string, field: string): void { + log.warning('[MANIFEST PARSING FAIL] deployment: {}, invalid or unsupported {}', [id, field]) +} - let schemaIpfsHashTry = schemaFileSplit.split('\n', 2) - if (schemaIpfsHashTry.length == 2) { - let schemaIpfsHash = schemaIpfsHashTry[0] - let schemaId = subgraphDeploymentManifest.id.concat('-').concat(schemaIpfsHash) - subgraphDeploymentManifest.schema = schemaId - subgraphDeploymentManifest.schemaIpfsHash = schemaIpfsHash +function readManifestSchema(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + let path = manifestSchemaPath(yamlField(yamlField(root, 'schema'), 'file')) + if (path === null) { + manifestWarning(manifest.id, 'schema.file') + return + } + let schemaId = manifest.id.concat('-').concat(path) + manifest.schema = schemaId + manifest.schemaIpfsHash = path + let context = new DataSourceContext() + context.setString('id', schemaId) + SubgraphDeploymentSchemaTemplate.createWithContext(path, context) +} - let context = new DataSourceContext() - context.setString('id', schemaId) - SubgraphDeploymentSchemaTemplate.createWithContext(schemaIpfsHash, context) - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaIpfsHashTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaIpfsHashTry.length.toString()]) - } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaFileSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaFileSplitTry.length.toString()]) +function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + // Keep the first usable network, falling back to templates when necessary. + let sections = ['dataSources', 'templates'] + for (let section = 0; section < sections.length; section++) { + let entries = yamlField(root, sections[section]) + if (entries === null || !entries.isArray()) continue + let sources = entries.toArray() + for (let i = 0; i < sources.length; i++) { + let network = yamlString(yamlField(sources[i], 'network')) + if (network !== null && validManifestNetwork(network)) { + manifest.network = network + return } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaSplitTry.length.toString()]) } + } + manifestWarning(manifest.id, 'network') +} - // We get the first occurrence of `network` since subgraphs can only have data sources for the same network - let networkSplitTry = manifest.split('network: ', 2) - if (networkSplitTry.length == 2) { - let networkSplit = networkSplitTry[1] - let networkTry = networkSplit.split('\n', 2) - if (networkTry.length == 2) { - let network = networkTry[0] +function validManifestNetwork(network: string): bool { + if (network.length > 256) return false + for (let i = 0; i < network.length; i++) { + let code = network.charCodeAt(i) + if (code <= 32 || code == 127) return false + } + return true +} - subgraphDeploymentManifest.network = network - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkTry.length.toString()]) - } - } else { - log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkSplitTry.length.toString()]) - } - let substreamsSplitTry = manifest.split('- kind: substreams', 2) - subgraphDeploymentManifest.poweredBySubstreams = substreamsSplitTry.length > 1 +function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAMLValue): void { + let sources = yamlField(root, 'dataSources') + if (sources === null || !sources.isArray() || sources.toArray().length == 0) { + manifestWarning(manifest.id, 'dataSources') + return + } - // startBlock calculation - let templatesSplit = manifest.split("templates:") - let nonTemplateManifestSplit = templatesSplit[0] // we take the left as we want to remove the templates for the source checks. - let sourcesSplit = nonTemplateManifestSplit.split("source:") // We want to know how many source definitions we have - let startBlockSplit = nonTemplateManifestSplit.split("startBlock: ") // And how many startBlock definitions we have to know if we should set startBlock to 0 - - if (sourcesSplit.length > startBlockSplit.length) { - subgraphDeploymentManifest.startBlock = BigInt.fromI32(0) - } else { - // need to figure the minimum startBlock defined, we skip i = 0 as we know it's not gonna contain a start block num, since it's before the first appearance of "startBlock:" - let min = BigInt.fromI32(0) - for(let i = 1; i < startBlockSplit.length; i++) { - let numString = startBlockSplit[i].split("\n", 1)[0].toString() - let num = BigInt.fromString(numString) - min = min == BigInt.fromI32(0) ? num : min <= num ? min : num - } - subgraphDeploymentManifest.startBlock = min + let dataSources = sources.toArray() + let minimum: BigInt | null = null + let validStartBlocks = true + let validKinds = true + let poweredBySubstreams = false + // Only inspect actual dataSources. Templates, comments and context values + // must not affect the minimum start block or the deployment's source kind. + for (let i = 0; i < dataSources.length; i++) { + let dataSource = dataSources[i] + let kind = yamlString(yamlField(dataSource, 'kind')) + if (kind === null) validKinds = false + else if (kind == 'substreams') poweredBySubstreams = true + + let source = yamlField(dataSource, 'source') + if (source === null || !source.isObject()) { + validStartBlocks = false + continue + } + let startBlockValue = yamlField(source, 'startBlock') + // A missing startBlock defaults to zero. An explicit null or malformed + // value is unknown, so we cannot reliably report a minimum. + let startBlock = startBlockValue === null ? BigInt.fromI32(0) : manifestStartBlock(startBlockValue) + if (startBlock === null) { + validStartBlocks = false + } else if (minimum === null || startBlock < minimum) { + minimum = startBlock } } - subgraphDeploymentManifest.save() + + if (poweredBySubstreams || validKinds) manifest.poweredBySubstreams = poweredBySubstreams + else manifestWarning(manifest.id, 'dataSources.kind') + if (validStartBlocks && minimum !== null) manifest.startBlock = minimum + else manifestWarning(manifest.id, 'dataSources.source.startBlock') +} + +export function handleSubgraphDeploymentManifest(content: Bytes): void { + let manifest = new SubgraphDeploymentManifest(dataSource.stringParam()) + manifest.manifest = content.toString() + // Match the native parser's input limit; retain the raw manifest on failure. + if (content.length > 10000000) { + manifestWarning(manifest.id, 'manifest size') + manifest.save() + return + } + let parsed = yaml.try_fromBytes(content) + if (!parsed.isOk) { + manifestWarning(manifest.id, 'YAML') + } else if (!parsed.value.isObject()) { + manifestWarning(manifest.id, 'manifest root') + } else { + readManifestSchema(manifest, parsed.value) + readManifestNetwork(manifest, parsed.value) + readManifestDataSources(manifest, parsed.value) + } + manifest.save() } diff --git a/yarn.lock b/yarn.lock index 4b66b15..348ff92 100644 --- a/yarn.lock +++ b/yarn.lock @@ -912,12 +912,12 @@ web3-eth-abi "4.4.1" yaml "2.7.0" -"@graphprotocol/graph-ts@0.36.0": - version "0.36.0" - resolved "https://registry.yarnpkg.com/@graphprotocol/graph-ts/-/graph-ts-0.36.0.tgz#68ed937806bc7b2f8fe0ae2e15447255144f6a41" - integrity sha512-yJNQK5YZWEThuawSboQQ+U4Fb2C78KBjmaoeOK7Nn0CFoChmHc+woRvW3yj+IKVSPc7JNHt4JSUHxVDJfUZbTA== +"@graphprotocol/graph-ts@0.38.2": + version "0.38.2" + resolved "https://registry.yarnpkg.com/@graphprotocol/graph-ts/-/graph-ts-0.38.2.tgz#e99bdb6e0e50485084c442d7a02409082a836ff8" + integrity sha512-87KIFSFs2+Te+mnmb7Y+M57oqzlLy20cIyPIRbn9qJfpZFSZHTKtBLT6KQmcsK0YkoWis9Ur3c3M2c9mmaaEHQ== dependencies: - assemblyscript "0.19.10" + assemblyscript "0.27.31" "@graphprotocol/pino-sentry-simple@0.7.1": version "0.7.1" @@ -2198,14 +2198,6 @@ array-union@^2.1.0: resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== -assemblyscript@0.19.10: - version "0.19.10" - resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.19.10.tgz#7ede6d99c797a219beb4fa4614c3eab9e6343c8e" - integrity sha512-HavcUBXB3mBTRGJcpvaQjmnmaqKHBGREjSPNsIvnAk2f9dj78y4BkMaSSdvBQYWcDDzsHQjyUC8stICFkD1Odg== - dependencies: - binaryen "101.0.0-nightly.20210723" - long "^4.0.0" - assemblyscript@0.19.23: version "0.19.23" resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.19.23.tgz#16ece69f7f302161e2e736a0f6a474e6db72134c" @@ -2215,6 +2207,14 @@ assemblyscript@0.19.23: long "^5.2.0" source-map-support "^0.5.20" +assemblyscript@0.27.31: + version "0.27.31" + resolved "https://registry.yarnpkg.com/assemblyscript/-/assemblyscript-0.27.31.tgz#07412b1bc42c67f78080dbaddca030ab74d3b9b2" + integrity sha512-Ra8kiGhgJQGZcBxjtMcyVRxOEJZX64kd+XGpjWzjcjgxWJVv+CAQO0aDBk4GQVhjYbOkATarC83mHjAVGtwPBQ== + dependencies: + binaryen "116.0.0-nightly.20240114" + long "^5.2.1" + astral-regex@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" @@ -2295,16 +2295,16 @@ binary-extensions@^2.0.0: resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== -binaryen@101.0.0-nightly.20210723: - version "101.0.0-nightly.20210723" - resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-101.0.0-nightly.20210723.tgz#b6bb7f3501341727681a03866c0856500eec3740" - integrity sha512-eioJNqhHlkguVSbblHOtLqlhtC882SOEPKmNFZaDuz1hzQjolxZ+eu3/kaS10n3sGPONsIZsO7R9fR00UyhEUA== - binaryen@102.0.0-nightly.20211028: version "102.0.0-nightly.20211028" resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-102.0.0-nightly.20211028.tgz#8f1efb0920afd34509e342e37f84313ec936afb2" integrity sha512-GCJBVB5exbxzzvyt8MGDv/MeUjs6gkXDvf4xOIItRBptYl0Tz5sm1o/uG95YK0L0VeG5ajDu3hRtkBP2kzqC5w== +binaryen@116.0.0-nightly.20240114: + version "116.0.0-nightly.20240114" + resolved "https://registry.yarnpkg.com/binaryen/-/binaryen-116.0.0-nightly.20240114.tgz#ad8bfbde77d4cb4715b93997114eefc30f45155b" + integrity sha512-0GZrojJnuhoe+hiwji7QFaL3tBlJoA+KFUN7ouYSDGZLSo9CKM8swQX8n/UcbR0d1VuZKU+nhogNzv423JEu5A== + bintrees@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" @@ -4855,12 +4855,7 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" -long@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" - integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== - -long@^5.2.0: +long@^5.2.0, long@^5.2.1: version "5.3.2" resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==