Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ Everytime a new release is merged into `master` there will be a new github relea

# Deploying the subgraph

The deployment manifest IPFS handler uses the native YAML API from `graph-ts` 0.38.2
and requires **Graph Node 0.37.0 or newer**. Malformed manifests retain their raw
content; derived fields that cannot be safely extracted remain unset and produce
warnings. Schema links support CIDv0 and CIDv1 in base32 or base58btc, optionally
with an IPFS file path. Unsupported links are skipped without creating a file data source.

The npm scripts are set up to deploy the subgraphs in one command. Mainnet is connected to a hook
where it will be deployed automatically when the `master` branch is updated. Therefore, we never
have to use npm scripts to directly deploy to `graph-network-mainnet`.
Expand Down Expand Up @@ -91,4 +97,4 @@ the schema.

Copyright © 2020 The Graph Foundation.

Licensed under the [MIT license](./LICENSE).
Licensed under the [MIT license](./LICENSE).
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"@graphprotocol/address-book": "^1.3.0",
"@graphprotocol/contracts": "6.2.0",
"@graphprotocol/graph-cli": "0.97.0",
"@graphprotocol/graph-ts": "0.36.0",
"@graphprotocol/graph-ts": "0.38.2",
"@types/node": "^14.0.13",
"@typescript-eslint/eslint-plugin": "^3.3.0",
"@typescript-eslint/parser": "^3.3.0",
Expand Down
128 changes: 128 additions & 0 deletions src/mappings/helpers/manifest.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import { BigInt, YAMLValue } from '@graphprotocol/graph-ts'

// Avoid YAMLValue's [] accessor: it asserts when a key is absent.
export function yamlField(value: YAMLValue | null, key: string): YAMLValue | null {
if (value === null || !value.isObject()) return null
return value.toObject().get(YAMLValue.newString(key))
}

export function yamlString(value: YAMLValue | null): string | null {
if (value === null || !value.isString()) return null
let text = value.toString().trim()
return text.length > 0 ? text : null
}

// Block numbers must be unsigned integers. YAML NUMBER also includes floats,
// and YAMLValue.toBigInt() does not protect against invalid numeric strings.
export function manifestStartBlock(value: YAMLValue): BigInt | null {
let text: string
if (value.isNumber()) {
text = value.toNumber()
} else if (value.isString()) {
text = value.toString().trim()
} else {
return null
}
if (text.length == 0 || text.length > 20) return null
for (let i = 0; i < text.length; i++) {
let code = text.charCodeAt(i)
if (code < 48 || code > 57) return null
}
if (text.length == 20 && text > '18446744073709551615') return null
return BigInt.fromString(text)
}

// Decode the CID encodings normally used in published manifests. Unsupported
// encodings are skipped rather than passed to a host function that can abort.
function decodeCid(text: string): Uint8Array | null {
if (text.length == 0 || text.length > 128) return null
let base58 = text.startsWith('Qm') || text.startsWith('z')
if (base58) {
let encoded = text.startsWith('z') ? text.slice(1) : text
let alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
let bytes = new Array<u8>()
for (let i = 0; i < encoded.length; i++) {
let carry = alphabet.indexOf(encoded.charAt(i))
if (carry < 0) return null
for (let j = 0; j < bytes.length; j++) {
carry += i32(bytes[j]) * 58
bytes[j] = u8(carry & 255)
carry >>= 8
}
while (carry > 0) {
bytes.push(u8(carry & 255))
carry >>= 8
}
}
for (let i = 0; i < encoded.length && encoded.charAt(i) == '1'; i++) bytes.push(0)
let result = new Uint8Array(bytes.length)
for (let i = 0; i < bytes.length; i++) result[i] = bytes[bytes.length - i - 1]
return result
}
if (!text.startsWith('b') && !text.startsWith('B')) return null
let alphabet = text.startsWith('b') ? 'abcdefghijklmnopqrstuvwxyz234567' : 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
let result = new Uint8Array((text.length - 1) * 5 / 8)
let bits = 0
let buffer = 0
let offset = 0
for (let i = 1; i < text.length; i++) {
let digit = alphabet.indexOf(text.charAt(i))
if (digit < 0) return null
buffer = (buffer << 5) | digit
bits += 5
if (bits >= 8) {
bits -= 8
result[offset++] = u8(buffer >> bits)
buffer &= (1 << bits) - 1
}
}
return bits < 5 && buffer == 0 ? result : null
}

function validCid(text: string): bool {
let bytes = decodeCid(text)
if (bytes === null) return false
if (text.startsWith('Qm')) {
return text.length == 46 && bytes.length == 34 && bytes[0] == 0x12 && bytes[1] == 0x20
}
// CIDv1 contains four unsigned varints: version, codec, hash code, digest
// length; followed by the digest. Graph Node supports digests up to 64 bytes.
let offset = 0
for (let field = 0; field < 4; field++) {
let value: u64 = 0
let terminated = false
for (let i = 0; i < 10 && offset < bytes.length; i++) {
let byte = bytes[offset++]
if (i == 9 && byte > 1) return false
value |= u64(byte & 127) << (i * 7)
if ((byte & 128) == 0) {
if (i > 0 && byte == 0) return false
terminated = true
break
}
}
if (!terminated) return false
if (field == 0 && value != 1) return false
if (field == 3) return value <= 64 && value == u64(bytes.length - offset)
}
return false
}

export function manifestSchemaPath(value: YAMLValue | null): string | null {
// Published manifests use { '/': '/ipfs/CID' }; also accept string links.
if (value !== null && value.isObject()) value = yamlField(value, '/')
let path = yamlString(value)
if (path === null || path.length > 2048) return null
if (path.startsWith('/ipfs/')) path = path.slice(6)
else if (path.startsWith('ipfs://')) path = path.slice(7)
let segments = path.split('/')
if (!validCid(segments[0])) return null
for (let i = 1; i < segments.length; i++) {
if (segments[i].length == 0 || segments[i] == '.' || segments[i] == '..') return null
}
for (let i = 0; i < path.length; i++) {
let code = path.charCodeAt(i)
if (code <= 32 || code == 127 || path.charAt(i) == '?' || path.charAt(i) == '#' || path.charAt(i) == '%' || path.charAt(i) == '\\') return null
}
return path
}
166 changes: 100 additions & 66 deletions src/mappings/ipfs.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt } from '@graphprotocol/graph-ts'
import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt, yaml, YAMLValue } from '@graphprotocol/graph-ts'
import {
SubgraphMeta,
SubgraphVersionMeta,
Expand All @@ -10,6 +10,7 @@ import {
SubgraphDeploymentSchema as SubgraphDeploymentSchemaTemplate
} from '../types/templates'
import { jsonToString } from './utils'
import { yamlField, yamlString, manifestStartBlock, manifestSchemaPath } from './helpers/manifest'

export function handleSubgraphMetadata(content: Bytes): void {
let id = dataSource.context().getString("id")
Expand Down Expand Up @@ -80,78 +81,111 @@ export function handleSubgraphDeploymentSchema(content: Bytes): void {
subgraphDeploymentSchema.save()
}

export function handleSubgraphDeploymentManifest(content: Bytes): void {
// Shouldn't need ID since the handler isn't gonna be called more than once, given that it's only on deployment creation.
let subgraphDeploymentManifest = new SubgraphDeploymentManifest(dataSource.stringParam())
if (content !== null) {
subgraphDeploymentManifest.manifest = content.toString()

let manifest = subgraphDeploymentManifest.manifest!
// we take the right side of the split, since it's the one which will have the schema ipfs hash
let schemaSplitTry = manifest.split('schema:\n', 2)
if (schemaSplitTry.length == 2) {
let schemaSplit = schemaSplitTry[1]

let schemaFileSplitTry = schemaSplit.split('/ipfs/', 2)
if (schemaFileSplitTry.length == 2) {
let schemaFileSplit = schemaFileSplitTry[1]
function manifestWarning(id: string, field: string): void {
log.warning('[MANIFEST PARSING FAIL] deployment: {}, invalid or unsupported {}', [id, field])
}

let schemaIpfsHashTry = schemaFileSplit.split('\n', 2)
if (schemaIpfsHashTry.length == 2) {
let schemaIpfsHash = schemaIpfsHashTry[0]
let schemaId = subgraphDeploymentManifest.id.concat('-').concat(schemaIpfsHash)
subgraphDeploymentManifest.schema = schemaId
subgraphDeploymentManifest.schemaIpfsHash = schemaIpfsHash
function readManifestSchema(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
let path = manifestSchemaPath(yamlField(yamlField(root, 'schema'), 'file'))
if (path === null) {
manifestWarning(manifest.id, 'schema.file')
return
}
let schemaId = manifest.id.concat('-').concat(path)
manifest.schema = schemaId
manifest.schemaIpfsHash = path
let context = new DataSourceContext()
context.setString('id', schemaId)
SubgraphDeploymentSchemaTemplate.createWithContext(path, context)
}

let context = new DataSourceContext()
context.setString('id', schemaId)
SubgraphDeploymentSchemaTemplate.createWithContext(schemaIpfsHash, context)
} else {
log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaIpfsHashTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaIpfsHashTry.length.toString()])
}
} else {
log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaFileSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaFileSplitTry.length.toString()])
function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
// Keep the first usable network, falling back to templates when necessary.
let sections = ['dataSources', 'templates']
for (let section = 0; section < sections.length; section++) {
let entries = yamlField(root, sections[section])
if (entries === null || !entries.isArray()) continue
let sources = entries.toArray()
for (let i = 0; i < sources.length; i++) {
let network = yamlString(yamlField(sources[i], 'network'))
if (network !== null && validManifestNetwork(network)) {
manifest.network = network
return
}
} else {
log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, schema file hash can't be retrieved. Error: schemaSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), schemaSplitTry.length.toString()])
}
}
manifestWarning(manifest.id, 'network')
}

// We get the first occurrence of `network` since subgraphs can only have data sources for the same network
let networkSplitTry = manifest.split('network: ', 2)
if (networkSplitTry.length == 2) {
let networkSplit = networkSplitTry[1]
let networkTry = networkSplit.split('\n', 2)
if (networkTry.length == 2) {
let network = networkTry[0]
function validManifestNetwork(network: string): bool {
if (network.length > 256) return false
for (let i = 0; i < network.length; i++) {
let code = network.charCodeAt(i)
if (code <= 32 || code == 127) return false
}
return true
}

subgraphDeploymentManifest.network = network
} else {
log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkTry.length.toString()])
}
} else {
log.warning("[MANIFEST PARSING FAIL] subgraphDeploymentManifest: {}, network can't be parsed. Error: networkSplitTry.length isn't 2, actual length: {}", [dataSource.stringParam(), networkSplitTry.length.toString()])
}
let substreamsSplitTry = manifest.split('- kind: substreams', 2)
subgraphDeploymentManifest.poweredBySubstreams = substreamsSplitTry.length > 1
function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
let sources = yamlField(root, 'dataSources')
if (sources === null || !sources.isArray() || sources.toArray().length == 0) {
manifestWarning(manifest.id, 'dataSources')
return
}

// startBlock calculation
let templatesSplit = manifest.split("templates:")
let nonTemplateManifestSplit = templatesSplit[0] // we take the left as we want to remove the templates for the source checks.
let sourcesSplit = nonTemplateManifestSplit.split("source:") // We want to know how many source definitions we have
let startBlockSplit = nonTemplateManifestSplit.split("startBlock: ") // And how many startBlock definitions we have to know if we should set startBlock to 0

if (sourcesSplit.length > startBlockSplit.length) {
subgraphDeploymentManifest.startBlock = BigInt.fromI32(0)
} else {
// need to figure the minimum startBlock defined, we skip i = 0 as we know it's not gonna contain a start block num, since it's before the first appearance of "startBlock:"
let min = BigInt.fromI32(0)
for(let i = 1; i < startBlockSplit.length; i++) {
let numString = startBlockSplit[i].split("\n", 1)[0].toString()
let num = BigInt.fromString(numString)
min = min == BigInt.fromI32(0) ? num : min <= num ? min : num
}
subgraphDeploymentManifest.startBlock = min
let dataSources = sources.toArray()
let minimum: BigInt | null = null
let validStartBlocks = true
let validKinds = true
let poweredBySubstreams = false
// Only inspect actual dataSources. Templates, comments and context values
// must not affect the minimum start block or the deployment's source kind.
for (let i = 0; i < dataSources.length; i++) {
let dataSource = dataSources[i]
let kind = yamlString(yamlField(dataSource, 'kind'))
if (kind === null) validKinds = false
else if (kind == 'substreams') poweredBySubstreams = true

let source = yamlField(dataSource, 'source')
if (source === null || !source.isObject()) {
validStartBlocks = false
continue
}
let startBlockValue = yamlField(source, 'startBlock')
// A missing startBlock defaults to zero. An explicit null or malformed
// value is unknown, so we cannot reliably report a minimum.
let startBlock = startBlockValue === null ? BigInt.fromI32(0) : manifestStartBlock(startBlockValue)
if (startBlock === null) {
validStartBlocks = false
} else if (minimum === null || startBlock < minimum) {
minimum = startBlock
}
}
subgraphDeploymentManifest.save()

if (poweredBySubstreams || validKinds) manifest.poweredBySubstreams = poweredBySubstreams
else manifestWarning(manifest.id, 'dataSources.kind')
if (validStartBlocks && minimum !== null) manifest.startBlock = minimum
else manifestWarning(manifest.id, 'dataSources.source.startBlock')
}

export function handleSubgraphDeploymentManifest(content: Bytes): void {
let manifest = new SubgraphDeploymentManifest(dataSource.stringParam())
manifest.manifest = content.toString()
// Match the native parser's input limit; retain the raw manifest on failure.
if (content.length > 10000000) {
manifestWarning(manifest.id, 'manifest size')
manifest.save()
return
}
let parsed = yaml.try_fromBytes(content)
if (!parsed.isOk) {
manifestWarning(manifest.id, 'YAML')
} else if (!parsed.value.isObject()) {
manifestWarning(manifest.id, 'manifest root')
} else {
readManifestSchema(manifest, parsed.value)
readManifestNetwork(manifest, parsed.value)
readManifestDataSources(manifest, parsed.value)
}
manifest.save()
}
Loading
Loading