WIP: Data source context

.github/workflows/tests-l2.yaml

@@ -0,0 +1,32 @@
+name: Tests (L2)
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # Install commands
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20'
+          registry-url: https://registry.npmjs.org
+      - name: yarn add ts-node
+        run: yarn add ts-node
+      - name: yarn install
+        run: yarn install
+
+      # Run scripts
+      - name: Prep addressess
+        run: ./node_modules/.bin/ts-node config/testAddressesL2.ts && ./node_modules/.bin/mustache ./config/generatedAddresses.json ./config/addresses.template.ts > ./config/addresses.ts
+      - name: Prep test L2
+        run: ./node_modules/.bin/mustache ./config/generatedAddresses.json subgraph.template.yaml > subgraph.yaml && ./node_modules/@graphprotocol/graph-cli/bin/run codegen --output-dir src/types/
+      - name: Test
+        run: ./node_modules/@graphprotocol/graph-cli/bin/run test -v 0.6.0-rc.2

package.json

@@ -47,7 +47,7 @@
   },
   "devDependencies": {
     "@graphprotocol/contracts": "6.2.0",
-    "@graphprotocol/graph-cli": "0.68.0",
+    "@graphprotocol/graph-cli": "0.68.5",
     "@graphprotocol/graph-ts": "0.32.0",
     "@types/node": "^14.0.13",
     "@typescript-eslint/eslint-plugin": "^3.3.0",

schema.graphql

@@ -629,29 +629,51 @@ type SubgraphDeploymentManifest @entity(immutable:true) {
   network: String
   "Whether the subgraph is a SpS/SbS. Null if we can't parse it"
   poweredBySubstreams: Boolean
+  "Start block for the deployment. It's the lowest startBlock found (0 if some data source doesn't contain a start block)"
+  startBlock: BigInt
+  "List of data sources and data source templates"
+  dataSources: [SubgraphDeploymentManifestDataSource!]! @derivedFrom(field:"manifest")
 }
 
-# TODO - add when we have the ability to parse data sources
-# """Data source obtained from the subgraph manifest"""
-# type DataSource @entity {
-#   "Unique identifier of the data source. Such as contract address"
-#   id: ID!
-#   "Data source name in the manifest"
-#   name: String!
-#   "Networks that the subgraph deployment is indexing"
-#   networks: [String!]!
-#   "Contract"
-#   contract: Contract!
-#   "ABI of the contract"
-#   abi: String!
-# }
-
-# type Contract @entity {
-#   "Address of the contract"
-#   id: ID!
-#   "Contract name"
-#   name: String
-# }
+type SubgraphDeploymentManifestDataSource @entity(immutable:true) {
+  "IPFS Hash + data source index. Only for 'kind: ethereum' data sources"
+  id: ID!
+  "Name of the data source"
+  name: String
+  "Contract address that the data source listenes to"
+  address: Bytes
+  "Start block for this data source"
+  startBlock: BigInt
+  "Network that this data source is indexing"
+  network: String
+  "API Version"
+  apiVersion: String
+  "Handlers"
+  handlers: [SubgraphDeploymentManifestHandler!]! @derivedFrom(field:"dataSource")
+  "Manifest that this data source is linked to"
+  manifest: SubgraphDeploymentManifest!
+  "Whether it's a template or not"
+  isTemplate: Boolean
+}
+
+type SubgraphDeploymentManifestHandler @entity(immutable:true) {
+  "Data source id + handler index"
+  id: ID!
+  "Type of the handler. Can be Event, Call or Block"
+  type: EventHandlerType
+  "Handler function"
+  handler: String
+  "Event/Call signature. Null if block handler"
+  signature: String
+  "Link to data source or data source template"
+  dataSource: SubgraphDeploymentManifestDataSource!
+}
+
+enum EventHandlerType {
+  EventHandler,
+  CallHandler,
+  BlockHandler
+}
 
 """
 Meta for the Indexer along with parameters and staking data

src/mappings/ipfs.ts

@@ -1,10 +1,11 @@
-import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext } from '@graphprotocol/graph-ts'
+import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt } from '@graphprotocol/graph-ts'
 import {
   SubgraphMeta,
   SubgraphVersionMeta,
   GraphAccountMeta,
   SubgraphDeploymentSchema,
   SubgraphDeploymentManifest,
+  SubgraphDeploymentManifestDataSource,
 } from '../types/schema'
 import {
   SubgraphDeploymentSchema as SubgraphDeploymentSchemaTemplate
@@ -133,6 +134,98 @@ export function handleSubgraphDeploymentManifest(content: Bytes): void {
     }
     let substreamsSplitTry = manifest.split('- kind: substreams', 2)
     subgraphDeploymentManifest.poweredBySubstreams = substreamsSplitTry.length > 1
+
+    let templatesSplit = manifest.split("templates:")
+    // startBlock calculation
+    let nonTemplateManifestSplit = templatesSplit[0] // we take the left as we want to remove the templates for the source checks.
+    let sourcesSplit = nonTemplateManifestSplit.split("source:") // We want to know how many source definitions we have
+    let startBlockSplit = nonTemplateManifestSplit.split("startBlock: ") // And how many startBlock definitions we have to know if we should set startBlock to 0
+
+    if (sourcesSplit.length > startBlockSplit.length) {
+      subgraphDeploymentManifest.startBlock = BigInt.fromI32(0)
+    } else {
+      // need to figure the minimum startBlock defined, we skip i = 0 as we know it's not gonna contain a start block num, since it's before the first appearance of "startBlock:"
+      let min = BigInt.fromI32(0)
+      for(let i = 1; i < startBlockSplit.length; i++) {
+        let numString = startBlockSplit[i].split("\n", 1)[0].toString()
+        let num = BigInt.fromString(numString)
+        min = min == BigInt.fromI32(0) ? num : min <= num ? min : num
+      }
+      subgraphDeploymentManifest.startBlock = min
+    }
+
+    // DataSource and DataSourceTemplate creations, we reuse the templatesSplit
+    let dataSourceSplit = templatesSplit[0].split("kind: ethereum").slice(1) // Remove first value since it's gonna be the 'dataSources:' definition
+    createDataSources(dataSourceSplit, subgraphDeploymentManifest.id)
+    // Templates
+    if (templatesSplit.length > 1) {
+      let dataSourceTemplatesSplit = templatesSplit[1].split("kind: ethereum").slice(1) // Remove first value since it's gonna be the 'dataSources:' definition
+      createDataSourceTemplates(dataSourceTemplatesSplit, subgraphDeploymentManifest.id, dataSourceSplit.length) // we need to pass an offset to make sure the IDs stay consistent
+    }
+
   }
   subgraphDeploymentManifest.save()
+}
+
+function createDataSources(dataSourceStrings: String[], manifestId: string): void {
+  for(let i = 0; i < dataSourceStrings.length; i++) {
+    let dataSourceString = dataSourceStrings[i]
+    let dataSource = new SubgraphDeploymentManifestDataSource(manifestId.concat('-').concat(i.toString()))
+    dataSource.manifest = manifestId
+    dataSource.name = getManifestFieldFromExtract(dataSourceString, "name")
+    dataSource.address = Bytes.fromHexString(getManifestFieldFromExtract(dataSourceString, "address"))
+    let startBlock = getManifestFieldFromExtract(dataSourceString, "startBlock")
+    dataSource.startBlock = BigInt.fromString(startBlock == "" ? "0" : startBlock)
+    dataSource.network = getManifestFieldFromExtract(dataSourceString, "network")
+    dataSource.apiVersion = getManifestFieldFromExtract(dataSourceString, "apiVersion")
+    dataSource.isTemplate = false
+    dataSource.save()
+  }
+}
+
+
+function createDataSourceTemplates(dataSourceTemplateStrings: String[], manifestId: string, indexOffset: number): void {
+  for(let i = 0; i < dataSourceTemplateStrings.length; i++) {
+    let idIndex = i + indexOffset
+    let dataSourceTemplateString = dataSourceTemplateStrings[i]
+    let dataSourceTemplate = new SubgraphDeploymentManifestDataSource(manifestId.concat('-').concat(idIndex.toString()))
+    dataSourceTemplate.manifest = manifestId
+    dataSourceTemplate.name = getManifestFieldFromExtract(dataSourceTemplateString, "name")
+    dataSourceTemplate.apiVersion = getManifestFieldFromExtract(dataSourceTemplateString, "apiVersion")
+    dataSourceTemplate.isTemplate = true
+    dataSourceTemplate.save()
+  }
+}
+
+// To Do
+function createEventHandlers(): void {
+
+}
+
+function createCallHandlers(): void {
+
+}
+
+function createBlockHandlers(): void {
+
+}
+
+export function getManifestFieldFromExtract(manifestExtract: String, fieldName: String): string {
+  let res = ""
+  let fieldNameSplit = manifestExtract.split(fieldName+":")
+  if (fieldNameSplit.length > 1) {
+    res = cleanStr(fieldNameSplit[1].split("\n")[0])
+  }
+  return res
+}
+
+function cleanStr(str: String): string {
+  let res = str.trim()
+  if (res.startsWith("'") || res.startsWith("\"")) {
+    res = res.slice(1)
+  }
+  if (res.endsWith("'") || res.endsWith("\"")) {
+    res = res.slice(0, res.length - 1)
+  }
+  return res
 }

subgraph.template.yaml

@@ -710,6 +710,8 @@ templates:
       handler: handleSubgraphDeploymentManifest
       entities:
         - SubgraphDeploymentManifest
+        - SubgraphDeploymentManifestDataSource
+        - SubgraphDeploymentManifestHandler
       abis:
         - name: EpochManager
           file: ./node_modules/@graphprotocol/contracts/dist/abis/EpochManager.json

tests/curation.test.ts

@@ -70,14 +70,15 @@ createMockedFunction(controllerAddress, 'getGovernor', 'getGovernor():(address)'
   // L2 graph network init EpochManager call
   createMockedFunction(graphAddress, 'blockNum', 'blockNum():(uint256)')
   .withArgs([])
-  .returns([ethereum.Value.fromI32(0)])
+  .returns([ethereum.Value.fromI32(5)])
 
 describe('Signalled', () => {
   beforeAll(() => {
     // We need epoch length and delegation ratio to be nonzero for these tests
     let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
     graphNetwork.delegationRatio = delegationRatio
     graphNetwork.epochLength = epochLength
+    graphNetwork.lastLengthUpdateBlock = 1
     graphNetwork.save()
 
     // When _stake is successfully run, before any event we are considering here can be emitted
@@ -202,6 +203,7 @@ describe('Burned', () => {
     let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
     graphNetwork.delegationRatio = delegationRatio
     graphNetwork.epochLength = epochLength
+    graphNetwork.lastLengthUpdateBlock = 1
     graphNetwork.save()
 
     // When _stake is successfully run, before any event we are considering here can be emitted
@@ -313,6 +315,7 @@ describe('ParameterUpdated', () => {
     let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
     graphNetwork.delegationRatio = delegationRatio
     graphNetwork.epochLength = epochLength
+    graphNetwork.lastLengthUpdateBlock = 1
     graphNetwork.save()
 
     // When _stake is successfully run, before any event we are considering here can be emitted

tests/gns.test.ts

@@ -23,7 +23,7 @@ createMockedFunction(controllerAddress, 'getGovernor', 'getGovernor():(address)'
   // L2 graph network init EpochManager call
   createMockedFunction(graphAddress, 'blockNum', 'blockNum():(uint256)')
   .withArgs([])
-  .returns([ethereum.Value.fromI32(0)])
+  .returns([ethereum.Value.fromI32(5)])
 
 // CONSTANTS
 const blockNumber = BigInt.fromI32(1)

tests/helpers.test.ts

@@ -26,7 +26,7 @@ createMockedFunction(controllerAddress, 'getGovernor', 'getGovernor():(address)'
 // L2 graph network init EpochManager call
 createMockedFunction(graphAddress, 'blockNum', 'blockNum():(uint256)')
 .withArgs([])
-.returns([ethereum.Value.fromI32(0)])
+.returns([ethereum.Value.fromI32(1)])
 
 afterAll(() => {
   // Clear the store in order to start the next test off on a clean slate

tests/staking.test.ts

@@ -109,7 +109,7 @@ createMockedFunction(controllerAddress, 'getGovernor', 'getGovernor():(address)'
 // L2 graph network init EpochManager call
 createMockedFunction(graphAddress, 'blockNum', 'blockNum():(uint256)')
   .withArgs([])
-  .returns([ethereum.Value.fromI32(0)])
+  .returns([ethereum.Value.fromI32(5)])
 
 // INDEXER STAKE RELATED TESTS
 describe('INDEXER STAKE', () => {
@@ -180,6 +180,7 @@ describe('INDEXER STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -233,6 +234,7 @@ describe('INDEXER STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -282,6 +284,7 @@ describe('INDEXER STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -342,6 +345,7 @@ describe('INDEXER STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -382,6 +386,7 @@ describe('DELEGATOR STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -453,6 +458,7 @@ describe('DELEGATOR STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -543,6 +549,7 @@ describe('DELEGATOR STAKE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -600,6 +607,7 @@ describe('ALLOCATION LIFE CYCLE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted
@@ -670,6 +678,7 @@ describe('ALLOCATION LIFE CYCLE', () => {
       let graphNetwork = createOrLoadGraphNetwork(blockNumber, controllerAddress)
       graphNetwork.delegationRatio = delegationRatio
       graphNetwork.epochLength = epochLength
+      graphNetwork.lastLengthUpdateBlock = 1
       graphNetwork.save()
 
       // When _stake is successfully run, before any event we are considering here can be emitted

tests/yaml.test.ts

@@ -0,0 +1,14 @@
+import { test, assert } from 'matchstick-as/assembly/index'
+import { getManifestFieldFromExtract } from '../src/mappings/ipfs'
+
+test('Check get field from extract', () => {
+  let extract = "dataSources:\n  - kind: ethereum\n    mapping:\n      abis:\n        - file:\n            /: /ipfs/QmdxvRJHRaDpLKYdiQJ294CxtEQExftBmnFxsqrMKcyja2\n          name: milady\n      apiVersion: 0.0.7\n      entities:\n        - Approval\n        - ApprovalForAll\n        - OwnershipTransferred\n        - Transfer\n      eventHandlers:\n        - event: 'Approval(indexed address,indexed address,indexed uint256)'\n          handler: handleApproval\n        - event: 'ApprovalForAll(indexed address,indexed address,bool)'\n          handler: handleApprovalForAll\n        - event: 'OwnershipTransferred(indexed address,indexed address)'\n          handler: handleOwnershipTransferred\n        - event: 'Transfer(indexed address,indexed address,indexed uint256)'\n          handler: handleTransfer\n      file:\n        /: /ipfs/QmXtmRDcZNNR2FnzAgULcLQVnV89fEAowJXipSpfCgKZEp\n      kind: ethereum/events\n      language: wasm/assemblyscript\n    name: milady\n    network: mainnet\n    source:\n      abi: milady\n      address: '0x5af0d9827e0c53e4799bb226655a1de152a425a5'\n      startBlock: 13090020\ndescription: Mailady NFT Subgraph\nschema:\n  file:\n    /: /ipfs/QmP8HWfXhWums3QTxMJ24e83ZnUkHmRVmHo6aTMYh31N72\nspecVersion: 0.0.5\n"
+  let name = getManifestFieldFromExtract(extract, "name")
+  let apiVersion = getManifestFieldFromExtract(extract, "apiVersion")
+  let network = getManifestFieldFromExtract(extract, "network")
+  let address = getManifestFieldFromExtract(extract, "address")
+  assert.stringEquals("milady", name)
+  assert.stringEquals("0.0.7", apiVersion)
+  assert.stringEquals("mainnet", network)
+  assert.stringEquals("0x5af0d9827e0c53e4799bb226655a1de152a425a5", address)
+})