Feature/local stack tests

packages/sdk/.drone.yml

@@ -7,6 +7,216 @@ trigger:
   event: push
 
 steps:
+  ##### START LOCAL STACK #####
+  - name: prepare local-stack-env
+    image: node
+    commands:
+      - node packages/sdk/tests/prepare-test-env.js
+
+  - name: bellecour-fork
+    detach: true
+    image: ghcr.io/foundry-rs/foundry:latest
+    pull: always
+    expose:
+      - 8545
+    commands:
+      - anvil --host 0.0.0.0 --port 8545 --block-time 1 --hardfork berlin --fork-url $(cat local-stack-env/BELLECOUR_FORK_URL) --fork-block-number $(cat local-stack-env/BELLECOUR_FORK_BLOCK) --chain-id 134 --gas-limit 6700000 --gas-price 0
+
+  - name: ipfs
+    image: ipfs/go-ipfs:v0.9.1
+    detach: true
+    expose:
+      - 8080
+      - 5001
+
+  - name: market-mongo
+    image: mongo:6.0.3
+    detach: true
+    expose:
+      - 27017
+
+  - name: market-redis
+    image: redis:7.0.7-alpine
+    detach: true
+    commands:
+      - redis-server --appendonly yes
+    expose:
+      - 6379
+
+  - name: result-proxy-mongo
+    image: library/mongo:4.2
+    detach: true
+    commands:
+      - mongod --bind_ip_all --port 13202
+    expose:
+      - 13202
+
+  - name: graphnode-postgres
+    image: postgres:12
+    detach: true
+    commands:
+      - docker-entrypoint.sh postgres -cshared_preload_libraries=pg_stat_statements
+    expose:
+      - 5432
+    environment:
+      POSTGRES_USER: graphnode
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: graphnode-db
+
+  - name: graphnode
+    image: graphprotocol/graph-node:v0.27.0
+    detach: true
+    expose:
+      - 8000
+      - 8020
+    environment:
+      postgres_host: graphnode-postgres
+      postgres_port: 5432
+      postgres_user: graphnode
+      postgres_pass: password
+      postgres_db: graphnode-db
+      ipfs: ipfs:5001
+      ethereum: bellecour:http://bellecour-fork:8545
+    commands:
+      # bellecour-fork healthcheck
+      - while ! nc -z bellecour-fork 8545 ; do sleep 1 ; done && echo "bellecour-fork ready"
+      # ipfs healthcheck
+      - while ! nc -z ipfs 8080 ; do sleep 1 ; done && echo "ipfs gateway ready"
+      - while ! nc -z ipfs 5001 ; do sleep 1 ; done && echo "ipfs upload ready"
+      # graphnode-postgres healthcheck
+      - while ! nc -z graphnode-postgres 5432 ; do sleep 1 ; done && echo "graphnode-postgres ready"
+      # start
+      - export GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER=$(cat local-stack-env/BELLECOUR_FORK_BLOCK)
+      - start
+
+  - name: sms
+    image: iexechub/iexec-sms:7.1.0
+    detach: true
+    environment:
+      TZ: Europe/Paris
+      IEXEC_SMS_BLOCKCHAIN_NODE_ADDRESS: http://bellecour-fork:8545
+      IEXEC_HUB_ADDRESS: '0x3eca1B216A7DF1C7689aEb259fFB83ADFB894E7f'
+      IEXEC_TEE_WORKER_PRE_COMPUTE_IMAGE: docker.io/iexechub/tee-worker-pre-compute:7.1.0-sconify-5.3.15-debug
+      IEXEC_TEE_WORKER_PRE_COMPUTE_FINGERPRINT: 9f0f782d6edc611baa23ca0978f555ee58ea70e092640c961e75c25e9e4b0f22
+      IEXEC_TEE_WORKER_PRE_COMPUTE_HEAP_SIZE_GB: 4
+      IEXEC_TEE_WORKER_POST_COMPUTE_IMAGE: docker.io/iexechub/tee-worker-post-compute:7.1.1-sconify-5.3.15-debug
+      IEXEC_TEE_WORKER_POST_COMPUTE_FINGERPRINT: face1376b97131e2dc75a556381d47a2e03bed9e1bc11e462471f99d1eefae50
+      IEXEC_TEE_WORKER_POST_COMPUTE_HEAP_SIZE_GB: 4
+      IEXEC_IGNORED_SGX_ADVISORIES: INTEL-SA-00161,INTEL-SA-00289,INTEL-SA-00334,INTEL-SA-00381,INTEL-SA-00389,INTEL-SA-00220,INTEL-SA-00270,INTEL-SA-00293,INTEL-SA-00320,INTEL-SA-00329,INTEL-SA-00477
+      IEXEC_SCONE_TOLERATED_INSECURE_OPTIONS: debug-mode,hyperthreading,outdated-tcb
+      IEXEC_SMS_DISPLAY_DEBUG_SESSION: 'true'
+      IEXEC_SCONE_CAS_HOST: foo
+      IEXEC_SMS_IMAGE_LAS_IMAGE: foo
+    expose:
+      - 13300
+    commands:
+      # bellecour-fork healthcheck
+      - sleep 30
+      # start
+      - java -jar /app/iexec-sms.jar
+
+  - name: result-proxy
+    image: iexechub/iexec-result-proxy:7.1.0
+    detach: true
+    environment:
+      IEXEC_PRIVATE_CHAIN_ADDRESS: http://bellecour-fork:8545
+      IEXEC_PUBLIC_CHAIN_ADDRESS: http://bellecour-fork:8545
+      IEXEC_HUB_ADDRESS: '0x3eca1B216A7DF1C7689aEb259fFB83ADFB894E7f'
+      MONGO_HOST: result-proxy-mongo
+      MONGO_PORT: 13202
+      IEXEC_IPFS_HOST: ipfs
+    expose:
+      - 13200
+    commands:
+      # bellecour-fork healthcheck
+      # result-proxy-mongo healthcheck
+      # ipfs healthcheck
+      - sleep 30
+      # start
+      - exec java -Djava.security.egd=file:/dev/./urandom -jar /iexec-result-proxy.jar
+
+  - name: market-watcher
+    image: iexechub/iexec-market-watcher:6.4
+    detach: true
+    environment:
+      CHAIN: BELLECOUR
+      ETH_WS_HOST: ws://bellecour-fork:8545
+      ETH_RPC_HOST: http://bellecour-fork:8545
+      MONGO_HOST: market-mongo
+      REDIS_HOST: market-redis
+    commands:
+      # bellecour-fork healthcheck
+      - while ! nc -z bellecour-fork 8545 ; do sleep 1 ; done && echo "bellecour-fork ready"
+      # market-mongo healthcheck
+      - while ! nc -z market-mongo 27017 ; do sleep 1 ; done && echo "market-mongo ready"
+      # market-redis healthcheck
+      - while ! nc -z market-redis 6379 ; do sleep 1 ; done && echo "market-redis ready"
+      # start
+      - export START_BLOCK=$(cat local-stack-env/BELLECOUR_FORK_BLOCK)
+      - cd /app
+      - npm start
+
+  - name: market-api
+    image: iexechub/iexec-market-api:6.4
+    detach: true
+    expose:
+      - 3000
+    environment:
+      CHAINS: BELLECOUR_FORK
+      BELLECOUR_FORK_ETH_RPC_HOST: http://bellecour-fork:8545
+      BELLECOUR_FORK_CHAIN_ID: 134
+      BELLECOUR_FORK_IS_NATIVE: 'true'
+      BELLECOUR_FORK_IEXEC_ADDRESS: '0x3eca1B216A7DF1C7689aEb259fFB83ADFB894E7f'
+      MONGO_HOST: market-mongo
+      REDIS_HOST: market-redis
+      RATE_LIMIT_MAX: 10000
+      RATE_LIMIT_PERIOD: 60000
+      MAX_OPEN_ORDERS_PER_WALLET: 1000
+    commands:
+      # bellecour-fork healthcheck
+      - while ! nc -z bellecour-fork 8545 ; do sleep 1 ; done && echo "bellecour-fork ready"
+      # market-mongo healthcheck
+      - while ! nc -z market-mongo 27017 ; do sleep 1 ; done && echo "market-mongo ready"
+      # market-redis healthcheck
+      - while ! nc -z market-redis 6379 ; do sleep 1 ; done && echo "market-redis ready"
+      # start
+      - export START_BLOCK=$(cat local-stack-env/BELLECOUR_FORK_BLOCK)
+      - cd /app
+      - npm start
+
+  - name: local subgraph deployment
+    image: node:18
+    environment:
+      GRAPHNODE_URL: http://graphnode:8020
+      IPFS_URL: http://ipfs:5001
+    commands:
+      # install deps
+      - cd packages/subgraph
+      - npm ci
+      # ipfs healthcheck
+      - while ! curl http://ipfs:5001 > /dev/null 2>&1 ; do sleep 1 ; done && echo "ipfs ready"
+      # graphnode healthcheck
+      - while ! curl http://graphnode:8020 > /dev/null 2>&1 ; do sleep 1 ; done && echo "graphnode admin ready"
+      - export START_BLOCK=$(cat local-stack-env/BELLECOUR_FORK_BLOCK)
+      - npm run all
+
+  - name: services healthcheck
+    image: bash
+    commands:
+      # bellecour-fork healthcheck
+      - while ! nc -z bellecour-fork 8545 ; do sleep 1 ; done && echo "bellecour-fork ready"
+      # ipfs healthcheck
+      - while ! nc -z ipfs 8080 ; do sleep 1 ; done && echo "ipfs gateway ready"
+      - while ! nc -z ipfs 5001 ; do sleep 1 ; done && echo "ipfs upload ready"
+      # market-api healthcheck
+      - while ! nc -z market-api 3000 ; do sleep 1 ; done && echo "market-api ready"
+      # sms healthcheck
+      - while ! nc -z sms 13300 ; do sleep 1 ; done && echo "sms ready"
+      # result-proxy healthcheck
+      - while ! nc -z result-proxy 13200 ; do sleep 1 ; done && echo "result-proxy ready"
+
+  ### START LOCAL STACK END ###
+
   - name: only sdk package
     image: bash
     commands:

packages/sdk/.eslintignore

@@ -1,4 +1,6 @@
 **/coverage/
 **/demo/
 **/dist/
-**/node_modules/
+**/node_modules/
+
+tests/prepare-test-env.js

packages/sdk/CHANGELOG.md

@@ -4,9 +4,19 @@ All notable changes to this project will be documented in this file.
 
 ## NEXT
 
+### Added
+
+- Added `workerpool` option for `processProtectedData` to override the workerpool to use
+- Added a dockerized local stack for testing
+
 ### Changed
 
+- Run tests on a local stack forked from bellecour
 - Support ENS names for `owner` option of `fetchProtectedData`
+- Fixed a bug that allowed `"any"` to be passed as `protectedData` to `processProtectedData`
+- Fixed a bug that allowed `"any"` to be passed as `app` to `processProtectedData`
+- Fixed a bug that may cause the user to pay gas fees when creating a protected data
+- Changed URL validation to be more permissive
 
 ## [0.5.1] (2024-01-11)
 

packages/sdk/package.json

@@ -19,7 +19,9 @@
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write \"(src|tests)/**/*.ts\"",
-    "check-format": "prettier --check \"(src|tests)/**/*.ts|tests/**/*.ts\""
+    "check-format": "prettier --check \"(src|tests)/**/*.ts|tests/**/*.ts\"",
+    "stop-test-stack": "cd tests && docker compose down --remove-orphans",
+    "start-test-stack": "cd tests && npm run stop-test-stack && node prepare-test-env.js && docker compose up -d"
   },
   "repository": {
     "type": "git",

packages/sdk/src/dataProtector/processProtectedData.ts

@@ -8,6 +8,7 @@ import { fetchOrdersUnderMaxPrice } from '../utils/fetchOrdersUnderMaxPrice.js';
 import { pushRequesterSecret } from '../utils/pushRequesterSecret.js';
 import {
   addressOrEnsOrAnySchema,
+  addressOrEnsSchema,
   positiveNumberSchema,
   secretsSchema,
   stringSchema,
@@ -24,14 +25,15 @@ export const processProtectedData = async ({
   args,
   inputFiles,
   secrets,
+  workerpool,
 }: IExecConsumer & ProcessProtectedDataParams): Promise<Taskid> => {
   try {
     const requester = await iexec.wallet.getAddress();
-    const vApp = addressOrEnsOrAnySchema()
+    const vApp = addressOrEnsSchema()
       .required()
       .label('authorizedApp')
       .validateSync(app);
-    const vProtectedData = addressOrEnsOrAnySchema()
+    const vProtectedData = addressOrEnsSchema()
       .required()
       .label('protectedData')
       .validateSync(protectedData);
@@ -43,6 +45,10 @@ export const processProtectedData = async ({
       .validateSync(inputFiles);
     const vArgs = stringSchema().label('args').validateSync(args);
     const vSecrets = secretsSchema().label('secrets').validateSync(secrets);
+    const vWorkerpool = addressOrEnsOrAnySchema()
+      .default(WORKERPOOL_ADDRESS)
+      .label('workerpool')
+      .validateSync(workerpool);
     const isIpfsStorageInitialized =
       await iexec.storage.checkStorageTokenExists(requester);
     if (!isIpfsStorageInitialized) {
@@ -54,6 +60,7 @@ export const processProtectedData = async ({
       vProtectedData,
       {
         app: vApp,
+        workerpool: vWorkerpool,
         requester,
       }
     );
@@ -62,10 +69,10 @@ export const processProtectedData = async ({
       requester,
       minTag: SCONE_TAG,
       maxTag: SCONE_TAG,
-      workerpool: WORKERPOOL_ADDRESS,
+      workerpool: vWorkerpool,
     });
     const workerpoolOrderbook = await iexec.orderbook.fetchWorkerpoolOrderbook({
-      workerpool: WORKERPOOL_ADDRESS,
+      workerpool: vWorkerpool,
       app: vApp,
       dataset: vProtectedData,
       minTag: SCONE_TAG,
@@ -88,7 +95,7 @@ export const processProtectedData = async ({
       appmaxprice: vMaxPrice,
       workerpoolmaxprice: vMaxPrice,
       tag: SCONE_TAG,
-      workerpool: WORKERPOOL_ADDRESS,
+      workerpool: underMaxPriceOrders.workerpoolorder.workerpool,
       params: {
         iexec_input_files: vInputFiles,
         iexec_developer_logger: true,

packages/sdk/src/dataProtector/protectDataObservable.ts

@@ -120,7 +120,7 @@ export const protectDataObservable = ({
           cid,
         });
 
-        const { provider, signer } =
+        const { provider, signer, txOptions } =
           await iexec.config.resolveContractsClient();
 
         const contract = new ethers.Contract(contractAddress, ABI, provider);
@@ -140,7 +140,8 @@ export const protectDataObservable = ({
             vName,
             JSON.stringify(schema),
             multiaddrBytes,
-            checksum
+            checksum,
+            txOptions
           )
           .then((tx) => tx.wait())
           .catch((e: Error) => {

packages/sdk/src/dataProtector/types.ts

@@ -111,6 +111,11 @@ export type ProcessProtectedDataParams = {
    * It is represented as a mapping of numerical identifiers to corresponding secrets.
    */
   secrets?: Record<number, string>;
+
+  /**
+   * The workerpool to use for the application's execution. (default iExec production workerpool)
+   */
+  workerpool?: AddressOrENS | 'any';
 };
 
 type ProtectDataDataExtractedMessage = {

packages/sdk/src/utils/validators.ts

@@ -17,7 +17,8 @@ const isZeroStringTest = (value: string) => value === '0';
 export const stringSchema = () =>
   string().strict().typeError('${path} should be a string');
 
-export const urlSchema = () => string().url('${path} should be a url');
+export const urlSchema = () =>
+  string().matches(/^http[s]?:\/\//, '${path} should be a url');
 
 export const addressSchema = () =>
   string()