feat: add bulk processing support (#525)

Co-authored-by: Pierre Jeanjacquot <[email protected]>

Author: SeddikBellamine

packages/sdk/package-lock.json

@@ -61,9 +61,9 @@
     "debug": "^4.3.4",
     "ethers": "^6.13.2",
     "graphql-request": "^6.0.0",
-    "iexec": "^8.20.0",
+    "iexec": "^8.22.0",
     "jszip": "^3.7.1",
-    "kubo-rpc-client": "^4.1.1",
+    "kubo-rpc-client": "^5.4.1",
     "magic-bytes.js": "^1.0.15",
     "typechain": "^8.3.2",
     "yup": "^1.0.2"

packages/sdk/package.json

@@ -118,6 +118,7 @@ abstract class IExecDataProtectorModule {
         { ethProvider: this.ethProvider },
         {
           ipfsGatewayURL: ipfsGateway,
+          ipfsNodeURL: ipfsNode,
           ...this.options?.iexecOptions,
           allowExperimentalNetworks: this.options.allowExperimentalNetworks,
         }

packages/sdk/src/lib/IExecDataProtectorModule.ts

@@ -8,6 +8,8 @@ import {
   GrantedAccessResponse,
   ProcessProtectedDataParams,
   ProcessProtectedDataResponse,
+  ProcessBulkRequestParams,
+  ProcessBulkRequestResponse,
   ProtectDataParams,
   ProtectedData,
   GetResultFromCompletedTaskParams,
@@ -19,11 +21,15 @@ import {
   TransferResponse,
   WaitForTaskCompletionResponse,
   WaitForTaskCompletionParams,
+  PrepareBulkRequestParams,
+  PrepareBulkRequestResponse,
 } from '../types/index.js';
 import { getGrantedAccess } from './getGrantedAccess.js';
 import { getProtectedData } from './getProtectedData.js';
 import { getResultFromCompletedTask } from './getResultFromCompletedTask.js';
 import { grantAccess } from './grantAccess.js';
+import { prepareBulkRequest } from './prepareBulkRequest.js';
+import { processBulkRequest } from './processBulkRequest.js';
 import { processProtectedData } from './processProtectedData.js';
 import { protectData } from './protectData.js';
 import { revokeAllAccess } from './revokeAllAccess.js';
@@ -83,6 +89,29 @@ class IExecDataProtectorCore extends IExecDataProtectorModule {
     });
   }
 
+  async prepareBulkRequest(
+    args: PrepareBulkRequestParams
+  ): Promise<PrepareBulkRequestResponse> {
+    await this.init();
+    await isValidProvider(this.iexec);
+    return prepareBulkRequest({
+      ...args,
+      iexec: this.iexec,
+    });
+  }
+
+  async processBulkRequest<Params extends ProcessBulkRequestParams>(
+    args: Params
+  ): Promise<ProcessBulkRequestResponse<Params>> {
+    await this.init();
+    await isValidProvider(this.iexec);
+    return processBulkRequest({
+      ...args,
+      iexec: this.iexec,
+      defaultWorkerpool: this.defaultWorkerpool,
+    });
+  }
+
   // ----- READ METHODS -----
   async getProtectedData(
     args?: GetProtectedDataParams

packages/sdk/src/lib/dataProtectorCore/IExecDataProtectorCore.ts

@@ -21,6 +21,7 @@ export const getGrantedAccess = async ({
   isUserStrict = false,
   page,
   pageSize,
+  bulkOnly = false,
 }: IExecConsumer & GetGrantedAccessParams): Promise<GrantedAccessResponse> => {
   const vProtectedData = addressOrEnsSchema()
     .label('protectedData')
@@ -38,6 +39,7 @@ export const getGrantedAccess = async ({
   const vPageSize = numberBetweenSchema(10, 1000)
     .label('pageSize')
     .validateSync(pageSize);
+  const vBulkOnly = booleanSchema().label('bulkOnly').validateSync(bulkOnly);
 
   try {
     const { count, orders } = await iexec.orderbook.fetchDatasetOrderbook({
@@ -48,6 +50,7 @@ export const getGrantedAccess = async ({
       isAppStrict: true,
       page: vPage,
       pageSize: vPageSize,
+      bulkOnly: vBulkOnly,
     });
     const grantedAccess = orders?.map((order) =>
       formatGrantedAccess(order.order, order.remaining)

packages/sdk/src/lib/dataProtectorCore/getGrantedAccess.ts

@@ -1,5 +1,5 @@
 import { ZeroAddress } from 'ethers';
-import { NULL_ADDRESS } from 'iexec/utils';
+import { DATASET_INFINITE_VOLUME, NULL_ADDRESS } from 'iexec/utils';
 import {
   ValidationError,
   WorkflowError,
@@ -9,6 +9,7 @@ import {
 import { formatGrantedAccess } from '../../utils/formatGrantedAccess.js';
 import {
   addressOrEnsSchema,
+  booleanSchema,
   isEnsTest,
   positiveIntegerStringSchema,
   positiveStrictIntegerStringSchema,
@@ -47,8 +48,9 @@ export const grantAccess = async ({
   protectedData,
   authorizedApp,
   authorizedUser,
-  pricePerAccess,
+  pricePerAccess = 0,
   numberOfAccess,
+  allowBulk = false,
   onStatusUpdate = () => {},
 }: IExecConsumer & GrantAccessParams): Promise<GrantedAccess> => {
   const vProtectedData = addressOrEnsSchema()
@@ -62,17 +64,37 @@ export const grantAccess = async ({
   const vAuthorizedUser = addressOrEnsSchema()
     .label('authorizedUser')
     .validateSync(authorizedUser);
-  const vPricePerAccess = positiveIntegerStringSchema()
+  let vPricePerAccess = positiveIntegerStringSchema()
     .label('pricePerAccess')
     .validateSync(pricePerAccess);
-  const vNumberOfAccess = positiveStrictIntegerStringSchema()
+  let vNumberOfAccess = positiveStrictIntegerStringSchema()
     .label('numberOfAccess')
     .validateSync(numberOfAccess);
+  const vAllowBulk = booleanSchema().label('allowBulk').validateSync(allowBulk);
   const vOnStatusUpdate =
     validateOnStatusUpdateCallback<OnStatusUpdateFn<GrantAccessStatuses>>(
       onStatusUpdate
     );
 
+  // Validate consistency between allowBulk, pricePerAccess and numberOfAccess
+  if (vAllowBulk) {
+    if (vPricePerAccess && vPricePerAccess !== '0') {
+      throw new ValidationError(
+        'allowBulk requires pricePerAccess to be 0 or undefined'
+      );
+    }
+    vPricePerAccess = '0';
+    if (
+      vNumberOfAccess &&
+      vNumberOfAccess !== DATASET_INFINITE_VOLUME.toString()
+    ) {
+      throw new ValidationError(
+        `allowBulk requires numberOfAccess to be ${DATASET_INFINITE_VOLUME.toString()} or undefined`
+      );
+    }
+    vNumberOfAccess = DATASET_INFINITE_VOLUME.toString();
+  }
+
   if (vAuthorizedApp && isEnsTest(vAuthorizedApp)) {
     const resolved = await iexec.ens.resolveName(vAuthorizedApp);
     if (!resolved) {
@@ -127,6 +149,7 @@ export const grantAccess = async ({
     title: 'CREATE_DATASET_ORDER',
     isDone: false,
   });
+
   const datasetorder = await iexec.order
     .createDatasetorder({
       dataset: vProtectedData,
@@ -145,6 +168,7 @@ export const grantAccess = async ({
         errorCause: e,
       });
     });
+
   vOnStatusUpdate({
     title: 'CREATE_DATASET_ORDER',
     isDone: true,
@@ -154,13 +178,15 @@ export const grantAccess = async ({
     title: 'PUBLISH_DATASET_ORDER',
     isDone: false,
   });
+
   await iexec.order.publishDatasetorder(datasetorder).catch((e) => {
     handleIfProtocolError(e);
     throw new WorkflowError({
       message: 'Failed to publish data access',
       errorCause: e,
     });
   });
+
   vOnStatusUpdate({
     title: 'PUBLISH_DATASET_ORDER',
     isDone: true,