Fix evals in CI

.github/workflows/evals.yml

@@ -23,6 +23,7 @@ jobs:
         run: |
           echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" > ./apps/sandbox-container/.dev.vars
           echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" > ./apps/workers-bindings/.dev.vars
+          echo "DEV_CLOUDFLARE_API_TOKEN=${{ secrets.DEV_CLOUDFLARE_API_TOKEN }}" >> ./apps/sandbox-container/.dev.vars
           echo "DEV_CLOUDFLARE_API_TOKEN=${{ secrets.DEV_CLOUDFLARE_API_TOKEN }}" >> ./apps/workers-bindings/.dev.vars
       - name: Verify .dev.vars file
         run: |

apps/sandbox-container/evals/utils.ts

@@ -1,9 +1,7 @@
-import { jsonSchemaToZod } from '@n8n/json-schema-to-zod'
 import { MCPClientManager } from 'agents/mcp/client'
-import { streamText, tool } from 'ai'
+import { jsonSchema, streamText, tool } from 'ai'
 import { z } from 'zod'
 
-import type { JsonSchemaObject } from '@n8n/json-schema-to-zod'
 import type { LanguageModelV1, StreamTextResult, ToolCallPart, ToolSet } from 'ai'
 
 export async function initializeClient(): Promise<MCPClientManager> {
@@ -23,10 +21,14 @@ export async function runTask(
 }> {
 	const tools = clientManager.listTools()
 	const toolSet: ToolSet = tools.reduce((acc, v) => {
+		if (!v.inputSchema.properties) {
+			v.inputSchema.properties = {}
+		}
+
 		acc[v.name] = tool({
-			parameters: jsonSchemaToZod(v.inputSchema as JsonSchemaObject),
+			parameters: jsonSchema(v.inputSchema as any),
 			description: v.description,
-			execute: async (args, opts) => {
+			execute: async (args: any, opts) => {
 				try {
 					const res = await clientManager.callTool(
 						{

apps/sandbox-container/package.json

@@ -15,7 +15,7 @@
 		"test": "vitest",
 		"types": "wrangler types --include-env=false",
 		"eval:dev": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest --testTimeout=60000 --config vitest.config.evals.ts'",
-		"eval:server": "concurrently \"tsx container/index.ts\" \"wrangler dev --var \"ENVIRONMENT:test\"\"",
+		"eval:server": "concurrently \"tsx container/index.ts\" \"wrangler dev --var ENVIRONMENT:test --var DEV_DISABLE_OAUTH:true --var DEV_CLOUDFLARE_EMAIL:[email protected]\"",
 		"eval:ci": "start-server-and-test --expect 404 eval:server http://localhost:8976 'vitest run --testTimeout=60000 --config vitest.config.evals.ts'"
 	},
 	"dependencies": {

apps/sandbox-container/server/context.ts

@@ -8,7 +8,7 @@ export interface Env {
 	MCP_SERVER_NAME: string
 	MCP_SERVER_VERSION: string
 	OPENAI_API_KEY: string
-	CONTAINER_MCP_AGENT: DurableObjectNamespace<ContainerMcpAgent>
+	MCP_OBJECT: DurableObjectNamespace<ContainerMcpAgent>
 	CONTAINER_MANAGER: DurableObjectNamespace<ContainerManager>
 	USER_CONTAINER: DurableObjectNamespace<UserContainer>
 	USER_BLOCKLIST: KVNamespace

apps/sandbox-container/server/index.ts

@@ -14,7 +14,6 @@ import { ContainerManager } from './containerManager'
 import { ContainerMcpAgent } from './containerMcp'
 import { UserContainer } from './userContainer'
 
-import type { McpAgent } from 'agents/mcp'
 import type { AuthProps } from '@repo/mcp-common/src/cloudflare-oauth-handler'
 import type { Env } from './context'
 
@@ -40,30 +39,16 @@ const ContainerScopes = {
 
 export default {
 	fetch: async (req: Request, env: Env, ctx: ExecutionContext) => {
-		// @ts-ignore
-		if (env.ENVIRONMENT === 'test') {
-			ctx.props = {
-				accessToken: 'foobar',
-				user: {
-					id: '123def',
-					email: '[email protected]',
-				},
-				accounts: [],
-			} as Props
-			return ContainerMcpAgent.mount('/sse', { binding: 'CONTAINER_MCP_AGENT' }).fetch(
-				req,
-				env as Record<string, DurableObjectNamespace<McpAgent> | any>,
-				ctx
-			)
-		}
-
-		if (env.ENVIRONMENT === 'dev' && env.DEV_DISABLE_OAUTH === 'true') {
+		if (
+			(env.ENVIRONMENT === 'dev' || env.ENVIRONMENT === 'test') &&
+			env.DEV_DISABLE_OAUTH === 'true'
+		) {
 			return await handleDevMode(ContainerMcpAgent, req, env, ctx)
 		}
 
 		return new OAuthProvider({
 			apiRoute: ['/mcp', '/sse'],
-			apiHandler: createApiHandler(ContainerMcpAgent, { binding: 'CONTAINER_MCP_AGENT' }),
+			apiHandler: createApiHandler(ContainerMcpAgent),
 			// @ts-ignore
 			defaultHandler: createAuthHandlers({ scopes: ContainerScopes, metrics }),
 			authorizeEndpoint: '/oauth/authorize',

apps/sandbox-container/wrangler.jsonc

@@ -17,7 +17,7 @@
 		"bindings": [
 			{
 				"class_name": "ContainerMcpAgent",
-				"name": "CONTAINER_MCP_AGENT"
+				"name": "MCP_OBJECT"
 			},
 			{
 				"class_name": "ContainerManager",
@@ -91,7 +91,7 @@
 				"bindings": [
 					{
 						"class_name": "ContainerMcpAgent",
-						"name": "CONTAINER_MCP_AGENT"
+						"name": "MCP_OBJECT"
 					},
 					{
 						"class_name": "ContainerManager",
@@ -144,7 +144,7 @@
 				"bindings": [
 					{
 						"class_name": "ContainerMcpAgent",
-						"name": "CONTAINER_MCP_AGENT"
+						"name": "MCP_OBJECT"
 					},
 					{
 						"class_name": "ContainerManager",

apps/workers-bindings/evals/utils.ts

@@ -60,7 +60,7 @@ export async function runTask(
 	})
 
 	// we need to consume the fill stream, so this is empty
-	/* eslint-ignore no-empty */
+	// eslint-disable-next-line no-empty
 	for await (const _ of res.fullStream) {
 	}