Merge pull request QwenLM#284 from QwenLM/feat/usage_stats_logging

feat: add usage statistics logging for Qwen integration

Author: tanzhenxin

docs/cli/configuration.md

@@ -523,3 +523,5 @@ You can opt out of usage statistics collection at any time by setting the `usage
   "usageStatisticsEnabled": false
 }
 ```
+
+Note: When usage statistics are enabled, events are sent to an Alibaba Cloud RUM collection endpoint.

packages/core/src/config/config.ts

@@ -43,7 +43,7 @@ import {
   DEFAULT_GEMINI_EMBEDDING_MODEL,
   DEFAULT_GEMINI_FLASH_MODEL,
 } from './models.js';
-import { ClearcutLogger } from '../telemetry/clearcut-logger/clearcut-logger.js';
+import { QwenLogger } from '../telemetry/qwen-logger/qwen-logger.js';
 import { shouldAttemptBrowserLaunch } from '../utils/browser.js';
 import { MCPOAuthConfig } from '../mcp/oauth-provider.js';
 import { IdeClient } from '../ide/ide-client.js';
@@ -360,7 +360,7 @@ export class Config {
     }
 
     if (this.getUsageStatisticsEnabled()) {
-      ClearcutLogger.getInstance(this)?.logStartSessionEvent(
+      QwenLogger.getInstance(this)?.logStartSessionEvent(
         new StartSessionEvent(this),
       );
     } else {

packages/core/src/core/__tests__/openaiTimeoutHandling.test.ts

@@ -15,6 +15,7 @@ vi.mock('openai');
 // Mock logger modules
 vi.mock('../../telemetry/loggers.js', () => ({
   logApiResponse: vi.fn(),
+  logApiError: vi.fn(),
 }));
 
 vi.mock('../../utils/openaiLogger.js', () => ({
@@ -290,28 +291,18 @@ describe('OpenAIContentGenerator Timeout Handling', () => {
   });
 
   describe('token estimation on timeout', () => {
-    it('should estimate tokens even when request times out', async () => {
+    it('should surface a clear timeout error when request times out', async () => {
       const timeoutError = new Error('Request timeout');
       mockOpenAIClient.chat.completions.create.mockRejectedValue(timeoutError);
 
-      // Mock countTokens to return a value
-      const mockCountTokens = vi.spyOn(generator, 'countTokens');
-      mockCountTokens.mockResolvedValue({ totalTokens: 100 });
-
       const request = {
         contents: [{ role: 'user' as const, parts: [{ text: 'Hello world' }] }],
         model: 'gpt-4',
       };
 
-      try {
-        await generator.generateContent(request, 'test-prompt-id');
-      } catch (_error) {
-        // Verify that countTokens was called for estimation
-        expect(mockCountTokens).toHaveBeenCalledWith({
-          contents: request.contents,
-          model: 'gpt-4',
-        });
-      }
+      await expect(
+        generator.generateContent(request, 'test-prompt-id'),
+      ).rejects.toThrow(/Request timeout after \d+s/);
     });
 
     it('should fall back to character-based estimation if countTokens fails', async () => {

packages/core/src/core/geminiChat.ts

@@ -158,14 +158,23 @@ export class GeminiChat {
     prompt_id: string,
     usageMetadata?: GenerateContentResponseUsageMetadata,
     responseText?: string,
+    responseId?: string,
   ): Promise<void> {
+    const authType = this.config.getContentGeneratorConfig()?.authType;
+
+    // Don't log API responses for openaiContentGenerator
+    if (authType === AuthType.QWEN_OAUTH || authType === AuthType.USE_OPENAI) {
+      return;
+    }
+
     logApiResponse(
       this.config,
       new ApiResponseEvent(
+        responseId || `gemini-${Date.now()}`,
         this.config.getModel(),
         durationMs,
         prompt_id,
-        this.config.getContentGeneratorConfig()?.authType,
+        authType,
         usageMetadata,
         responseText,
       ),
@@ -176,18 +185,27 @@ export class GeminiChat {
     durationMs: number,
     error: unknown,
     prompt_id: string,
+    responseId?: string,
   ): void {
     const errorMessage = error instanceof Error ? error.message : String(error);
     const errorType = error instanceof Error ? error.name : 'unknown';
 
+    const authType = this.config.getContentGeneratorConfig()?.authType;
+
+    // Don't log API errors for openaiContentGenerator
+    if (authType === AuthType.QWEN_OAUTH || authType === AuthType.USE_OPENAI) {
+      return;
+    }
+
     logApiError(
       this.config,
       new ApiErrorEvent(
+        responseId,
         this.config.getModel(),
         errorMessage,
         durationMs,
         prompt_id,
-        this.config.getContentGeneratorConfig()?.authType,
+        authType,
         errorType,
       ),
     );
@@ -320,6 +338,7 @@ export class GeminiChat {
         prompt_id,
         response.usageMetadata,
         JSON.stringify(response),
+        response.responseId,
       );
 
       this.sendPromise = (async () => {
@@ -563,6 +582,7 @@ export class GeminiChat {
         prompt_id,
         this.getFinalUsageMetadata(chunks),
         JSON.stringify(chunks),
+        chunks[chunks.length - 1]?.responseId,
       );
     }
     this.recordHistory(inputContent, outputContent);

packages/core/src/core/openaiContentGenerator.test.ts

@@ -23,6 +23,7 @@ vi.mock('openai');
 // Mock logger modules
 vi.mock('../telemetry/loggers.js', () => ({
   logApiResponse: vi.fn(),
+  logApiError: vi.fn(),
 }));
 
 vi.mock('../utils/openaiLogger.js', () => ({

packages/core/src/core/openaiContentGenerator.ts

@@ -22,8 +22,8 @@ import {
 } from '@google/genai';
 import { AuthType, ContentGenerator } from './contentGenerator.js';
 import OpenAI from 'openai';
-import { logApiResponse } from '../telemetry/loggers.js';
-import { ApiResponseEvent } from '../telemetry/types.js';
+import { logApiError, logApiResponse } from '../telemetry/loggers.js';
+import { ApiErrorEvent, ApiResponseEvent } from '../telemetry/types.js';
 import { Config } from '../config/config.js';
 import { openaiLogger } from '../utils/openaiLogger.js';
 
@@ -263,6 +263,7 @@ export class OpenAIContentGenerator implements ContentGenerator {
 
       // Log API response event for UI telemetry
       const responseEvent = new ApiResponseEvent(
+        response.responseId || 'unknown',
         this.model,
         durationMs,
         userPromptId,
@@ -291,41 +292,21 @@ export class OpenAIContentGenerator implements ContentGenerator {
           ? error.message
           : String(error);
 
-      // Estimate token usage even when there's an error
-      // This helps track costs and usage even for failed requests
-      let estimatedUsage;
-      try {
-        const tokenCountResult = await this.countTokens({
-          contents: request.contents,
-          model: this.model,
-        });
-        estimatedUsage = {
-          promptTokenCount: tokenCountResult.totalTokens,
-          candidatesTokenCount: 0, // No completion tokens since request failed
-          totalTokenCount: tokenCountResult.totalTokens,
-        };
-      } catch {
-        // If token counting also fails, provide a minimal estimate
-        const contentStr = JSON.stringify(request.contents);
-        const estimatedTokens = Math.ceil(contentStr.length / 4);
-        estimatedUsage = {
-          promptTokenCount: estimatedTokens,
-          candidatesTokenCount: 0,
-          totalTokenCount: estimatedTokens,
-        };
-      }
-
-      // Log API error event for UI telemetry with estimated usage
-      const errorEvent = new ApiResponseEvent(
+      // Log API error event for UI telemetry
+      const errorEvent = new ApiErrorEvent(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).requestID || 'unknown',
         this.model,
+        errorMessage,
         durationMs,
         userPromptId,
         this.config.getContentGeneratorConfig()?.authType,
-        estimatedUsage,
-        undefined,
-        errorMessage,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).type,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).code,
       );
-      logApiResponse(this.config, errorEvent);
+      logApiError(this.config, errorEvent);
 
       // Log error interaction if enabled
       if (this.config.getContentGeneratorConfig()?.enableOpenAILogging) {
@@ -414,6 +395,7 @@ export class OpenAIContentGenerator implements ContentGenerator {
 
           // Log API response event for UI telemetry
           const responseEvent = new ApiResponseEvent(
+            responses[responses.length - 1]?.responseId || 'unknown',
             this.model,
             durationMs,
             userPromptId,
@@ -445,40 +427,21 @@ export class OpenAIContentGenerator implements ContentGenerator {
               ? error.message
               : String(error);
 
-          // Estimate token usage even when there's an error in streaming
-          let estimatedUsage;
-          try {
-            const tokenCountResult = await this.countTokens({
-              contents: request.contents,
-              model: this.model,
-            });
-            estimatedUsage = {
-              promptTokenCount: tokenCountResult.totalTokens,
-              candidatesTokenCount: 0, // No completion tokens since request failed
-              totalTokenCount: tokenCountResult.totalTokens,
-            };
-          } catch {
-            // If token counting also fails, provide a minimal estimate
-            const contentStr = JSON.stringify(request.contents);
-            const estimatedTokens = Math.ceil(contentStr.length / 4);
-            estimatedUsage = {
-              promptTokenCount: estimatedTokens,
-              candidatesTokenCount: 0,
-              totalTokenCount: estimatedTokens,
-            };
-          }
-
-          // Log API error event for UI telemetry with estimated usage
-          const errorEvent = new ApiResponseEvent(
+          // Log API error event for UI telemetry
+          const errorEvent = new ApiErrorEvent(
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (error as any).requestID || 'unknown',
             this.model,
+            errorMessage,
             durationMs,
             userPromptId,
             this.config.getContentGeneratorConfig()?.authType,
-            estimatedUsage,
-            undefined,
-            errorMessage,
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (error as any).type,
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (error as any).code,
           );
-          logApiResponse(this.config, errorEvent);
+          logApiError(this.config, errorEvent);
 
           // Log error interaction if enabled
           if (this.config.getContentGeneratorConfig()?.enableOpenAILogging) {
@@ -518,40 +481,21 @@ export class OpenAIContentGenerator implements ContentGenerator {
           ? error.message
           : String(error);
 
-      // Estimate token usage even when there's an error in streaming setup
-      let estimatedUsage;
-      try {
-        const tokenCountResult = await this.countTokens({
-          contents: request.contents,
-          model: this.model,
-        });
-        estimatedUsage = {
-          promptTokenCount: tokenCountResult.totalTokens,
-          candidatesTokenCount: 0, // No completion tokens since request failed
-          totalTokenCount: tokenCountResult.totalTokens,
-        };
-      } catch {
-        // If token counting also fails, provide a minimal estimate
-        const contentStr = JSON.stringify(request.contents);
-        const estimatedTokens = Math.ceil(contentStr.length / 4);
-        estimatedUsage = {
-          promptTokenCount: estimatedTokens,
-          candidatesTokenCount: 0,
-          totalTokenCount: estimatedTokens,
-        };
-      }
-
-      // Log API error event for UI telemetry with estimated usage
-      const errorEvent = new ApiResponseEvent(
+      // Log API error event for UI telemetry
+      const errorEvent = new ApiErrorEvent(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).requestID || 'unknown',
         this.model,
+        errorMessage,
         durationMs,
         userPromptId,
         this.config.getContentGeneratorConfig()?.authType,
-        estimatedUsage,
-        undefined,
-        errorMessage,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).type,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (error as any).code,
       );
-      logApiResponse(this.config, errorEvent);
+      logApiError(this.config, errorEvent);
 
       // Allow subclasses to suppress error logging for specific scenarios
       if (!this.shouldSuppressErrorLogging(error, request)) {

packages/core/src/services/loopDetectionService.test.ts

@@ -19,6 +19,8 @@ import { LoopDetectionService } from './loopDetectionService.js';
 
 vi.mock('../telemetry/loggers.js', () => ({
   logLoopDetected: vi.fn(),
+  logApiError: vi.fn(),
+  logApiResponse: vi.fn(),
 }));
 
 const TOOL_CALL_LOOP_THRESHOLD = 5;

packages/core/src/telemetry/clearcut-logger/event-metadata-key.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-// Defines valid event metadata keys for Clearcut logging.
+// Defines valid event metadata keys for Qwen logging.
 export enum EventMetadataKey {
   GEMINI_CLI_KEY_UNKNOWN = 0,
 

packages/core/src/telemetry/integration.test.circular.ts

@@ -9,11 +9,12 @@
  */
 
 import { describe, it, expect } from 'vitest';
-import { ClearcutLogger } from './clearcut-logger/clearcut-logger.js';
+import { QwenLogger } from './qwen-logger/qwen-logger.js';
+import { RumEvent } from './qwen-logger/event-types.js';
 import { Config } from '../config/config.js';
 
 describe('Circular Reference Integration Test', () => {
-  it('should handle HttpsProxyAgent-like circular references in clearcut logging', () => {
+  it('should handle HttpsProxyAgent-like circular references in qwen logging', () => {
     // Create a mock config with proxy
     const mockConfig = {
       getTelemetryEnabled: () => true,
@@ -44,16 +45,20 @@ describe('Circular Reference Integration Test', () => {
     proxyAgentLike.sockets['cloudcode-pa.googleapis.com:443'] = [socketLike];
 
     // Create an event that would contain this circular structure
-    const problematicEvent = {
+    const problematicEvent: RumEvent = {
+      timestamp: Date.now(),
+      event_type: 'exception',
+      type: 'error',
+      name: 'api_error',
       error: new Error('Network error'),
       function_args: {
         filePath: '/test/file.txt',
         httpAgent: proxyAgentLike, // This would cause the circular reference
       },
-    };
+    } as RumEvent;
 
-    // Test that ClearcutLogger can handle this
-    const logger = ClearcutLogger.getInstance(mockConfig);
+    // Test that QwenLogger can handle this
+    const logger = QwenLogger.getInstance(mockConfig);
 
     expect(() => {
       logger?.enqueueLogEvent(problematicEvent);