feat: implement experimental xp-forwarded-for support

Author: karashiiro

src/auth-user.ts

@@ -9,6 +9,7 @@ import { Check } from '@sinclair/typebox/value';
 import * as OTPAuth from 'otpauth';
 import { FetchParameters } from './api-types';
 import debug from 'debug';
+import { generateXPFFHeader } from './xpff';
 
 const log = debug('twitter-scraper:auth-user');
 
@@ -310,26 +311,30 @@ export class TwitterUserAuth extends TwitterGuestAuth {
     }
   }
 
-  async installCsrfToken(headers: Headers): Promise<void> {
-    const cookies = await this.getCookies();
-    const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0');
-    if (xCsrfToken) {
-      headers.set('x-csrf-token', xCsrfToken.value);
-    }
-  }
-
   async installTo(headers: Headers): Promise<void> {
     headers.set('authorization', `Bearer ${this.bearerToken}`);
-    const cookie = await this.getCookieString();
-    headers.set('cookie', cookie);
-    if (this.guestToken) {
-      headers.set('x-guest-token', this.guestToken);
-    }
     headers.set(
       'user-agent',
       'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
     );
+
+    if (this.guestToken) {
+      // Guest token is optional for authenticated users
+      headers.set('x-guest-token', this.guestToken);
+    }
+
     await this.installCsrfToken(headers);
+
+    if (this.options?.experimental?.xpff) {
+      const guestId = await this.guestId();
+      if (guestId != null) {
+        const xpffHeader = await generateXPFFHeader(guestId);
+        headers.set('x-xp-forwarded-for', xpffHeader);
+      }
+    }
+
+    const cookie = await this.getCookieString();
+    headers.set('cookie', cookie);
   }
 
   private async initLogin(): Promise<FlowTokenResult> {

src/auth.ts

@@ -10,13 +10,17 @@ import {
 } from './rate-limit';
 import { AuthenticationError } from './errors';
 import debug from 'debug';
+import { generateXPFFHeader } from './xpff';
 
 const log = debug('twitter-scraper:auth');
 
 export interface TwitterAuthOptions {
   fetch: typeof fetch;
   transform: Partial<FetchTransformOptions>;
   rateLimitStrategy: RateLimitStrategy;
+  experimental: {
+    xpff?: boolean;
+  };
 }
 
 export interface TwitterAuth {
@@ -33,6 +37,9 @@ export interface TwitterAuth {
    */
   cookieJar(): CookieJar;
 
+  /**
+   * Returns the current cookies.
+   */
   getCookies(): Promise<Cookie[]>;
 
   /**
@@ -187,13 +194,25 @@ export class TwitterGuestAuth implements TwitterAuth {
       'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
     );
 
+    await this.installCsrfToken(headers);
+
+    if (this.options?.experimental?.xpff) {
+      const guestId = await this.guestId();
+      if (guestId != null) {
+        const xpffHeader = await generateXPFFHeader(guestId);
+        headers.set('x-xp-forwarded-for', xpffHeader);
+      }
+    }
+
+    headers.set('cookie', await this.getCookieString());
+  }
+
+  async installCsrfToken(headers: Headers): Promise<void> {
     const cookies = await this.getCookies();
     const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0');
     if (xCsrfToken) {
       headers.set('x-csrf-token', xCsrfToken.value);
     }
-
-    headers.set('cookie', await this.getCookieString());
   }
 
   protected async setCookie(key: string, value: string): Promise<void> {
@@ -238,6 +257,12 @@ export class TwitterGuestAuth implements TwitterAuth {
       : 'https://x.com';
   }
 
+  protected async guestId(): Promise<string | null> {
+    const cookies = await this.getCookies();
+    const guestIdCookie = cookies.find((cookie) => cookie.key === 'guest_id');
+    return guestIdCookie ? guestIdCookie.value : null;
+  }
+
   /**
    * Updates the authentication state with a new guest token from the Twitter API.
    */

src/scraper.ts

@@ -67,6 +67,16 @@ export interface ScraperOptions {
    * A handling strategy for rate limits (HTTP 429).
    */
   rateLimitStrategy: RateLimitStrategy;
+
+  /**
+   * Experimental features that may be added, changed, or removed at any time. Use with caution.
+   */
+  experimental: {
+    /**
+     * Enables the generation of the `x-xp-forwarded-for` header on requests. This may resolve some errors.
+     */
+    xpff: boolean;
+  };
 }
 
 /**
@@ -595,6 +605,9 @@ export class Scraper {
       fetch: this.options?.fetch,
       transform: this.options?.transform,
       rateLimitStrategy: this.options?.rateLimitStrategy,
+      experimental: {
+        xpff: this.options?.experimental?.xpff,
+      },
     };
   }
 

src/xpff.ts

@@ -0,0 +1,100 @@
+import debug from 'debug';
+
+const log = debug('twitter-scraper:xpff');
+
+let isoCrypto: Crypto | null = null;
+
+function getCrypto(): Crypto {
+  if (isoCrypto != null) {
+    return isoCrypto;
+  }
+
+  // In Node.js, the global `crypto` object is only available from v19.0.0 onwards.
+  // For earlier versions, we need to import the 'crypto' module.
+  if (typeof crypto === 'undefined') {
+    log('Global crypto is undefined, importing from crypto module...');
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const { webcrypto } = require('crypto');
+    isoCrypto = webcrypto;
+    return webcrypto;
+  }
+  isoCrypto = crypto;
+  return crypto;
+}
+
+async function sha256(message: string): Promise<Uint8Array> {
+  const msgBuffer = new TextEncoder().encode(message);
+  const hashBuffer = await getCrypto().subtle.digest('SHA-256', msgBuffer);
+  return new Uint8Array(hashBuffer);
+}
+
+// https://stackoverflow.com/a/40031979
+function buf2hex(buffer: ArrayBuffer): string {
+  return [...new Uint8Array(buffer)]
+    .map((x) => x.toString(16).padStart(2, '0'))
+    .join('');
+}
+
+// Adapted from https://github.com/dsekz/twitter-x-xp-forwarded-for-header
+export class XPFFHeaderGenerator {
+  constructor(private readonly seed: string) {}
+
+  private async deriveKey(guestId: string): Promise<Uint8Array> {
+    const combined = `${this.seed}${guestId}`;
+    const result = await sha256(combined);
+    return result;
+  }
+
+  async generateHeader(plaintext: string, guestId: string): Promise<string> {
+    log(`Generating XPFF key for guest ID: ${guestId}`);
+    const key = await this.deriveKey(guestId);
+    const nonce = getCrypto().getRandomValues(new Uint8Array(12));
+    const cipher = await getCrypto().subtle.importKey(
+      'raw',
+      key,
+      { name: 'AES-GCM' },
+      false,
+      ['encrypt'],
+    );
+    const encrypted = await getCrypto().subtle.encrypt(
+      {
+        name: 'AES-GCM',
+        iv: nonce,
+      },
+      cipher,
+      new TextEncoder().encode(plaintext),
+    );
+
+    // Combine nonce and encrypted data
+    const combined = new Uint8Array(nonce.length + encrypted.byteLength);
+    combined.set(nonce);
+    combined.set(new Uint8Array(encrypted), nonce.length);
+    const result = buf2hex(combined);
+
+    log(`XPFF header generated for guest ID ${guestId}: ${result}`);
+
+    return result;
+  }
+}
+
+const xpffBaseKey =
+  '0e6be1f1e21ffc33590b888fd4dc81b19713e570e805d4e5df80a493c9571a05';
+
+function xpffPlain(): string {
+  const timestamp = Date.now();
+  return JSON.stringify({
+    navigator_properties: {
+      hasBeenActive: 'true',
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+      webdriver: 'false',
+    },
+    created_at: timestamp,
+  });
+}
+
+export async function generateXPFFHeader(guestId: string): Promise<string> {
+  const generator = new XPFFHeaderGenerator(xpffBaseKey);
+  const plaintext = xpffPlain();
+  return generator.generateHeader(plaintext, guestId);
+}