feat: attempt to parse text as json, log failures

Author: karashiiro

package.json

@@ -7,7 +7,7 @@
     "scraper",
     "crawler"
   ],
-  "version": "0.19.0",
+  "version": "0.19.1",
   "main": "dist/default/cjs/index.js",
   "types": "./dist/types/index.d.ts",
   "exports": {

src/api.ts

@@ -105,7 +105,7 @@ export async function requestApi<T>(
     };
   }
 
-  const value: T = await res.json();
+  const value: T = await flexParseJson(res);
   if (res.headers.get('x-rate-limit-incoming') == '0') {
     auth.deleteToken();
     return { success: true, value };
@@ -114,6 +114,17 @@ export async function requestApi<T>(
   }
 }
 
+export async function flexParseJson<T>(res: Response): Promise<T> {
+  try {
+    return await res.json();
+  } catch {
+    log('Failed to parse response as JSON, trying text parse...');
+    const text = await res.text();
+    log('Response text:', text);
+    return JSON.parse(text);
+  }
+}
+
 /** @internal */
 export function addApiFeatures(o: object) {
   return {

src/auth-user.ts

@@ -1,5 +1,5 @@
 import { TwitterAuthOptions, TwitterGuestAuth } from './auth';
-import { requestApi } from './api';
+import { flexParseJson, requestApi } from './api';
 import { CookieJar } from 'tough-cookie';
 import { updateCookieJar } from './requests';
 import { Headers } from 'headers-polyfill';
@@ -325,6 +325,10 @@ export class TwitterUserAuth extends TwitterGuestAuth {
     if (this.guestToken) {
       headers.set('x-guest-token', this.guestToken);
     }
+    headers.set(
+      'user-agent',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    );
     await this.installCsrfToken(headers);
   }
 
@@ -659,7 +663,7 @@ export class TwitterUserAuth extends TwitterGuestAuth {
       return { status: 'error', err: await ApiError.fromResponse(res) };
     }
 
-    const flow: TwitterUserAuthFlowResponse = await res.json();
+    const flow: TwitterUserAuthFlowResponse = await flexParseJson(res);
     if (flow?.flow_token == null) {
       return {
         status: 'error',

src/auth.ts

@@ -2,7 +2,7 @@ import { Cookie, CookieJar, MemoryCookieStore } from 'tough-cookie';
 import { updateCookieJar } from './requests';
 import { Headers } from 'headers-polyfill';
 import fetch from 'cross-fetch';
-import { FetchTransformOptions } from './api';
+import { FetchTransformOptions, flexParseJson } from './api';
 import {
   RateLimitEvent,
   RateLimitStrategy,
@@ -182,6 +182,10 @@ export class TwitterGuestAuth implements TwitterAuth {
 
     headers.set('authorization', `Bearer ${this.bearerToken}`);
     headers.set('x-guest-token', token);
+    headers.set(
+      'user-agent',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+    );
 
     const cookies = await this.getCookies();
     const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0');
@@ -259,7 +263,7 @@ export class TwitterGuestAuth implements TwitterAuth {
       throw new AuthenticationError(await res.text());
     }
 
-    const o = await res.json();
+    const o = await flexParseJson<any>(res);
     if (o == null || o['guest_token'] == null) {
       throw new AuthenticationError('guest_token not found.');
     }