Skip to content

Commit 7c8ae1c

Browse files
cmackenzie1emily-shen
cmackenzie1
and
emily-shen
authored
feat: Use OAuth flow to generate R2 tokens for Pipelines (#7534)
* feat: Use OAuth flow to generate R2 tokens for Pipelines This commit changes the generateR2Tokens flow which will direct the user to the web browser to perform a OAuth flow to grant the Workers Pipelines client the ability to generate R2 tokens on behalf of the user. This will only run if the user does not provide the credentials as CLI parameters. Due to requiring user interactivity, and reliance on the callbacks, there is no easy way to support a "headless" mode for `wrangler pipelines create` (or `update`) unless the user provides the tokens as arguments. The same applies for testing this flow, which can only be done manually at this time. * fix: add forced delayed to allow r2 tokens time to sync Create odd-ducks-attack.md * Add docs around bespoke OAuth solution * fix: Wait for R2 token to sync After creating an R2 token, there is a slight delay before if can be used. Previously we would sleep for some amount of time, but this method is really sensitive to latency. Instead, use the S3 SDK and try using the token until we exhaust all attempts, or we finally succeed in using it. Each failure results in a constant backoff of 1 second. This commit does add the dependency `@aws-sdk/client-s3`. * fix pnpm-lock.yaml * fix: clear timeout if token retrieved successfully This uses the promise based version of `setTimeout` from NodeJS and registers the AbortController to handle cancellation signal. The http server `.close()` method is also registered to the abort controller for cleanup as `controller.abort()` is always called before returning the result. --------- Co-authored-by: emily-shen <[email protected]>
1 parent b8e5f63 commit 7c8ae1c

File tree

6 files changed

+1378
-140
lines changed

6 files changed

+1378
-140
lines changed

‎.changeset/odd-ducks-attack.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"wrangler": patch
3+
---
4+
5+
feat: Use OAuth flow to generate R2 tokens for Pipelines

‎packages/wrangler/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
"type:tests": "tsc -p ./src/__tests__/tsconfig.json && tsc -p ./e2e/tsconfig.json"
7070
},
7171
"dependencies": {
72+
"@aws-sdk/client-s3": "^3.721.0",
7273
"@cloudflare/kv-asset-handler": "workspace:*",
7374
"@esbuild-plugins/node-globals-polyfill": "^0.2.3",
7475
"@esbuild-plugins/node-modules-polyfill": "^0.2.2",

‎packages/wrangler/src/__tests__/pipelines.test.ts

Lines changed: 7 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -46,76 +46,7 @@ describe("pipelines", () => {
4646
endpoint: "https://0001.pipelines.cloudflarestorage.com",
4747
} satisfies Pipeline;
4848

49-
function mockCreateR2Token(bucket: string) {
50-
const requests = { count: 0 };
51-
msw.use(
52-
http.get(
53-
"*/accounts/:accountId/r2/buckets/:bucket",
54-
async ({ params }) => {
55-
expect(params.accountId).toEqual("some-account-id");
56-
expect(params.bucket).toEqual(bucket);
57-
requests.count++;
58-
return HttpResponse.json(
59-
{
60-
success: true,
61-
errors: [],
62-
messages: [],
63-
result: null,
64-
},
65-
{ status: 200 }
66-
);
67-
},
68-
{ once: true }
69-
),
70-
http.get(
71-
"*/user/tokens/permission_groups",
72-
async () => {
73-
requests.count++;
74-
return HttpResponse.json(
75-
{
76-
success: true,
77-
errors: [],
78-
messages: [],
79-
result: [
80-
{
81-
id: "2efd5506f9c8494dacb1fa10a3e7d5b6",
82-
name: "Workers R2 Storage Bucket Item Write",
83-
description:
84-
"Grants write access to Cloudflare R2 Bucket Scoped Storage",
85-
scopes: ["com.cloudflare.edge.r2.bucket"],
86-
},
87-
],
88-
},
89-
{ status: 200 }
90-
);
91-
},
92-
{ once: true }
93-
),
94-
http.post(
95-
"*/user/tokens",
96-
async () => {
97-
requests.count++;
98-
return HttpResponse.json(
99-
{
100-
success: true,
101-
errors: [],
102-
messages: [],
103-
result: {
104-
id: "service-token-id",
105-
name: "my-service-token",
106-
value: "my-secret-value",
107-
},
108-
},
109-
{ status: 200 }
110-
);
111-
},
112-
{ once: true }
113-
)
114-
);
115-
return requests;
116-
}
117-
118-
function mockCreeatR2TokenFailure(bucket: string) {
49+
function mockCreateR2TokenFailure(bucket: string) {
11950
const requests = { count: 0 };
12051
msw.use(
12152
http.get(
@@ -310,6 +241,7 @@ describe("pipelines", () => {
310241
);
311242
return requests;
312243
}
244+
313245
beforeAll(() => {
314246
__testSkipDelays();
315247
});
@@ -380,15 +312,6 @@ describe("pipelines", () => {
380312
`);
381313
});
382314

383-
it("should create a pipeline", async () => {
384-
const tokenReq = mockCreateR2Token("test-bucket");
385-
const requests = mockCreateRequest("my-pipeline");
386-
await runWrangler("pipelines create my-pipeline --r2 test-bucket");
387-
388-
expect(tokenReq.count).toEqual(3);
389-
expect(requests.count).toEqual(1);
390-
});
391-
392315
it("should create a pipeline with explicit credentials", async () => {
393316
const requests = mockCreateRequest("my-pipeline");
394317
await runWrangler(
@@ -398,7 +321,7 @@ describe("pipelines", () => {
398321
});
399322

400323
it("should fail a missing bucket", async () => {
401-
const requests = mockCreeatR2TokenFailure("bad-bucket");
324+
const requests = mockCreateR2TokenFailure("bad-bucket");
402325
await expect(
403326
runWrangler("pipelines create bad-pipeline --r2 bad-bucket")
404327
).rejects.toThrowError();
@@ -540,22 +463,21 @@ describe("pipelines", () => {
540463

541464
it("should update a pipeline with new bucket", async () => {
542465
const pipeline: Pipeline = samplePipeline;
543-
const tokenReq = mockCreateR2Token("new-bucket");
544466
mockShowRequest(pipeline.name, pipeline);
545467

546468
const update = JSON.parse(JSON.stringify(pipeline));
547469
update.destination.path.bucket = "new_bucket";
548470
update.destination.credentials = {
549471
endpoint: "https://some-account-id.r2.cloudflarestorage.com",
550472
access_key_id: "service-token-id",
551-
secret_access_key:
552-
"be22cbae9c1585c7b61a92fdb75afd10babd535fb9b317f90ac9a9ca896d02d7",
473+
secret_access_key: "my-secret-access-key",
553474
};
554475
const updateReq = mockUpdateRequest(update.name, update);
555476

556-
await runWrangler("pipelines update my-pipeline --r2 new-bucket");
477+
await runWrangler(
478+
"pipelines update my-pipeline --r2 new-bucket --access-key-id service-token-id --secret-access-key my-secret-access-key"
479+
);
557480

558-
expect(tokenReq.count).toEqual(3);
559481
expect(updateReq.count).toEqual(1);
560482
});
561483

‎packages/wrangler/src/pipelines/client.ts

Lines changed: 98 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
1+
import assert from "node:assert";
12
import { createHash } from "node:crypto";
3+
import http from "node:http";
4+
import { setTimeout as setTimeoutPromise } from "node:timers/promises";
25
import { fetchResult } from "../cfetch";
6+
import { getCloudflareApiEnvironmentFromEnv } from "../environment-variables/misc-variables";
7+
import { UserError } from "../errors";
8+
import { logger } from "../logger";
9+
import openInBrowser from "../open-in-browser";
310
import type { R2BucketInfo } from "../r2/helpers";
411

512
// ensure this is in sync with:
@@ -96,44 +103,102 @@ export type PermissionGroup = {
96103
scopes: string[];
97104
};
98105

99-
// Generate a Service Token to write to R2 for a pipeline
106+
interface S3AccessKey {
107+
accessKeyId: string;
108+
secretAccessKey: string;
109+
}
110+
111+
/**
112+
* Generate an R2 service token for the given account ID, bucket name, and pipeline name.
113+
*
114+
* This function kicks off its own OAuth process using the Workers Pipelines OAuth client requesting the scope
115+
* `pipelines:setup`. Once the user confirms, our OAuth callback endpoint will validate the request, exchange the
116+
* authorization code and return a bucket-scoped R2 token.
117+
*
118+
* This OAuth flow is distinct from the one used in `wrangler login` to ensure these tokens are generated server-side
119+
* and that only the tokens of concern are returned to the user.
120+
* @param accountId
121+
* @param bucketName
122+
* @param pipelineName
123+
*/
100124
export async function generateR2ServiceToken(
101-
label: string,
102125
accountId: string,
103-
bucket: string
104-
): Promise<ServiceToken> {
105-
const res = await fetchResult<PermissionGroup[]>(
106-
`/user/tokens/permission_groups`,
107-
{
108-
method: "GET",
109-
}
110-
);
111-
const perm = res.find(
112-
(g) => g.name == "Workers R2 Storage Bucket Item Write"
113-
);
114-
if (!perm) {
115-
throw new Error("Missing R2 Permissions");
116-
}
126+
bucketName: string,
127+
pipelineName: string
128+
): Promise<S3AccessKey> {
129+
// TODO: Refactor into startHttpServerWithTimeout function and update `getOauthToken`
130+
const controller = new AbortController();
131+
const signal = controller.signal;
117132

118-
// generate specific bucket write token for pipeline
119-
const body = JSON.stringify({
120-
policies: [
121-
{
122-
effect: "allow",
123-
permission_groups: [{ id: perm.id }],
124-
resources: {
125-
[`com.cloudflare.edge.r2.bucket.${accountId}_default_${bucket}`]: "*",
126-
},
127-
},
128-
],
129-
name: label,
130-
});
133+
// Create timeout promise to prevent hanging forever
134+
const timeoutPromise = setTimeoutPromise(120000, "timeout", { signal });
131135

132-
return await fetchResult<ServiceToken>(`/user/tokens`, {
133-
method: "POST",
134-
headers: API_HEADERS,
135-
body,
136+
// Create server promise to handle the callback and register the cleanup handler on the controller
137+
const serverPromise = new Promise<S3AccessKey>((resolve, reject) => {
138+
const server = http.createServer(async (request, response) => {
139+
assert(request.url, "This request doesn't have a URL"); // This should never happen
140+
141+
if (request.method !== "GET") {
142+
response.writeHead(405);
143+
response.end("Method not allowed.");
144+
return;
145+
}
146+
147+
const { pathname, searchParams } = new URL(
148+
request.url,
149+
`http://${request.headers.host}`
150+
);
151+
152+
if (pathname !== "/") {
153+
response.writeHead(404);
154+
response.end("Not found.");
155+
return;
156+
}
157+
158+
// Retrieve values from the URL parameters
159+
const accessKeyId = searchParams.get("access-key-id");
160+
const secretAccessKey = searchParams.get("secret-access-key");
161+
162+
if (!accessKeyId || !secretAccessKey) {
163+
reject(new UserError("Missing required URL parameters"));
164+
return;
165+
}
166+
167+
resolve({ accessKeyId, secretAccessKey } as S3AccessKey);
168+
// Do a final redirect to "clear" the URL of the sensitive URL parameters that were returned.
169+
response.writeHead(307, {
170+
Location:
171+
"https://welcome.developers.workers.dev/wrangler-oauth-consent-granted",
172+
});
173+
response.end();
174+
});
175+
176+
// Register cleanup handler
177+
signal.addEventListener("abort", () => {
178+
server.close();
179+
});
180+
server.listen(8976, "localhost");
136181
});
182+
183+
const env = getCloudflareApiEnvironmentFromEnv();
184+
const oauthDomain =
185+
env === "staging"
186+
? "oauth.pipelines-staging.cloudflare.com"
187+
: "oauth.pipelines.cloudflare.com";
188+
189+
const urlToOpen = `https://${oauthDomain}/oauth/login?accountId=${accountId}&bucketName=${bucketName}&pipelineName=${pipelineName}`;
190+
logger.log(`Opening a link in your default browser: ${urlToOpen}`);
191+
await openInBrowser(urlToOpen);
192+
193+
const result = await Promise.race([timeoutPromise, serverPromise]);
194+
controller.abort();
195+
if (result === "timeout") {
196+
throw new UserError(
197+
"Timed out waiting for authorization code, please try again."
198+
);
199+
}
200+
201+
return result as S3AccessKey;
137202
}
138203

139204
// Get R2 bucket information from v4 API

0 commit comments

Comments
 (0)