feat!: add AWS S3 Support (#165)

* chore: yarn add aws-sdk

* refactor: add storageProvider interface in router

* feat: add googleStorage.ts

* feat: add awsS3Storage

* refactor: adapt standaloneServer to new router refactoring

* chore: update README with AWS S3

* feat: export standard storage providers

Author: MattiasMTS

README.md

@@ -6,26 +6,29 @@
 
 <!-- toc -->
 
--   [Features](#features)
--   [Limitations](#limitations)
--   [Screenshots](#screenshots)
--   [Setup](#setup)
--   [Usage](#usage)
+- [Features](#features)
+- [Limitations](#limitations)
+- [Screenshots](#screenshots)
+- [Setup](#setup)
+- [Usage](#usage)
 
 ## Features
 
 - List all dbt models and tests
 - Get details on dbt models and tests like:
-   - Documentations
-   - Stats
-   - Columns
-   - Dependency graph
-   - Code source (raw and compiled) 
+  - Documentations
+  - Stats
+  - Columns
+  - Dependency graph
+  - Code source (raw and compiled)
 
 ## Limitations
 
-**This version only support Google Cloud Storage as backend to store `manifest.json`
-and `catalog.json` files.**
+As of now, the plugin only support the following backends:
+
+- [x] Google Cloud Storage
+- [x] AWS S3
+- [ ] Azure Blob Storage
 
 ## Screenshots
 
@@ -59,16 +62,16 @@ yarn --cwd packages/backend add @iiben_orgii/backstage-plugin-dbt-backend
 ```tsx
 // packages/app/src/components/catalog/EntityPage.tsx
 
-import { DbtPage, isDBTAvailable } from "@iiben_orgii/backstage-plugin-dbt"
+import { DbtPage, isDBTAvailable } from "@iiben_orgii/backstage-plugin-dbt";
 
 // Farther down at the serviceEntityPage declaration
 const serviceEntityPage = (
-    <EntityLayout>
-        {/* Place the following section where you want the tab to appear */}
-        <EntityLayout.Route if={isDBTAvailable} path="/dbt" title="dbt">
-            <DbtPage />
-        </EntityLayout.Route>
-    </EntityLayout>
+  <EntityLayout>
+    {/* Place the following section where you want the tab to appear */}
+    <EntityLayout.Route if={isDBTAvailable} path="/dbt" title="dbt">
+      <DbtPage />
+    </EntityLayout.Route>
+  </EntityLayout>
 );
 ```
 
@@ -78,15 +81,21 @@ const serviceEntityPage = (
 
 ```ts
 // packages/backend/src/plugins/dbt.ts
-import { createRouter } from '@iiben_orgii/backstage-plugin-dbt-backend';
-import { Router } from 'express';
-import { PluginEnvironment } from '../types';
+import {
+  createRouter,
+  GoogleStorageProvider,
+} from "@iiben_orgii/backstage-plugin-dbt-backend";
+import { Router } from "express";
+import { PluginEnvironment } from "../types";
+
+const storageProvider = new GoogleStorageProvider();
 
 export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
   return await createRouter({
     logger: env.logger,
+    storageProvider: storageProvider,
   });
 }
 ```
@@ -97,14 +106,14 @@ then you have to add the route as follows:
 
 ```ts
 // packages/backend/src/index.ts
-import dbt from './plugins/dbt';
+import dbt from "./plugins/dbt";
 
 async function main() {
-    //...
-    const dbtEnv = useHotMemoize(module, () => createEnv('dbt'));
-    //...
-    apiRouter.use('/dbt', await dbt(dbtEnv));
-    //...
+  //...
+  const dbtEnv = useHotMemoize(module, () => createEnv("dbt"));
+  //...
+  apiRouter.use("/dbt", await dbt(dbtEnv));
+  //...
 }
 ```
 
@@ -115,15 +124,16 @@ async function main() {
 You can define one bucket with all your manifest and catalog files.
 
 Add a file `application/packages/app/config.d.ts`:
+
 ```ts
 export interface Config {
-    dbtdoc: {
-        /**
-         * Frontend root URL
-         * @visibility frontend
-         */
-        bucket: string;
-    };
+  dbtdoc: {
+    /**
+     * Frontend root URL
+     * @visibility frontend
+     */
+    bucket: string;
+  };
 }
 ```
 
@@ -138,6 +148,7 @@ Update the file `application/packages/app/package.json` with
 ```
 
 Then you can add to your `app-config.yaml`:
+
 ```yaml
 dbtdoc:
   bucket: your-bucket-123
@@ -165,7 +176,8 @@ metadata:
 **Following path must be respect regardless your bucket setup (single or multi).**
 
 You can upload your `manifest.json` and `catalog.json` to a GCS Bucket as follow:
+
 - `{dbtdoc-bucket}/{kind}/{name}/manifest.json`
 - `{dbtdoc-bucket}/{kind}/{name}/catalog.json`
 
-For authentification to GCS Bucket, the plugin use ADC credentials [https://cloud.google.com/docs/authentication/provide-credentials-adc](https://cloud.google.com/docs/authentication/provide-credentials-adc).
+For authentification to GCS Bucket, the plugin use ADC credentials [https://cloud.google.com/docs/authentication/provide-credentials-adc](https://cloud.google.com/docs/authentication/provide-credentials-adc).

packages/dbt-backend/package.json

@@ -32,6 +32,7 @@
     "@backstage/backend-common": "^0.19.1",
     "@backstage/config": "^1.0.8",
     "@types/express": "*",
+    "aws-sdk": "^2.1472.0",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "node-fetch": "^3.3.1",

packages/dbt-backend/src/index.ts

@@ -1 +1,3 @@
-export * from './service/router';
+export * from "./service/router";
+export * from "./service/awsS3Storage";
+export * from "./service/googleStorage";

packages/dbt-backend/src/service/awsS3Storage.ts

@@ -0,0 +1,21 @@
+import { S3 } from 'aws-sdk';
+
+/**
+ * AwsS3StorageProvider is a class that provides functionality for downloading files
+ * from Amazon S3 storage using the AWS SDK.
+ */
+export class AwsS3StorageProvider {
+  private s3 = new S3();
+
+  /**
+   * Downloads a file from Amazon S3 storage.
+   * @param bucket - The name of the S3 bucket.
+   * @param filePath - The path to the file within the bucket.
+   * @returns A Promise that resolves to the file's contents as a Buffer.
+   */
+  async downloadFile(bucket: string, filePath: string): Promise<Buffer> {
+    const params = { Bucket: bucket, Key: filePath };
+    const { Body } = await this.s3.getObject(params).promise();
+    return Body as Buffer;
+  }
+}

packages/dbt-backend/src/service/googleStorage.ts

@@ -0,0 +1,20 @@
+import { Storage } from '@google-cloud/storage';
+
+/**
+ * GoogleStorageProvider is a class that provides functionality for downloading files
+ * from Google Cloud Storage.
+ */
+export class GoogleStorageProvider {
+  private storage = new Storage();
+
+  /**
+   * Downloads a file from Google Cloud Storage.
+   * @param bucket - The name of the storage bucket.
+   * @param filePath - The path to the file within the bucket.
+   * @returns A Promise that resolves to the file's contents as a Buffer.
+   */
+  async downloadFile(bucket: string, filePath: string): Promise<Buffer> {
+    const [contents] = await this.storage.bucket(bucket).file(filePath).download();
+    return contents;
+  }
+}

packages/dbt-backend/src/service/router.ts

@@ -1,40 +1,59 @@
 import { errorHandler } from '@backstage/backend-common';
-import express from 'express';
+import express, { Request, Response } from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 
-const { Storage } = require('@google-cloud/storage');
-
-// Creates a client
-const storage = new Storage();
-
+/**
+ * Abstracts the storage provider to allow for different
+ * storage providers to be used.
+ */
+export interface StorageProvider {
+  /**
+   * Download a file from the storage provider.
+   * @param bucket - The name of the storage bucket.
+   * @param filePath - The path to the file within the bucket.
+   * @returns A Promise that resolves to the file's contents as a Buffer.
+   */
+  downloadFile(bucket: string, filePath: string): Promise<Buffer>;
+}
 
 export interface RouterOptions {
   logger: Logger;
+  storageProvider: StorageProvider;
 }
 
-export async function createRouter(
-  options: RouterOptions,
-): Promise<express.Router> {
-  const { logger } = options;
-
+/**
+ * Creates an Express router for handling storage-related requests.
+ * @param options - Configuration options for the router.
+ * @returns An Express router that handles storage requests.
+ */
+export function createRouter(options: RouterOptions): express.Router {
+  const { logger, storageProvider } = options;
   const router = Router();
   router.use(express.json());
 
-  router.get('/manifest/:bucket/:kind/:name', async (req, response) => {
-    const file_path = `${req.params.kind}/${req.params.name}/manifest.json`
-    logger.info(`Get manifest under ${req.params.bucket}/${req.params.kind}/${req.params.name}/manifest.json`)
-    const contents = await storage.bucket(req.params.bucket).file(file_path).download();
-    const result = JSON.parse(contents.toString())
-    response.json(result);
+  async function handleRequest(req: Request, res: Response, type: 'manifest' | 'catalog') {
+    const { bucket, kind, name } = req.params;
+    const filePath = `${kind}/${name}/${type}.json`;
+    const fullPath = `${bucket}/${filePath}`;
+
+    try {
+      logger.info(`Get ${type} under ${fullPath}`);
+      const contents = await storageProvider.downloadFile(bucket, filePath);
+      const result = JSON.parse(contents.toString());
+      res.json(result);
+    } catch (error: any) {
+      logger.error(`Error getting ${type} under ${fullPath}: ${error.message}`);
+      res.status(500).json({ error: `Error getting ${type} under ${fullPath}: ${error.message}` });
+    }
+  }
+
+  router.get('/manifest/:bucket/:kind/:name', async (req: Request, res: Response) => {
+    await handleRequest(req, res, 'manifest');
   });
 
-  router.get('/catalog/:bucket/:kind/:name', async (req, response) => {
-    const file_path = `${req.params.kind}/${req.params.name}/catalog.json`
-    logger.info(`Get catalog under ${req.params.bucket}/${req.params.kind}/${req.params.name}/catalog.json`)
-    const contents = await storage.bucket(req.params.bucket).file(file_path).download();
-    const result = JSON.parse(contents.toString())
-    response.json(result);
+  router.get('/catalog/:bucket/:kind/:name', async (req: Request, res: Response) => {
+    await handleRequest(req, res, 'catalog');
   });
 
   router.use(errorHandler());

packages/dbt-backend/src/service/standaloneServer.ts

@@ -1,7 +1,16 @@
 import { createServiceBuilder } from '@backstage/backend-common';
 import { Server } from 'http';
 import { Logger } from 'winston';
-import { createRouter } from './router';
+
+// Import the router creation function based on your modularized structure
+import { createRouter, RouterOptions } from './router';
+import { GoogleStorageProvider } from './googleStorage';
+
+// Create the storage provider instance, default
+// uses the Google Cloud Storage provider to maintain
+// regression.
+const storageProvider = new GoogleStorageProvider();
+
 
 export interface ServerOptions {
   port: number;
@@ -14,13 +23,20 @@ export async function startStandaloneServer(
 ): Promise<Server> {
   const logger = options.logger.child({ service: 'dbt-backend' });
   logger.debug('Starting application server...');
-  const router = await createRouter({
+
+  // Create router options and include the logger
+  const routerOptions: RouterOptions = {
     logger,
-  });
+    storageProvider,
+  };
+
+  // Create the router using the modularized router creation function
+  const router = createRouter(routerOptions);
 
   let service = createServiceBuilder(module)
     .setPort(options.port)
     .addRouter('/dbt', router);
+
   if (options.enableCors) {
     service = service.enableCors({ origin: 'http://localhost:3000' });
   }