Merge pull request #141 from dotkom/feat/use-s3-for-file-transfer

Implement s3 integration

next-env.d.ts

@@ -1,3 +1,15 @@
 /// <reference types="next" />
 /// <reference types="next/types/global" />
 /// <reference types="next-images" />
+
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      AWS_REGION?: string;
+      AWS_ACCESS_KEY_ID?: string;
+      AWS_SECRET_ACCESS_KEY?: string;
+      AWS_S3_BUCKET_NAME?: string;
+      LAMBDA_PRESIGN_UPLOAD_ENDPOINT?: string;
+    }
+  }
+}

package.json

@@ -3,10 +3,11 @@
   "version": "2.2.1",
   "private": true,
   "dependencies": {
-    "@dotkomonline/design-system": "^0.22.0",
+    "@dotkomonline/design-system": "^0.22.2",
     "@reduxjs/toolkit": "^1.4.0",
     "@sentry/browser": "^5.0.3",
     "@sentry/node": "^5.4.3",
+    "@types/aws-sdk": "^2.7.0",
     "@types/file-saver": "^2.0.0",
     "@types/get-stream": "^3.0.2",
     "@types/jsdom": "^16.2.4",
@@ -19,6 +20,7 @@
     "@types/react-dom": "16.9.8",
     "@types/react-redux": "^7.1.0",
     "@types/styled-components": "^5.1.4",
+    "aws-sdk": "^2.1560.0",
     "core-js": "^3.0.1",
     "file-saver": "^2.0.1",
     "get-stream": "^6.0.1",

src/constants/backend.ts

@@ -1,3 +1,4 @@
 export const LAMBDA_ENDPOINT = '/api/generate-receipt';
+export const LAMBDA_PRESIGN_UPLOAD_ENDPOINT = '/api/presign-upload-url';
 
 export const OW4_ADDRESS = process.env.NEXT_PUBLIC_OW4_ADDRESS || 'https://online.ntnu.no';

src/form/state.ts

@@ -2,6 +2,8 @@ import { ApiBodyError } from './../lambda/errors';
 import { Group } from 'models/groups';
 import { readDataUrlAsFile2 } from 'utils/readDataUrlAsFile';
 import { readFileAsDataUrl } from 'utils/readFileAsDataUrl';
+import { uploadFile } from "../utils/uploadFile";
+import { downloadFileFromS3Bucket } from "../utils/downloadFileFromS3Bucket";
 
 export type ReceiptType = 'card' | 'deposit';
 export type SendMode = 'download' | 'email' | 'teapot';
@@ -56,7 +58,9 @@ export interface IDeserializedState {
 }
 
 export const deserializeReceipt = async (state: IState): Promise<IDeserializedState> => {
-  const attachments = await Promise.all(state.attachments.map(async (file) => readFileAsDataUrl(file)));
+  const attachments = await Promise.all(
+    state.attachments.map(async (file) => uploadFile(file))
+  );
   const signature = await readFileAsDataUrl(state.signature || new File([], 'newfile'));
   return {
     ...state,
@@ -67,9 +71,11 @@ export const deserializeReceipt = async (state: IState): Promise<IDeserializedSt
 
 export const serializeReceipt = async (deserializedState: IDeserializedState): Promise<IState> => {
   try {
-    const attachments = await Promise.all(
-      deserializedState.attachments.map(async (dataUrl) => readDataUrlAsFile2(dataUrl))
-    );
+    const illegalAttachment = deserializedState.attachments.find((attachment) => !attachment.startsWith("uploads/"));
+    if (illegalAttachment) {
+      throw new TypeError('Illegal attachment');
+    }
+    const attachments = await Promise.all(deserializedState.attachments.map(downloadFileFromS3Bucket));
     const signature = await readDataUrlAsFile2(deserializedState.signature);
     return {
       ...deserializedState,

src/form/validation.ts

@@ -29,8 +29,8 @@ export const ACCOUNT_NUMBER_REGEX = new RegExp(/^\d{4} \d{2} \d{5}$/);
 export const COMMITTEE_EMAIL_REGEX = new RegExp(/^.{2,50}@online\.ntnu\.no$/);
 export const EMAIL_REGEX = new RegExp(/^[a-zA-Z0-9.!#$%&’*+/=?^_{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$/);
 export const CARD_DETAIL_REGEX = new RegExp(/^.{5,30}$/);
-export const FILE_SIZE_WARN = 7 * 1024 * 1024; // 7 MB
-export const FILE_SIZE_MAX = 9 * 1024 * 1024; // 9 MB
+export const FILE_SIZE_WARN = 20 * 1024 * 1024; // 20 MB
+export const FILE_SIZE_MAX = 25 * 1024 * 1024; // 25 MB
 
 export const STATE_VALIDATION: StateValidators = {
   fullname: [

src/lambda/handler.ts

@@ -10,6 +10,7 @@ import { readFileAsDataUrl } from 'utils/readFileAsDataUrl';
 import { sendEmail } from './sendEmail';
 import { ApiBodyError, ApiValidationError } from './errors';
 import { pdfGenerator } from './browserlessGenerator';
+import { uploadFileToS3Bucket } from "../utils/uploadFileToS3Bucket";
 
 export interface SuccessBody {
   message: string;
@@ -48,12 +49,16 @@ export const generateReceipt = async (data: IDeserializedState | null): Promise<
   }
   const validState = state as NonNullableState;
   const pdf = await pdfGenerator(validState);
-  const pdfFile = new File([pdf], 'receipt.pdf', { type: 'application/pdf' });
-  const pdfString = await readFileAsDataUrl(pdfFile);
 
   if (state.mode === 'download') {
-    return DOWNLOAD_SUCCESS_MESSAGE(pdfString);
+    const dato = `${new Date().toISOString().split('T')[0]}`;
+    const filename = `kvittering-${dato}-${+Date.now()}.pdf`;
+    const downloadUrl = await uploadFileToS3Bucket(pdf, `receipts/${filename}`);
+    return DOWNLOAD_SUCCESS_MESSAGE(downloadUrl);
   } else if (state.mode === 'email') {
+    const pdfFile = new File([pdf], 'receipt.pdf', { type: 'application/pdf' });
+    const pdfString = await readFileAsDataUrl(pdfFile);
+
     await sendEmail(pdfString, state);
     return EMAIL_SUCCESS_MESSAGE;
   } else if (state.mode === 'teapot') {

src/pages/api/presign-upload-url.ts

@@ -0,0 +1,27 @@
+import { NextApiRequest, NextApiResponse, PageConfig } from 'next';
+
+import { SuccessBody } from 'lambda/handler';
+import { sentryMiddleware } from 'lambda/sentry';
+import { ErrorData } from 'lambda/errors';
+import { getPresignedS3URL } from "../../utils/getPresignedS3URL";
+
+const handler = async (req: NextApiRequest, res: NextApiResponse<SuccessBody | ErrorData>) => {
+  const { filename, contentType } = req.body;
+
+  try {
+    const data = await getPresignedS3URL(filename, contentType);
+    res.status(200).json({ message: "Presigned URL", data: JSON.stringify(data) });
+  } catch (error) {
+    res.status(500).json({ message: "Failed to get presigned URL", data: error });
+  }
+};
+
+export const config: PageConfig = {
+  api: {
+    bodyParser: {
+      sizeLimit: '25mb',
+    },
+  },
+};
+
+export default sentryMiddleware(handler);

src/redux/actions/authActions.ts

@@ -68,7 +68,6 @@ export const loginAction = createAsyncThunk('user/login', async (_, { dispatch,
     const user: User | null = await getManager().getUser();
     if (user) {
       const newForm = await processUser(user, form);
-      console.log({ newForm });
       updateForm(dispatch, newForm);
     } else {
       logInRedirect(form);
@@ -96,6 +95,7 @@ export const catchCallbackAction = createAsyncThunk('user/catchCallback', async
     window.location.hash = '';
   } catch (err) {
     /** Do nothing if no user data is present */
+    window.location.hash = '';
     return;
   }
 });

src/redux/actions/submitActions.ts

@@ -3,9 +3,7 @@ import { createAsyncThunk } from '@reduxjs/toolkit';
 
 import { NonNullableState } from 'lambda/generatePDF';
 import { getFileName } from 'lambda/tools/format';
-import { downloadFile } from 'utils/download';
 import { postReceipt } from 'utils/postReceipt';
-import { readDataUrlAsFile } from 'utils/readDataUrlAsFile';
 import { downloadFinished, downloadStarted, loadingDone, setResponse } from 'redux/reducers/statusReducer';
 import { State } from 'redux/store';
 import { SuccessBody } from 'lambda/handler';
@@ -14,10 +12,13 @@ const handleDownload = async (response: SuccessBody, state: NonNullableState) =>
   if (response.data) {
     /** Use the same filename that would be generated when sending a mail */
     const fileName = getFileName(state);
-    const pdfFile = await readDataUrlAsFile(response.data, fileName);
-    if (pdfFile) {
-      downloadFile(pdfFile);
-    }
+    // response.data is a URL to the file
+
+    const a = document.createElement('a');
+    document.body.appendChild(a);
+    a.href = response.data;
+    a.download = fileName;
+    a.click();
   }
 };
 

src/utils/downloadFileFromS3Bucket.ts

@@ -0,0 +1,32 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+  {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  } :
+  undefined;
+
+AWS.config.update({
+  region: process.env.AWS_REGION,
+  credentials,
+})
+
+export async function downloadFileFromS3Bucket(key: string): Promise<File> {
+  if (!process.env.AWS_S3_BUCKET_NAME) {
+    throw new Error('AWS_S3_BUCKET_NAME is not set');
+  }
+
+  const s3 = new AWS.S3({
+    apiVersion: '2006-03-01',
+  });
+
+  const params = {
+    Bucket: process.env.AWS_S3_BUCKET_NAME,
+    Key: key,
+  };
+
+  const data = await s3.getObject(params).promise();
+
+  return new File([data.Body as Blob], key, { type: data.ContentType });
+}

src/utils/getPresignedS3URL.ts

@@ -0,0 +1,41 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+  {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  } :
+  undefined;
+
+AWS.config.update({
+  region: process.env.AWS_REGION,
+  credentials,
+})
+
+const s3 = new AWS.S3({
+  apiVersion: '2006-03-01',
+});
+
+const createPresignedPost = (params: AWS.S3.PresignedPost.Params): Promise<AWS.S3.PresignedPost> => new Promise((resolve, reject) => {
+  s3.createPresignedPost(params, function (err, data) {
+    if (err) {
+      reject(err);
+    } else {
+      resolve(data)
+    }
+  });
+});
+
+export const getPresignedS3URL = async (name: string, contentType: string): Promise<{ url: string, fields: {[key: string]: string}}> => {
+  return await createPresignedPost({
+    Bucket: "receipt-form",
+    Fields: {
+      key: `uploads/${+new Date()}-${name}`,
+      "Content-Type": contentType,
+    },
+    Conditions: [
+      ["content-length-range", 0, 1024 * 1024 * 10],
+    ],
+    Expires: 60,
+  })
+}

src/utils/uploadFile.ts

@@ -0,0 +1,42 @@
+import { LAMBDA_PRESIGN_UPLOAD_ENDPOINT } from "constants/backend";
+
+export const uploadFile = async (file: File): Promise<string> => {
+  if (!LAMBDA_PRESIGN_UPLOAD_ENDPOINT) {
+    throw new Error('LAMBDA_PRESIGN_UPLOAD_ENDPOINT is not set');
+  }
+
+  // Request to get the presigned POST data
+  const response = await fetch(LAMBDA_PRESIGN_UPLOAD_ENDPOINT, {
+    method: 'POST',
+    headers: {
+      Accept: 'application/json',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ filename: file.name, contentType: file.type }),
+  });
+
+  const body = await response.json();
+  if (!response.ok) {
+    console.error(await response.text());
+    throw new Error('Failed to get presigned post data');
+  }
+
+  const { url, fields }: {url: string, fields: {[key: string]: string}} = JSON.parse(body.data);
+
+  const formData = new FormData();
+  for (const key in fields) {
+    formData.append(key, fields[key]);
+  }
+  formData.append('file', file);
+
+  const uploadResponse = await fetch(url, {
+    method: 'POST',
+    body: formData,
+  });
+
+  if (!uploadResponse.ok) {
+    throw new Error('Failed to upload file');
+  }
+
+  return fields.key
+};

src/utils/uploadFileToS3Bucket.ts

@@ -0,0 +1,35 @@
+import AWS from "aws-sdk";
+
+const credentials = (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) ?
+  {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  } :
+  undefined;
+
+AWS.config.update({
+  region: process.env.AWS_REGION,
+  credentials,
+})
+
+// upload file to S3 bucket and make it publicly downloadable
+export async function uploadFileToS3Bucket(file: Uint8Array, key: string): Promise<string> {
+  if (!process.env.AWS_S3_BUCKET_NAME) {
+    throw new Error('AWS_S3_BUCKET_NAME is not set');
+  }
+  const s3 = new AWS.S3({
+    apiVersion: '2006-03-01',
+    params: { Bucket: process.env.AWS_S3_BUCKET_NAME },
+  });
+
+  const params = {
+    Bucket: process.env.AWS_S3_BUCKET_NAME,
+    Key: key,
+    Body: file,
+    ACL: 'public-read',
+  };
+
+  const result = await s3.upload(params).promise();
+
+  return result.Location;
+}