More integration tests, New github action to run integration tests

.github/workflows/webapp-integration-tests.yml

@@ -0,0 +1,43 @@
+
+name: Run Webapp Integration Tests
+
+on:
+  pull_request:
+    branches: [ "develop" ]
+
+jobs:
+  integration-tests:
+    name: Build app and run integration tests
+    runs-on: ubuntu-latest
+    env:
+        TERM: xterm
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup firefox
+        id: setup-firefox
+        uses: browser-actions/setup-firefox@v1
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Set environment variables
+        run: ./setemptyenv.sh
+
+      - name: Install webapp dependencies
+        working-directory: ./webapp
+        run: npm install
+
+      - name: run ./install.sh
+        run: ./install.sh
+
+      - name: run integration tests
+        working-directory: ./webapp
+        run: npm run test:integration

docker-compose.yml

@@ -8,7 +8,7 @@ services:
     container_name: unstructured-api
     environment:
         - PORT=9500
-    image: localhost:5000/unstructured-api
+    image: localhost:9500/unstructured-api
 
   docker_rabbitmq:
     image: rabbitmq:3.13.1-management

setemptyenv.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+
+# Function to copy .env.example to .env
+copy_env_example() {
+	# Check if .env.example exists
+	if [ ! -f ".env.example" ]; then
+		echo ".env.example file does not exist."
+		exit 1
+	fi
+
+	# Copy contents of .env.example to .env
+	cp .env.example .env
+	echo ".env file has been created with the contents of .env.example."
+}
+
+# Navigate to the ./webapp directory
+cd ./webapp || { 
+	echo "Failed to navigate to ./webapp. Directory does not exist."; 
+	exit 1; 
+}
+
+copy_env_example
+
+cd ../agent-backend || {
+    echo "Failed to navigate to ../agent-backend. Directory may not exist";
+    exit 1;
+}
+
+copy_env_example
+
+cd ../vector-db-proxy || {
+    echo "Failed to navigate to ../vector-db-proxy. Directory may not exist";
+    exit 1;
+}
+
+copy_env_example

vector-db-proxy/.env.example

@@ -0,0 +1,21 @@
+HOST=0.0.0.0
+PORT=9001
+MONGO_URI=mongodb://docker_mongo:27017
+MONGO_DB_NAME=test
+RABBITMQ_HOST=docker_rabbitmq
+RABBITMQ_PORT=5672
+RABBITMQ_STREAM=streaming
+RABBITMQ_EXCHANGE=agentcloud
+RABBITMQ_ROUTING_KEY=key
+RABBITMQ_USERNAME=guest
+RABBITMQ_PASSWORD=guest
+QDRANT_URI=http://qdrant:6334
+QDRANT_HOST=http://qdrant
+QDRANT_PORT=6334
+REDIS_HOST=docker_redis
+REDIS_PORT=6379
+WEBAPP_HOST=webapp_next
+THREAD_PERCENTAGE_UTILISATION=0.8
+USE_GPU=false
+LOGGING_LEVEL=warn
+UNSTRUCTURED_API_URL=http://unstructured-api:9500/general/v0/general

webapp/.env.example

@@ -59,6 +59,8 @@ STRIPE_ADDON_USERS_PRODUCT_ID=
 STRIPE_ADDON_STORAGE_PRODUCT_ID=
 STRIPE_WEBHOOK_SECRET=
 STRIPE_ACCOUNT_SECRET=
+SKIP_EMAIL=1
+SKIP_STRIPE=1
 NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=
 WEBAPP_TEST_BASE_URL=http://localhost:3000
 NEXT_PUBLIC_LOCAL_UNSTRUCTURED=1

webapp/src/lib/middleware/auth/checksession.ts

@@ -4,7 +4,7 @@ import { dynamicResponse } from '@dr';
 
 export default function checkSession(req, res, next) {
 	if (!res.locals.account?._id && !res.locals.isAgentBackend) {
-		// console.log("checkSession, reslocals: ", res.locals);
+		console.log('checkSession, reslocals: ', res.locals);
 		if (res.locals.isSocket) {
 			return res?.locals?.socket?.disconnect();
 		} else {

webapp/src/lib/middleware/auth/fetchsession.ts

@@ -7,14 +7,15 @@ const log = debug('webapp:session');
 export default async function fetchSession(req, res, next) {
 	// log('req.session:', req.session);
 	if (req.session && (req.session.accountId || req.session.passport?.user)) {
+		log('req.session.account', req.session.accountId);
 		let account: Account;
 		if (req.session.accountId) {
 			account = await getAccountById(req.session.accountId);
 		} else if (req.session.passport?.user) {
 			const { oauthId, provider } = req.session.passport?.user;
 			account = await getAccountByOAuthOrEmail(oauthId, provider, null);
 		}
-		// log('account:', account);
+		log('account:', account);
 		if (account) {
 			res.locals.account = {
 				_id: account._id.toString(),

webapp/src/test/account.ts

@@ -43,7 +43,7 @@ beforeAll(async () => {
 
 describe('account tests', () => {
 
-	test('register new accounts', async () => {
+	test.only('register new accounts', async () => {
 		let response = await fetch(`${process.env.WEBAPP_TEST_BASE_URL}/forms/account/register`, {
 			method: 'POST',
 			headers: {
@@ -94,7 +94,7 @@ describe('account tests', () => {
 		await db.db().collection('accounts').deleteMany({ email: accountDetails.account3_email }); //delete these accounts so we can stress test logins in the next test
 	}, 60 * SECONDS); //extended timeout due to multiple account creations
 
-	test('stress test registration with many accounts', async () => {
+	test.only('stress test registration with many accounts', async () => {
 		const accounts = [
 			{ name: accountDetails.account1_name, email: accountDetails.account1_email, password: accountDetails.account1_password },
 			{ name: accountDetails.account2_name, email: accountDetails.account2_email, password: accountDetails.account2_password },
@@ -131,7 +131,7 @@ describe('account tests', () => {
 
 
 	//TODO: refactor this to do it with a loop
-	test('login as new users - 11 logins', async () => {
+	test.only('login as new users - 11 logins', async () => {
 		let response = await fetch(`${process.env.WEBAPP_TEST_BASE_URL}/forms/account/login`, {
 			method: 'POST',
 			headers: {
@@ -300,7 +300,7 @@ describe('account tests', () => {
 	}, 60 * SECONDS); // extended timeout due to multiple account logins
 
 
-	test('get account', async () => {
+	test.only('get account', async () => {
 		const url = `${process.env.WEBAPP_TEST_BASE_URL}/account.json`;
 		const accounts = [
 			{ email: accountDetails.account1_email, sessionCookie: sessionCookie1 },
@@ -353,7 +353,7 @@ describe('account tests', () => {
 	//test with valid token??
 
 	//sets the role to prevent redirects to onboarding in further tests
-	test('set role - onboarding', async () => {
+	test.only('set role - onboarding', async () => {
 		const accounts = [
 			accountDetails.account1_email,
 			accountDetails.account2_email,