From 2aedc125c5d4fbaebd267ffb327dcfe3a5cf2b25 Mon Sep 17 00:00:00 2001 From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com> Date: Sat, 7 Sep 2024 13:58:42 +0530 Subject: [PATCH 1/3] save state in file for unexpected edits and teardown --- test/4.plugins/separation/1.setup.js | 40 ++++++-- test/4.plugins/separation/2.teardown.js | 130 ++++++++++++++---------- test/4.plugins/separation/testState.js | 43 ++++++++ 3 files changed, 152 insertions(+), 61 deletions(-) create mode 100644 test/4.plugins/separation/testState.js diff --git a/test/4.plugins/separation/1.setup.js b/test/4.plugins/separation/1.setup.js index ace820696b1..2fae99ad219 100644 --- a/test/4.plugins/separation/1.setup.js +++ b/test/4.plugins/separation/1.setup.js @@ -1,19 +1,25 @@ -var request = require('supertest'); -var should = require('should'); -var testUtils = require("../../testUtils"); -var plugins = require("../../../plugins/pluginManager"); +let request = require('supertest'); +const should = require('should'); +const testUtils = require("../../testUtils"); +const plugins = require("../../../plugins/pluginManager"); +const { saveState, loadState, clearState } = require('./testState'); + request = request(testUtils.url); -var TEMP_KEY = ""; -var API_KEY_ADMIN = ""; -var APP_ID = ""; -var APP_KEY = ""; +let TEMP_KEY = ""; +let API_KEY_ADMIN = ""; +let APP_ID = ""; +let APP_KEY = ""; describe('Retrieve API-KEY', function() { before('Create db connection', async function() { + // Clear any existing state + clearState(); + testUtils.db = await plugins.dbConnection("countly"); testUtils.client = testUtils.db.client; }); + it('should create user', function(done) { testUtils.db.collection("members").findOne({global_admin: true}, function(err, member) { if (err) { @@ -30,6 +36,22 @@ describe('Retrieve API-KEY', function() { }); }); +// Save state after each test +afterEach(function() { + const state = { + TEMP_KEY, + API_KEY_ADMIN, + API_KEY_USER: testUtils.get("API_KEY_USER"), + APP_ID, + APP_KEY, + USER_ID: testUtils.get("USER_ID"), + ADMIN_ID: testUtils.get("ADMIN_ID"), + username: testUtils.username, + email: testUtils.email + }; + saveState(state); +}); + describe('Creating users', function() { describe('global admin', function() { it('should create user', function(done) { @@ -87,4 +109,4 @@ describe('Create app', function() { done(); }); }); -}); \ No newline at end of file +}); diff --git a/test/4.plugins/separation/2.teardown.js b/test/4.plugins/separation/2.teardown.js index ae5555d44dd..c43e360119b 100644 --- a/test/4.plugins/separation/2.teardown.js +++ b/test/4.plugins/separation/2.teardown.js @@ -1,44 +1,46 @@ -var request = require('supertest'); -var should = require('should'); -var testUtils = require("../../testUtils"); +let request = require('supertest'); +const should = require('should'); +const testUtils = require("../../testUtils"); +const { loadState, clearState } = require('./testState'); + request = request(testUtils.url); -var API_KEY_ADMIN = ""; -var API_KEY_USER = ""; -var TEMP_KEY = ""; -var APP_ID = ""; -var USER_ID = ""; -var ADMIN_ID = ""; +let API_KEY_ADMIN = ""; +let API_KEY_USER = ""; +let TEMP_KEY = ""; +let APP_ID = ""; +let USER_ID = ""; +let ADMIN_ID = ""; -describe('Deleting app', function() { - it('should delete app', function(done) { - API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN"); - API_KEY_USER = testUtils.get("API_KEY_USER"); - TEMP_KEY = testUtils.get("TEMP_KEY"); - APP_ID = testUtils.get("APP_ID"); - USER_ID = testUtils.get("USER_ID"); - ADMIN_ID = testUtils.get("ADMIN_ID"); - var params = {app_id: APP_ID}; - request - .get('/i/apps/delete?api_key=' + API_KEY_ADMIN + "&args=" + JSON.stringify(params)) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - var ob = JSON.parse(res.text); - ob.should.have.property('result', 'Success'); - done(); - }); +describe('Teardown', function() { + before(function() { + const state = loadState(); + console.log("--------------", state, "--------------"); + if (state) { + API_KEY_ADMIN = state.API_KEY_ADMIN; + API_KEY_USER = state.API_KEY_USER; + TEMP_KEY = state.TEMP_KEY; + APP_ID = state.APP_ID; + USER_ID = state.USER_ID; + ADMIN_ID = state.ADMIN_ID; + } + // If any value is still undefined, try to get it from testUtils + API_KEY_ADMIN = API_KEY_ADMIN || testUtils.get("API_KEY_ADMIN"); + API_KEY_USER = API_KEY_USER || testUtils.get("API_KEY_USER"); + TEMP_KEY = TEMP_KEY || testUtils.get("TEMP_KEY"); + APP_ID = APP_ID || testUtils.get("APP_ID"); + USER_ID = USER_ID || testUtils.get("USER_ID"); + ADMIN_ID = ADMIN_ID || testUtils.get("ADMIN_ID"); }); -}); -describe('Deleting user', function() { - describe('delete simple user', function() { - it('should delete successfully', function(done) { - var params = {user_ids: [USER_ID]}; + describe('Deleting app', function() { + it('should delete app', function(done) { + if (!APP_ID) { + return done(); + } + var params = {app_id: APP_ID}; request - .get('/i/users/delete?api_key=' + API_KEY_ADMIN + "&args=" + JSON.stringify(params)) + .get('/i/apps/delete?api_key=' + API_KEY_ADMIN + "&args=" + JSON.stringify(params)) .expect(200) .end(function(err, res) { if (err) { @@ -50,23 +52,47 @@ describe('Deleting user', function() { }); }); }); - describe('delete admin', function() { - it('should delete successfully', function(done) { - var params = {user_ids: [ADMIN_ID]}; - request - .get('/i/users/delete?api_key=' + TEMP_KEY + "&args=" + JSON.stringify(params)) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - var ob = JSON.parse(res.text); - ob.should.have.property('result', 'Success'); - done(); - }); + + describe('Deleting user', function() { + describe('delete simple user', function() { + it('should delete successfully', function(done) { + console.log("-----", USER_ID, "------"); + var params = {user_ids: [USER_ID]}; + request + .get('/i/users/delete?api_key=' + API_KEY_ADMIN + "&args=" + JSON.stringify(params)) + .expect(200) + .end(function(err, res) { + if (err) { + return done(err); + } + var ob = JSON.parse(res.text); + ob.should.have.property('result', 'Success'); + done(); + }); + }); }); - after('Close db connection', async function() { - testUtils.client.close(); + describe('delete admin', function() { + it('should delete successfully', function(done) { + console.log("-------", ADMIN_ID, "------"); + var params = {user_ids: [ADMIN_ID]}; + request + .get('/i/users/delete?api_key=' + TEMP_KEY + "&args=" + JSON.stringify(params)) + .expect(200) + .end(function(err, res) { + if (err) { + return done(err); + } + var ob = JSON.parse(res.text); + ob.should.have.property('result', 'Success'); + done(); + }); + }); + after('Close db connection and clear state', async function() { + if (testUtils.client) { + await testUtils.client.close(); + } + clearState(); + }); }); }); -}); \ No newline at end of file +}); diff --git a/test/4.plugins/separation/testState.js b/test/4.plugins/separation/testState.js new file mode 100644 index 00000000000..edf00c11a36 --- /dev/null +++ b/test/4.plugins/separation/testState.js @@ -0,0 +1,43 @@ +const fs = require('fs'); +const path = require('path'); + +const STATE_FILE = path.join(__dirname, 'testState.json'); + +function saveState(state) { + const fullState = { + API_KEY_ADMIN: state.API_KEY_ADMIN || '', + API_KEY_USER: state.API_KEY_USER || '', + TEMP_KEY: state.TEMP_KEY || '', + APP_ID: state.APP_ID || '', + USER_ID: state.USER_ID || '', + ADMIN_ID: state.ADMIN_ID || '' + }; + fs.writeFileSync(STATE_FILE, JSON.stringify(fullState)); +} + +function loadState() { + if (fs.existsSync(STATE_FILE)) { + const state = JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')); + return { + API_KEY_ADMIN: state.API_KEY_ADMIN || '', + API_KEY_USER: state.API_KEY_USER || '', + TEMP_KEY: state.TEMP_KEY || '', + APP_ID: state.APP_ID || '', + USER_ID: state.USER_ID || '', + ADMIN_ID: state.ADMIN_ID || '' + }; + } + return null; +} + +function clearState() { + if (fs.existsSync(STATE_FILE)) { + fs.unlinkSync(STATE_FILE); + } +} + +module.exports = { + saveState, + loadState, + clearState +}; From f5ec0aeb94a8f8fed468c273685a6404d734b4fb Mon Sep 17 00:00:00 2001 From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com> Date: Sat, 7 Sep 2024 17:55:43 +0530 Subject: [PATCH 2/3] add a simple parallel test runner --- testRunner.js | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 testRunner.js diff --git a/testRunner.js b/testRunner.js new file mode 100644 index 00000000000..bd877fef5df --- /dev/null +++ b/testRunner.js @@ -0,0 +1,165 @@ +const glob = require('glob'); +const path = require('path'); +const { spawn } = require('child_process'); + +// Parse command line arguments +const args = process.argv.slice(2); +let includePlugins = []; +let excludePlugins = []; + +args.forEach(arg => { + if (arg.startsWith('--include=')) { + includePlugins = arg.replace('--include=', '').split(','); + } + else if (arg.startsWith('--exclude=')) { + excludePlugins = arg.replace('--exclude=', '').split(','); + } +}); + +// Find all plugin names +const pluginNames = glob.sync('plugins/*').map(plugin => path.basename(plugin)); + +// Function to get test file path for a plugin +function getTestPath(pluginName) { + const possiblePaths = [ + path.resolve(`plugins/${pluginName}/tests`), + path.resolve(`plugins/${pluginName}/tests.js`) + ]; + + for (const testPath of possiblePaths) { + try { + require.resolve(testPath); + return testPath; + } + catch (err) { + // ignore + } + } + return null; +} + +// Filter plugins based on include/exclude lists and existing tests +let pluginsWithTests = pluginNames.filter(plugin => getTestPath(plugin) !== null); + +if (includePlugins.length > 0) { + pluginsWithTests = pluginsWithTests.filter(plugin => includePlugins.includes(plugin)); +} +else if (excludePlugins.length > 0) { + pluginsWithTests = pluginsWithTests.filter(plugin => !excludePlugins.includes(plugin)); +} + +// Set the maximum number of processes (default to number of CPU cores) +const MAX_PROCESSES = process.env.MAX_PROCESSES || require('os').cpus().length; + +// Setup and teardown file paths +const SETUP_FILE = 'test/4.plugins/separation/1.setup.js'; +const TEARDOWN_FILE = 'test/4.plugins/separation/2.teardown.js'; + +// Function to run Mocha for multiple plugins +function runMochaForPlugins(plugins) { + return new Promise((resolve) => { + console.log(`Running tests for plugins: ${plugins.join(', ')}`); + + const mochaArgs = [ + 'mocha', + '--reporter', 'min', + '--timeout', '50000', + '--colors', + // '--debug', + // '--trace-warnings', + // '--trace-deprecation' + ]; + + // Add symlink-related arguments only if COUNTLY_CONFIG__SYMLINKED is true + if (process.env.COUNTLY_CONFIG__SYMLINKED === 'true') { + mochaArgs.push('--preserve-symlinks', '--preserve-symlinks-main'); + } + + // Add setup file, test paths, and teardown file to the arguments + mochaArgs.push( + SETUP_FILE, + ...plugins.map(getTestPath).filter(Boolean), + TEARDOWN_FILE + ); + + console.log(mochaArgs); + // process.exit(0); + + const mochaProcess = spawn('npx', mochaArgs, { + stdio: ['inherit', 'pipe', 'pipe'], + env: { ...process.env } + }); + + let output = ''; + mochaProcess.stdout.on('data', (data) => { + output += data.toString(); + }); + mochaProcess.stderr.on('data', (data) => { + output += data.toString(); + }); + + mochaProcess.on('close', (code) => { + resolve({ + plugins, + success: code === 0, + output + }); + }); + }); +} + +// Divide plugins among processes +function chunkArray(array, chunks) { + const result = []; + const chunkSize = Math.ceil(array.length / chunks); + for (let i = 0; i < array.length; i += chunkSize) { + result.push(array.slice(i, i + chunkSize)); + } + return result; +} + +// Run tests in batches +async function runTestsInBatches() { + const pluginChunks = chunkArray(pluginsWithTests, MAX_PROCESSES); + const results = await Promise.all(pluginChunks.map(runMochaForPlugins)); + + let allPassed = true; + const failedPlugins = []; + + console.log('\n=== Test Results ===\n'); + + results.forEach(result => { + if (result.success) { + console.log(`✅ Plugins passed: ${result.plugins.join(', ')}`); + } + else { + console.log(`❌ Plugins failed: ${result.plugins.join(', ')}`); + failedPlugins.push(...result.plugins); + allPassed = false; + } + }); + + if (!allPassed) { + console.log('\n=== Detailed Output for Failed Tests ===\n'); + + results.forEach(result => { + if (!result.success) { + console.log(`--- Output for failed plugins: ${result.plugins.join(', ')} ---`); + console.log(result.output); + console.log('---\n'); + } + }); + + console.error(`Tests failed for the following plugins: ${failedPlugins.join(', ')}`); + process.exit(1); + } + else { + console.log('\nAll test batches completed successfully'); + process.exit(0); + } +} + +console.log(`Found ${pluginsWithTests.length} plugins with tests`); +console.log(`Running tests with a maximum of ${MAX_PROCESSES} processes`); +console.log(`Symlink preservation is ${process.env.COUNTLY_CONFIG__SYMLINKED === 'true' ? 'enabled' : 'disabled'}`); +runTestsInBatches(); \ No newline at end of file From faac7e45e766a3abe4534b950b086d4be0ebd806 Mon Sep 17 00:00:00 2001 From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com> Date: Sat, 7 Sep 2024 22:16:48 +0530 Subject: [PATCH 3/3] test runner gives output --- package.json | 1 + testRunner.js | 41 +++++++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index f578768aa4c..661601b8b7f 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "apidoc": "^1.0.1", "apidoc-template": "^0.0.2", "docdash": "^2.0.1", + "glob": "^11.0.0", "grunt-contrib-watch": "1.1.0", "grunt-eslint": "24.3.0", "grunt-mocha-nyc": "1.0.3", diff --git a/testRunner.js b/testRunner.js index bd877fef5df..3a338b56f9a 100644 --- a/testRunner.js +++ b/testRunner.js @@ -1,3 +1,4 @@ +// TODO: run set and teardown just once? const glob = require('glob'); const path = require('path'); const { spawn } = require('child_process'); @@ -62,7 +63,7 @@ function runMochaForPlugins(plugins) { const mochaArgs = [ 'mocha', - '--reporter', 'min', + '--reporter', 'spec', '--timeout', '50000', '--colors', // '--debug', @@ -90,19 +91,45 @@ function runMochaForPlugins(plugins) { env: { ...process.env } }); - let output = ''; + let failureOutput = ''; + let currentPluginOutput = ''; + let currentPlugin = ''; + mochaProcess.stdout.on('data', (data) => { - output += data.toString(); + const dataStr = data.toString(); + currentPluginOutput += dataStr; + + // Check if this is the start of a new plugin's tests + const pluginMatch = dataStr.match(/^ {2}([^\n]+)/); + if (pluginMatch) { + if (currentPlugin && !failureOutput.includes(currentPlugin)) { + // If the previous plugin had no failures, print its output + process.stdout.write(currentPluginOutput); + } + currentPlugin = pluginMatch[1]; + currentPluginOutput = dataStr; + } + + // Check for test failures + if (dataStr.includes('✖')) { + failureOutput += currentPluginOutput; + } }); + mochaProcess.stderr.on('data', (data) => { - output += data.toString(); + failureOutput += data.toString(); }); mochaProcess.on('close', (code) => { + // Print output for the last plugin if it was successful + if (currentPlugin && !failureOutput.includes(currentPlugin)) { + process.stdout.write(currentPluginOutput); + } + resolve({ plugins, success: code === 0, - output + failureOutput }); }); }); @@ -144,9 +171,7 @@ async function runTestsInBatches() { results.forEach(result => { if (!result.success) { - console.log(`--- Output for failed plugins: ${result.plugins.join(', ')} ---`); - console.log(result.output); - console.log('---\n'); + console.log(result.failureOutput); } });