Merge pull request #47 from cortex-lab/py_ci

Python CI

.gitignore

@@ -1,3 +1,5 @@
 .idea/*
 .DS_Store
 node_modules/*
+.env
+.pem

.travis.yml

@@ -0,0 +1,15 @@
+language: node_js
+node_js: 12
+env:
+  - NODE_ENV=test
+branches:
+  only:
+  - master
+jobs:
+  include:
+    - name: "windows"
+      os: windows
+      env: NODE_ENV=test DOTENV_CONFIG_PATH=.\\test\\fixtures\\.env.test
+    - name: "linux"
+      os: linux
+      env: NODE_ENV=test DOTENV_CONFIG_PATH=./test/fixtures/.env.test

CHANGELOG.md

@@ -1,7 +1,29 @@
 # Changelog
 
-## [Latest](https://github.com/cortex-lab/matlab-ci/commits/master) [1.2.0]
+## [Latest](https://github.com/cortex-lab/matlab-ci/commits/master) [2.0.0]
+### Added
+
+ - there are now three major modules: lib, serve and main
+ - lots and lots of new tests
+ - support for running any tests: now CI calls a custom shell script
+ - new settings file and config module to validate settings
+ - branch pass list option in settings
+ - force flag for badge API
+ - function for converting string to bool in robust way
+ - function for saving test records
 
+
+### Modified
+
+ - complete rewrite of code
+ - index.js renamed to main.js
+ - preparing the environment may be optional
+ - errors during test function are now saved into test record db
+ - new config options (see readme)
+ - kill child processes in job timer
+
+
+## [1.2.0]
 ### Modified
 
 - test log formatted in html
@@ -23,11 +45,7 @@
 - suppress warnings about shadowing builtins in runAllTests
 - run tests in subfolders
 - filter out performance tests
-<<<<<<< HEAD
-- skip tests for commits to branches named 'documentation'
-=======
 - skip checks for commits to branches named 'documentation'
->>>>>>> master
 
 ## [1.0.0]
 ### Added

README.md

@@ -1,31 +1,73 @@
 # MATLAB-ci
+[![Build Status](https://travis-ci.com/cortex-lab/matlab-ci.svg?branch=master)](https://travis-ci.com/cortex-lab/matlab-ci)
 
 A small set of modules written in Node.js for running automated tests of MATLAB code in response to GitHub events.  Also submits code coverage to the Coveralls API.
 
+Currently unsupported:
+* Running tests on forked repositories
+* Testing multiple repos (unless they are submodules) 
+
 ## Getting Started
 
 Run the install script to install all dependencies, then create your .env file containing your App's tokens, secrets, etc.
 
 ### Prerequisites
 
-Requires MATLAB 2017a or later, Node.js and Git Bash.  The following Node.js modules are required:
+Requires Git Bash, npm v6.14 or later and Node.js v12.19.0 or later.  For MATLAB tests use MATLAB 2017a or later.
 
 ```
-npm install --save express dotenv @octokit/app @octokit/request ...
-github-webhook-handler xml2js localtunnel
+npm install ./matlab-ci
 ```
 
 ### Installing
 
-Make sure runAllTests.m is on your MATLAB paths
+Create a shell/batch script for preparing your environment, and one for running the tests (i.e. calling Python or MATLAB).
+Add these to the settings.json file in config:
+```
+{
+  "setup_function": "./prep_env.BAT",
+  "test_function": "./run_tests.BAT",
+  "listen_port": 3000,
+  "timeout": 480000,
+  "program": "python",
+  "strict_coverage": false,
+  "events": {
+    "push": {
+      "checks": null,
+      "ref_ignore": ["documentation", "gh-pages"]
+    },
+    "pull_request": {
+      "checks": ["continuous-integration", "coverage"],
+      "actions": ["opened", "synchronize", "reopened"],
+      "ref_ignore": ["documentation", "gh-pages"]
+    }
+  }
+}
+``` 
+Some extra optional settings:
+
+- `shell` - optional shell to use when calling scripts (see `child_process.execFile` options).
+- `events:event:ref_include` - same as `ref_ignore`, but a pass list instead of block list.
+- `kill_children` - if present and true, `tree-kill` is used to kill the child processes, required 
+if shell/batch script forks test process (e.g. a batch script calls python).
+
+Finally, ensure these scripts are executable by node:
+```
+chmod u+x ./run_tests.BAT
+chmod u+x ./prep_env.BAT
+```
 
 ## Running the tests
 
-TODO
+```
+mocha ./test
+```
 
 ## Deployment
 
-To work properly you need to create install a Github app on your target repository and download the private key.  Update your .env file like so:
+To work properly you need to create install a 
+[Github app](https://docs.github.com/en/free-pro-team@latest/developers/apps/creating-a-github-app)
+on your target repository and download the private key.  Update your .env file like so:
 
 ```
 GITHUB_PRIVATE_KEY=path\to\private-key.pem
@@ -42,11 +84,17 @@ TUNNEL_SUBDOMAIN=
 To run at startup create a batch file with the following command:
 
 ```batch
-cmd /k node -r dotenv/config dotenv_config_path=/Path/To/Env/Vars ./Path/To/index.js 
+cmd /k node -r dotenv/config dotenv_config_path=/Path/To/Env/Vars ./Path/To/main.js 
 ```
 
 Create a shortcut in your startup folder ([Windows-logo] + [R] in Windows-10 and enter the command `shell:startup`)
 
+## Test script
+Your test script must do the following:
+1. Accept a commit ID as an input arg
+2. Save the results into the JSON cache file without duplication
+3. For code coverage the script must either save the coverage directly, or export a Cobertura formatted XML file.
+
 ## Built With
 
 * [LocalTunnel](https://localtunnel.me) - A secure tunneling service

config/config.js

@@ -0,0 +1,79 @@
+const userSettings = require('./settings.json') || {};  // User settings
+const path = require('path');
+env = process.env.NODE_ENV || 'production';
+const appdata = process.env.APPDATA || process.env.HOME;
+const dataPath = process.env.APPDATA? path.join(appdata, 'CI') : path.join(appdata, '.ci');
+const fixtureDir = path.resolve(__dirname, '..', 'test', 'fixtures');
+const dbFilename = '.db.json';
+let settings;
+
+// Defaults for when there's no user file; will almost certainly fail
+defaults = {
+    setup_function: null,
+    test_function: null,
+    listen_port: 3000,
+    timeout: 8*60000,
+    program: "python",
+    strict_coverage: false,
+    events: {
+        push: {
+            checks: null,
+            ref_ignore: ["documentation", "gh-pages"]
+        },
+        pull_request: {
+            checks: ["continuous-integration", "coverage"],
+            actions: ["opened", "synchronize", "reopen"],
+            ref_ignore: ["documentation", "gh-pages"]
+        }
+    },
+    dataPath: dataPath,
+    dbFile: path.join(dataPath, dbFilename)
+}
+
+// Settings for the tests
+testing = {
+    listen_port: 3000,
+    timeout: 60000,
+    setup_function: 'prep_env.BAT',
+    test_function: "run_tests.BAT",
+    events: {
+        push: {
+            checks: "continuous-integration",
+            ref_ignore: "documentation"
+        },
+        pull_request: {
+            checks: ["coverage", "continuous-integration"],
+            actions: ["opened", "synchronize"],
+            ref_ignore: ["documentation", "gh-pages"]
+        }
+    },
+    dataPath: fixtureDir,
+    dbFile: path.join(fixtureDir, dbFilename)  // cache of test results
+}
+
+// Pick the settings to return
+if (env.startsWith('test')) {
+  settings = testing;
+} else if (userSettings) {
+  settings = userSettings;
+  if (!('dbFile' in settings)) {
+    settings.dbFile = path.join(dataPath, dbFilename)
+  }
+  if (!('dataPath' in settings)) {
+      settings.dataPath = dataPath;
+  }
+} else {
+  settings = defaults;
+}
+
+// Check ENV set up correctly
+required = ['GITHUB_PRIVATE_KEY', 'GITHUB_APP_IDENTIFIER', 'GITHUB_WEBHOOK_SECRET',
+            'WEBHOOK_PROXY_URL', 'REPO_PATH', 'REPO_NAME', 'REPO_OWNER', 'TUNNEL_HOST',
+            'TUNNEL_SUBDOMAIN'];
+missing = required.filter(o => { return !process.env[o] });
+if (missing.length > 0) {
+    errMsg = `Env not set correctly; the following variables not found: \n${missing.join(', ')}`
+    throw ReferenceError(errMsg)
+}
+
+module.exports = { settings }

config/settings.json

@@ -0,0 +1,19 @@
+{
+  "setup_function": "prep_env.BAT",
+  "test_function": "run_tests.BAT",
+  "listen_port": 3000,
+  "timeout": 480000,
+  "program": "python",
+  "strict_coverage": false,
+  "events": {
+    "push": {
+      "checks": null,
+      "ref_ignore": ["documentation", "gh-pages"]
+    },
+    "pull_request": {
+      "checks": ["continuous-integration", "coverage"],
+      "actions": ["opened", "synchronize", "reopened"],
+      "ref_ignore": ["documentation", "gh-pages"]
+    }
+  }
+}