Merge pull request #7 from simon-dube/develop

Executions and CLI

.gitignore

@@ -3,3 +3,4 @@
 .pytest_cache/*
 *.db
 /carmin-server/.coverage
+/CONFIG.json

.travis.yml

@@ -1,5 +1,10 @@
+sudo: required
+
 language: python
 
+services:
+    - docker
+
 python:
 - 3.4
 - 3.5

README.md

@@ -161,12 +161,8 @@ Let's add some data to the server with the `PUT /path/{completePath}` method:
 ```bash
 curl -X "PUT" "http://localhost:8080/path/admin/new_user.txt" \
      -H 'apiKey: [secret-api-key]' \
-     -d $'{
-  "type": "File",
-  "base64Content": "bmV3IENBUk1JTiB1c2VyCg=="
-}'
+     -d "Jane Doe"
 ```
-
 The server should reply with a `201: Created` code, indicating that the resource was successfully
 uploaded to the server.
 
@@ -186,7 +182,7 @@ The server should return a `Path` object, which describes the resource that we u
   "platformPath": "http://localhost:8080/path/admin/new_user.txt",
   "lastModificationDate": 1521740108,
   "isDirectory": false,
-  "size": 12,
+  "size": 8,
   "mimeType": "text/plain"
 }
 ```

VERSION

@@ -0,0 +1 @@
+0.1

cli/carmin_server.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""CARMIN-Server
+
+A lightweight server for the execution of remote pipelines.
+
+Usage:
+    carmin-server [--help] [--version] COMMAND [OPTIONS...]
+
+Options:
+    -h, --help     Print help page and quit
+    -v, --version  Print version information and quit
+
+Commands:
+    setup  Install and configure the server
+    run    Launch the server
+    """
+
+from subprocess import call
+from pathlib import Path
+from docopt import docopt
+from cli_helper import project_root
+
+
+def get_version():
+    root_path = project_root()
+    version_file = open(Path(root_path, 'VERSION'))
+    return version_file.read().strip()
+
+
+if __name__ == '__main__':
+    args = docopt(__doc__, options_first=True, version=get_version())
+
+    argv = [args['COMMAND']] + args['OPTIONS']
+
+    if args['COMMAND'] == 'setup':
+        import carmin_server_setup
+        try:
+            exit(call(['python3', 'carmin_server_setup.py'] + argv))
+        except KeyboardInterrupt:
+            exit()
+    elif args['COMMAND'] == 'run':
+        import carmin_server_run
+        try:
+            exit(call(['python3', 'carmin_server_run.py'] + argv))
+        except KeyboardInterrupt:
+            pass
+    elif args['COMMAND'] in ['help', None]:
+        exit(call(['python3', 'carmin_server.py', '--help']))
+    else:
+        exit("{} is not a carmin-server command. See 'carmin-server --help.'".
+             format(args['COMMAND']))

cli/carmin_server_run.py

@@ -0,0 +1,51 @@
+"""Usage: carmin-server run [options]
+
+Launches the server
+
+Options:
+    -p <port>, --port <port>  The server will listen on this port
+    -c, --container           Launch the server inside a Docker container
+    """
+
+import json
+from subprocess import call
+from pathlib import Path
+from docopt import docopt
+
+from cli_helper import project_root
+
+
+def config_dict():
+    root_dir = Path(__file__).resolve().parent.parent
+    config_file = Path(root_dir, 'CONFIG.json')
+    try:
+        with open(config_file) as f:
+            return json.load(f)
+    except FileNotFoundError:
+        return {}
+    except TypeError:
+        return {}
+
+
+CONFIG = config_dict()
+
+if __name__ == '__main__':
+    args = docopt(__doc__)
+    port = args.get('--port') or '8080'
+    try:
+        port = int(port)
+    except ValueError:
+        print("Invalid port number. Port must be an integer.")
+        exit(1)
+    if args.get('--container'):
+        call(['docker', 'build', '-t=carmin-server', '..'])
+        call([
+            'docker', 'run', '-p', '{}:8080'.format(port), '-e',
+            'DATABASE_URI="sqlite:////carmin-db/app.db"', '-v',
+            '{}:/carmin-assets/pipelines'.format(
+                CONFIG.get('PIPELINE_DIRECTORY')),
+            '-v', '{}:/carmin-assets/data'.format(
+                CONFIG.get('DATA_DIRECTORY')), 'carmin-server'
+        ])
+    else:
+        call(['python3', '-m', 'server', str(port)], cwd=project_root())

cli/carmin_server_setup.py

@@ -0,0 +1,63 @@
+"""Usage: carmin-server setup [options]
+
+Options:
+    -p <path>, --pipeline-directory <path>  Specify path for pipeline directory
+    -d <path>, --data-directory <path>      Specify path for data directory
+    -w <path>, --database <path>            Specify path for database
+
+"""
+
+import json
+from pathlib import Path
+from docopt import docopt
+
+
+def is_interactive(invocation_list):
+    return not (invocation_list.get('--database')
+                and invocation_list.get('--pipeline-directory')
+                and invocation_list.get('--data-directory'))
+
+
+def write_to_config_file(config):
+    root_dir = Path(__file__).resolve().parent.parent
+    config_file = Path(root_dir, 'CONFIG.json')
+    with open(config_file, 'w') as f:
+        json.dump(config, f)
+
+
+def print_install_banner():
+    width = 50
+    delimiter = '-' * width
+    print('{0}\nCARMIN-Server Setup (Press CTRL-C to quit)\n{0}'.format(
+        delimiter))
+
+
+ask_pipeline = "Enter path to pipeline directory: "
+ask_data = "Enter path to data directory: "
+ask_database = "Enter path or URI to the database (to use the default sqlite database, leave blank): "
+
+if __name__ == '__main__':
+    args = docopt(__doc__)
+    try:
+        if is_interactive(args):
+            print_install_banner()
+            step_count = 1
+            if not args.get('--pipeline-directory'):
+                pipeline_path = input('{}. {}'.format(step_count,
+                                                      ask_pipeline))
+                step_count += 1
+            if not args.get('--data-directory'):
+                data_path = input('{}. {}'.format(step_count, ask_data))
+                step_count += 1
+            if not args.get('--database'):
+                database_path = input('{}. {}'.format(step_count,
+                                                      ask_database))
+            config_dict = {
+                "PIPELINE_DIRECTORY": pipeline_path,
+                "DATA_DIRECTORY": data_path,
+                "DATABASE_URL": database_path
+            }
+            write_to_config_file(config_dict)
+        exit("\nCARMIN-Server was successfully configured.")
+    except KeyboardInterrupt:
+        exit()

cli/cli_helper.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def project_root():
+    return Path(__file__).resolve().parent.parent

server/__init__.py

@@ -1,15 +1,26 @@
+import sys
 from server.server_helper import create_app
 from server.api import declare_api
 
 app = create_app()
 
 from server.logging.setup import log_response, log_exception
+from server.resources.post_processors import *
 
 
 def main():
     declare_api(app)
     start_up()
-    app.run(host='0.0.0.0', port=int(8080))
+    if len(sys.argv) > 1:
+        port = sys.argv[1]
+        try:
+            port = int(port)
+        except ValueError:
+            print("Invalid port number. Port must be an integer.")
+            exit(1)
+    else:
+        port = 8080
+    app.run(host='0.0.0.0', port=port)
 
 
 from server.startup_validation import start_up

server/common/error_codes_and_messages.py

@@ -77,6 +77,28 @@ def errors_as_list() -> List:
     120, "Invalid value '{}' for query parameter '{}'.")
 CANNOT_MODIFY_PARAMETER = ErrorCodeAndMessage(
     125, "'{}' cannot be modified on an existing Execution.")
-INVALID_INVOCATION = ErrorCodeAndMessage(130, "Invalid invocation")
+INVOCATION_INITIALIZATION_FAILED = ErrorCodeAndMessage(
+    130,
+    "The execution was created with identifier '{}', but its initialization failed during the invocation validation."
+)
 CANNOT_REPLAY_EXECUTION = ErrorCodeAndMessage(
     135, "An execution cannot be replayed. Current status: '{}'")
+INVALID_EXECUTION_TIMEOUT = ErrorCodeAndMessage(
+    140, "Invalid execution timeout. Must be between {} and {} seconds.")
+CANNOT_KILL_NOT_RUNNING_EXECUTION = ErrorCodeAndMessage(
+    145, "Cannot kill a non running execution. Current status: '{}'")
+CANNOT_KILL_FINISHING_EXECUTION = ErrorCodeAndMessage(
+    150,
+    "The execution processes are not running, thus the execution is most probably finishing and cannot be killed."
+)
+CANNOT_GET_RESULT_NOT_COMPLETED_EXECUTION = ErrorCodeAndMessage(
+    155,
+    "The execution is not done yet, thus results cannot be queried. Current status: '{}'. Please try again later."
+)
+CORRUPTED_EXECUTION = ErrorCodeAndMessage(
+    160,
+    "There is an unrecoverable problem with this execution. Please create a new execution."
+)
+UNSUPPORTED_DESCRIPTOR_TYPE = ErrorCodeAndMessage(
+    165, "The descriptor type '{}' is not supported.")
+PAGE_NOT_FOUND = ErrorCodeAndMessage(404, "Page Not Found")

server/config.py

@@ -1,12 +1,6 @@
 import os
 basedir = os.path.abspath(os.path.dirname(__file__))
 
-SUPPORTED_PROTOCOLS = ["http", "https", "ftp", "sftp", "ftps", "scp", "webdav"]
-
-SUPPORTED_MODULES = [
-    "Processing", "Data", "AdvancedData", "Management", "Commercial"
-]
-
 DEFAULT_PROD_DB_URI = os.path.join(basedir, 'database/app.db')
 SQLITE_DEFAULT_PROD_DB_URI = 'sqlite:///{}'.format(DEFAULT_PROD_DB_URI)
 

server/database/models/execution.py

@@ -21,6 +21,7 @@ class Execution(db.Model):
         identifier (str):
         name (str):
         pipeline_identifier (str):
+        descriptor (str):
         timeout (int):
         status (ExecutionStatus):
         study_identifier (str):
@@ -33,6 +34,7 @@ class Execution(db.Model):
         identifier (str):
         name (str):
         pipeline_identifier (str):
+        descriptor (str):
         timeout (int):
         status (ExecutionStatus):
         study_identifier (str):
@@ -45,6 +47,7 @@ class Execution(db.Model):
     identifier = Column(String, primary_key=True, default=execution_uuid)
     name = Column(String, nullable=False)
     pipeline_identifier = Column(String, nullable=False)
+    descriptor = Column(String, nullable=False)
     timeout = Column(Integer)
     status = Column(Enum(ExecutionStatus), nullable=False)
     study_identifier = Column(String)

server/database/models/execution_process.py

@@ -1,5 +1,5 @@
 from flask_restful import fields
-from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy import Column, String, Integer, Boolean, ForeignKey
 from server.database import db
 
 
@@ -9,12 +9,15 @@ class ExecutionProcess(db.Model):
     Args:
         execution_identifier (str):
         pid (int):
+        is_execution(bool):
 
     Attributes:
         execution_identifier (str):
         pid (int):
+        is_execution (bool):
     """
 
     execution_identifier = Column(
         String, ForeignKey("execution.identifier"), primary_key=True)
-    pid = Column(Integer, nullable=False)
+    pid = Column(Integer, primary_key=True)
+    is_execution = Column(Boolean, nullable=False)

server/database/queries/executions.py

@@ -1,12 +1,14 @@
 from typing import List
 from server.database.models.execution import Execution
+from server.database.models.execution_process import ExecutionProcess
 
 
-def get_all_executions_for_user(username: str, db_session) -> List[Execution]:
+def get_all_executions_for_user(username: str, limit: int, offset: int,
+                                db_session) -> List[Execution]:
     return list(
         db_session.query(Execution).filter(
             Execution.creator_username == username).order_by(
-                Execution.created_at.desc()))
+                Execution.created_at.desc()).offset(offset).limit(limit))
 
 
 def get_execution(identifier: str, db_session) -> Execution:
@@ -16,3 +18,9 @@ def get_execution(identifier: str, db_session) -> Execution:
 def get_execution_count_for_user(username: str, db_session) -> int:
     return db_session.query(Execution).filter(
         Execution.creator_username == username).count()
+
+
+def get_execution_processes(execution_identifier: str,
+                            db_session) -> List[ExecutionProcess]:
+    return db_session.query(ExecutionProcess).filter(
+        ExecutionProcess.execution_identifier == execution_identifier).all()

server/logging/config.py

@@ -42,11 +42,13 @@
     },
     'loggers': {
         'request-response': {
+            'level': 'INFO',
             'propagate': False,
             'handlers': ['request-context']
         },
         'server-error': {
-            'propagata': False,
+            'level': 'WARNING',
+            'propagate': False,
             'handlers': ['unexpected-crash']
         }
     }