Merge remote-tracking branch 'origin/staging' into insights_ui_tests

.gitlab-ci.yml

@@ -65,12 +65,6 @@ discover-submodules:
   dependencies:
     - discover-submodules
 
-wait-api-image:
-  <<: *docker_wait_submodule_img_template
-  before_script:
-    - REPO=mistio/mist.api
-    - IMG=mist
-
 wait-ui-image:
   <<: *docker_wait_submodule_img_template
   before_script:
@@ -90,6 +84,46 @@ wait-tests-image:
     - IMG=tests_base
 
 
+# Rebuild api image.
+
+rebuild-api-image:
+  stage: build
+  variables:
+    NUM: "60"
+    SLEEP: "10"
+    REPO: mistio/mist.api
+    IMG: mist
+  script:
+    - SUBMODULE_SHA=$(grep ^$REPO, all-submodules.csv | cut -d, -f2)
+    - echo SUBMODULE_SHA is $SUBMODULE_SHA
+    - GCR_IMG="gcr.io/mist-ops/$IMG"
+    - DOCKERHUB_IMG="mist/$IMG"
+    - IMGTAG="$GCR_IMG:$SUBMODULE_SHA"
+    - echo IMTAG is $IMGTAG
+
+    - echo "Will pull docker image $IMGTAG"
+    - for i in $(seq 1 $NUM); do docker pull $IMGTAG && OK=1 && break || sleep $SLEEP; done
+    - if [ -z "$OK" ]; then echo "ERROR Couldn't pull $IMGTAG!" && exit 1; fi
+
+    - echo "Will rebuild api image"
+    - sed -i "s~^FROM.*~FROM $IMGTAG~" docker/mist/Dockerfile
+    - docker build -t $IMG/io-$CI_COMMIT_SHA --build-arg IO_VERSION_SHA=$CI_COMMIT_SHA --build-arg IO_VERSION_NAME=$CI_COMMIT_REF_NAME docker/mist
+
+    - |
+        for img_name in $GCR_IMG $DOCKERHUB_IMG; do
+          for ref in $CI_COMMIT_SHA $CI_COMMIT_REF_SLUG; do
+            IMGTAG2=$img_name:io-$ref
+            echo "Will tag and push $IMGTAG2"
+            docker tag $IMG/io-$CI_COMMIT_SHA $IMGTAG2
+            docker push $IMGTAG2
+          done
+        done
+  tags:
+    - shell-docker-light
+  dependencies:
+    - discover-submodules
+
+
 # Build secondary images needed for deploying. Tag with SHA and branch name.
 
 .docker_build_img_template: &docker_build_img_template

README.md

@@ -20,45 +20,37 @@ Recommended hardware resources are:
 
 Mist.io is a large application split into microservices which are packages in
 docker containers. The easiest way to run it is by using `docker-compose`. So,
-in order to run it, one needs to install a recent version of `docker` and
-`docker-compose`.
-
-There are two ways to run this application with docker-compose:
-
-
-### Single file deployment
+in order to run it, one needs to install a recent version of
+[docker](https://docs.docker.com/engine/installation/) and
+[docker-compose](https://docs.docker.com/compose/install/).
 
 If you want to install a stable release, head over to mist.io's github
 [releases](https://github.com/mistio/mist.io/releases/) and follow the
 instructions there.
 
 Alternatively, if you want to run the latest development version of mist.io,
-simply download the `docker-compose.yml` file from this repository and put it
-in a directory. The directory name is used by `docker-compose` as the `project`
-name, so use something descriptive and unique, like `mist.io`.
+run the following:
 
-You also need to set the environmental variable `MIST_TAG=staging`. Do that by
-either running `export MIST_TAG=staging` or for it to persist, do
-`echo MIST_TAG=staging > .env`.
+```bash
+mkdir mist.io && cd mist.io && echo 'MIST_TAG=master' > .env
+wget https://raw.githubusercontent.com/mistio/mist.io/master/docker-compose.yml
+docker-compose up -d
+```
 
+After a few minutes (depending on your connection) all mist.io containers will
+be downloaded and started in the background.
 
-### Development deployment
+Run `docker-compose ps`. All containers should be in the UP state, except
+shortlived container elasticsearch-manage.
 
-Clone this git repo and all its submodules with something like:
-
-    git clone --recursive https://github.com/mistio/mist.io.git
-    cd mist.io
-
-This may take some time.
-
-This setup will mount the checked out code into the containers. By cloning the
-directory, now there's also a `docker-compose.override.yml` file in the current
-directory in addition to `docker-compose.yml` and is used to modify the
-configuration for development mode.
+To run a different mist.io version, replace `master` with a different branch's
+name in the above `echo` and `wget` commands.
 
 
 ## Running mist.io
 
+Make sure you're inside the directory containing the `docker-compose.yml` file.
+
 Switch to the directory containing the `docker-compose.yml` file and run
 
     docker-compose up -d
@@ -72,28 +64,56 @@ To create a user for the first time, first run
 This should drop you in a shell into one of the mist.io containers. In there,
 run
 
-    ./bin/adduser --admin --docker-cloud [email protected]
+    ./bin/adduser --admin [email protected]
 
 Replace the email address with yours. Try running `./bin/adduser -h` for more
 options. The `--docker-cloud` flag will add the docker daemon hosting the
 mist.io installation as a docker cloud in the created account.
 
-Visit http://localhost and login with the email and password specified above.
+Mist.io binds on port 80 of the host. Visit http://localhost and login with the
+email and password specified above.
 
 Welcome to mist.io! Enjoy!
 
 
 ## Configuring mist.io
 
 After the initial `docker-compose up -d`, you'll see that a configuration file
-is created in `./config/settings.py`. Edit this file to modify configuration
-and restart services (for example with `docker-compose restart`).
+is created in `./config/settings.py`. Edit this file to modify configuration.
+Any changes to the `./config/settings.py` require a restart to take effect:
+
+    docker-compose restart
+
+
+### Required configuration
+
+#### URL
+
+If running on anything other than `localhost`, you'll need to set the
+`CORE_URI` setting in `./config/settings.py`. Example:
+
+    CORE_URI = "http://198.51.100.12"
+
+
+### Mail settings
+
+In some cases, such as user registration, forgotten passwords, user invitations
+etc, mist.io needs to send emails. By default, mist.io is configured to use a
+mock mailer. To see logs sent by mist.io, run
+
+    docker-compose logs -f mailmock
+
+If you wish to use a real SMTP server, edit `./config/settings.py` and modify
+`MAILER_SETTINGS`.
+
+Don't forget to restart docker-compose for changes to take effect.
 
 
 ## Managing mist.io
 
 Mist.io is managed using `docker-compose`. Look that up for details. Some
-useful commands:
+useful commands follow. Keep in mind that you need to run these from inside the
+directory containing the `docker-compose.yml` file:
 
     # See status of all applications
     docker-compose ps
@@ -123,3 +143,24 @@ useful commands:
 
     # Completely remove all containers and data volumes.
     docker-compose down -v
+
+
+## Development deployment
+
+If you're planning to modify mist.io's source code, an alternative installation
+method is recommended.
+
+Clone this git repo and all its submodules with something like:
+
+    git clone --recursive https://github.com/mistio/mist.io.git
+    cd mist.io
+    docker-compose up -d
+
+This may take some time.
+
+This setup will mount the checked out code into the containers. By cloning the
+directory, now there's also a `docker-compose.override.yml` file in the current
+directory in addition to `docker-compose.yml` and is used to modify the
+configuration for development mode.
+
+The above instructions for running and managing mist.io apply.

docker-compose.override.yml

@@ -39,6 +39,7 @@ services:
       - ./elasticsearch:/elasticsearch
   scheduler: *backend
   poller: *backend
+  beat: *backend
   hubshell: *backend
   api: *backend
   sockjs: *backend

docker-compose.yml

@@ -86,6 +86,10 @@ services:
     <<: *backend
     command: bin/wait-all bin/poller
 
+  beat:
+    <<: *backend
+    command: bin/wait-all bin/beat
+
   hubshell:
     <<: *backend
     command: bin/wait-all bin/hubshell

docker/mist/Dockerfile

@@ -0,0 +1,14 @@
+FROM mist/mist:io-staging
+
+ARG IO_VERSION_SHA
+ARG IO_VERSION_NAME
+
+# Variables defined solely by ARG are accessible as environmental variables
+# during build but not during runtime. To persist these in the image, they're
+# redefined as ENV in addition to ARG.
+ENV VERSION_REPO=mistio/mist.io \
+    VERSION_SHA=$IO_VERSION_SHA \
+    VERSION_NAME=$IO_VERSION_NAME
+
+RUN echo "{\"sha\":\"$VERSION_SHA\",\"name\":\"$VERSION_NAME\",\"repo\":\"$VERSION_REPO\",\"modified\":false}" \
+        > /mist-version.json