From 1e9cfc821c5ca6628358444c5402757ad85de977 Mon Sep 17 00:00:00 2001 From: Dirk Roorda Date: Mon, 7 Mar 2022 12:42:36 +0100 Subject: [PATCH] cmdi metadata file --- .gitignore | 2 + docs/cmdi/SHEBANQ.cmdi.xml | 211 +++++++++++ scripts/Dockerfile | 40 --- scripts/docker/build-shebanq-db.sh | 1 + scripts/docker/configtemplate.sh | 112 ++++++ scripts/docker/doconfig.sh | 42 +++ scripts/docker/fill.py | 37 ++ scripts/docker/functions.sh | 281 +++++++++++++++ scripts/docker/install.sh | 467 +++++++++++++++++++++++++ scripts/docker/localize.sh | 29 ++ scripts/docker/provision.sh | 171 +++++++++ scripts/docker/shebanq-db/Dockerfile | 9 + scripts/docker/shebanq-db/shebanq.cnf | 11 + scripts/docker/shebanq/Dockerfile | 53 +++ scripts/docker/shebanq/grants.sql | 10 + scripts/docker/shebanq/unconfigure.sql | 13 + scripts/docker/shebanq/wsgi.conf | 17 + scripts/docker/templates/apache.conf | 95 +++++ scripts/docker/templates/host.cfg | 1 + scripts/docker/templates/mail.cfg | 2 + scripts/docker/templates/mql.cfg | 1 + scripts/docker/templates/mqlimportopt | 1 + scripts/docker/templates/mysqldumpopt | 3 + scripts/docker/templates/mysqlrootopt | 9 + scripts/docker/templates/staticdata.sh | 27 ++ scripts/docker/templates/user.sql | 4 + 26 files changed, 1609 insertions(+), 40 deletions(-) create mode 100644 docs/cmdi/SHEBANQ.cmdi.xml delete mode 100644 scripts/Dockerfile create mode 100644 scripts/docker/build-shebanq-db.sh create mode 100644 scripts/docker/configtemplate.sh create mode 100644 scripts/docker/doconfig.sh create mode 100644 scripts/docker/fill.py create mode 100644 scripts/docker/functions.sh create mode 100755 scripts/docker/install.sh create mode 100755 scripts/docker/localize.sh create mode 100755 scripts/docker/provision.sh create mode 100644 scripts/docker/shebanq-db/Dockerfile create mode 100644 scripts/docker/shebanq-db/shebanq.cnf create mode 100644 scripts/docker/shebanq/Dockerfile create mode 100644 scripts/docker/shebanq/grants.sql create mode 100644 scripts/docker/shebanq/unconfigure.sql create mode 100644 scripts/docker/shebanq/wsgi.conf create mode 100644 scripts/docker/templates/apache.conf create mode 100644 scripts/docker/templates/host.cfg create mode 100644 scripts/docker/templates/mail.cfg create mode 100644 scripts/docker/templates/mql.cfg create mode 100644 scripts/docker/templates/mqlimportopt create mode 100644 scripts/docker/templates/mysqldumpopt create mode 100644 scripts/docker/templates/mysqlrootopt create mode 100755 scripts/docker/templates/staticdata.sh create mode 100644 scripts/docker/templates/user.sql diff --git a/.gitignore b/.gitignore index b93b6c78..891b3588 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ site/* build/* log/* _local +_local/* +_docker/* diff --git a/docs/cmdi/SHEBANQ.cmdi.xml b/docs/cmdi/SHEBANQ.cmdi.xml new file mode 100644 index 00000000..91b66da3 --- /dev/null +++ b/docs/cmdi/SHEBANQ.cmdi.xml @@ -0,0 +1,211 @@ + + +
+ rogierkraf + 2013-11-30+02:00 + clarin.eu:cr1:p_1342181139640 + CLARIN Netherlands
+ + + + SearchPage + https://shebanq.ancient-data.org/ + + + Resource + https://github.com/ETCBC/bhsa + + + Resource + https://doi.org/10.5281/zenodo.1007624 + + + Resource + https://doi.org/10.17026/dans-2z3-arxf + + + Resource + https://doi.org/10.17026/dans-z6y-skyh + + + + + + + + + SHEBANQ + SHEBANQ: System for HEBrew Text: ANnotations for Queries and Markup + 2014 + https://shebanq.ancient-data.org/DANShttps://github.com/ETCBC/shebanq/blob/master/docs/cmdi/SHEBANQ.cmdi.xml + + published + 2022-03-07 + + CLARIN-NLCLARIN in the Netherlands184.021.003NWOhttp://www.clarin.nlJan OdijkNational Coordinator
Utrecht, the Netherlands
j.odijk@uu.nlUiL-OTSUtrecht University20092015NetherlandsNL + The Hebrew Text Database by the ETCBC (Eep Talstra Centre for Bible and Computer, formerly WIVU Werkgroep Informatica Vrije Universiteit) contains the Biblia Hebraica Stuttgartensia Amstelodamensis (BHSA) dataset, which consists of the BHS version of the text of the Hebrew Bible plus linguistic annotations by the ETCBC. The annotations of the ETCBC have been produced in the form of features of words, phrases, clauses, sentences, etc. The information in these features result from algorithmic linguistic analysis aided by human decisions. The database can be queried by means of a language that is optimized to deal with data that is modeled as objects + features. SHEBANQ builds a bridge between the linguistically annotated Hebrew Text corpus and biblical scholars by (1) making this text, including its annotations, available to scholars; (2) demonstrating how queries can function to address research questions; the query saver and the metadata added to them will be a growing repository of valuable best practices of what queries are used in addressing research questions and how they contribute to answering these questions; (3) giving textual scholarship a more empirical basis, by creating the opportunity that claims made in scholarly articles (e.g.: “this syntactic pattern is not attested elsewhere in the Hebrew Bible”) can be accompanied by the unique identifiers that refer to the saved queries that have led to the claim. The WIVU database is a resource under long-term development. New features are being added, new corrections are being made over time. When SHEBANQ was created, it was fed with textual/linguistic data in LAF format, produced by an adhoc conversion. Nowadays SHEBANQ's data resides in https://github.com/ETCBC/bhsa, and there is a pipeline from the ETCBC production environment to the BHSA on GitHub and from there to the databases that power SHEBANQ. See https://github.com/ETCBC/pipeline. + + + + search tool + written language tool + corpus exploration + corpus searching + browsing + analysis + visualisation + Browsing and SearchingData analysis + Linguistics + Religion Studies + + + historical linguistics + + + + + + syntax + + + + + + morpho-syntax + + + + + + yes + Hebrewhbo + yes + + -12 + 02 + + + + + yes + Aramaicarc + yes + + -12 + 02 + + + + + + Online available + + graphical user interface + web application + + + text + + + text + UTF8 + query results + + text/plain + + + + text + UTF8 + query results + + text/csv + + + + + + Creative Commons (non-commercial) + public + + 0 + + EUR + + + + + Dirk Roorda + dirk.roorda@di.huc.knaw.nl + Data Archiving and Networked Services (DANS) + + + + + SHEBANQ - help + user + https://github.com/ETCBC/shebanq/wiki + + eng + + + + article + background on the data resource + yes + + Roorda, D. (2018). Coding the Hebrew Bible, Research Data Journal for the Humanities and Social Sciences, Volume 3, Issue 1, July 31, 2018, pp. 27-41. Leiden, Brill https://doi.org/10.1163/24523666-01000011 + + + + in bookscientific backgroundyesRoorda, D. 2017. The Hebrew Bible as Data: Laboratory - Sharing - Experiences. In: Odijk, J and van Hessen, A. (eds.) CLARIN in the Low Countries, Pp. 217–229. London: Ubiquity Press. DOI: https://doi.org/10.5334/bbi.18. License: CC-BY 4.0 + unpublished + scientific background + no + + Roorda, D. (2015). The Hebrew Bible as Data: Laboratory - Sharing - Experiences http://arxiv.org/abs/1501.01866 + + + + + article + scientific background + yes + + Roorda, D. (2014). LAF-Fabric: a data analysis tool for Linguistic Annotation Framework with an application to the Hebrew Bible, Computational Linguistics in the Netherlands Journal, Volume 4, December 2014, pp. 105-109 http://www.clinjournal.org/sites/clinjournal.org/files/08-Roorda-etal-CLIN2014.pdf and http://arxiv.org/abs/1410.0286 + + + + + + https://shebanq.ancient-data.org/static/apple-touch-icon.png + + + + + + SHEBANQ + SHEBANQ: System for HEBrew Text: ANnotations for Queries and Markup + CLARIN-NL + http://portal.clarin.nl/node/4180 + + dirk.roorda@di.huc.knaw.nl + + + + + + Prof. Dr. W.T. (Willem) van Peursen + w.t.van.peursen@vu.nl + + + + + + Python + continuous + + + + + diff --git a/scripts/Dockerfile b/scripts/Dockerfile deleted file mode 100644 index 684b1950..00000000 --- a/scripts/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && \ - apt-get install -y python3.10 python3.10-dev - -RUN apt-get update && \ - apt-get install -y libexpat1 apache2 apache2-utils ssl-cert libapache2-mod-wsgi-py3 - -RUN apt-get update && \ - apt-get install -y python3-pip && \ - pip3 install markdown - -RUN apt-get update && \ - apt-get install -y mariadb-server libmysqlclient-dev - -ARG emdrosversion="3.7.3" -ARG emdrosdir="/opt/emdros" - -WORKDIR build -COPY packages/emdros-${emdrosversion}.tar.gz . - -WORKDIR emdros-${emdrosversion} -RUN tar xf emdros-${emdrosversion}.tar.gz && \ - ./configure \ - --prefix=${emdrosdir} \ - --with-sqlite3=no \ - --with-mysql=yes \ - --with-swig-language-java=no \ - --with-swig-language-python2=no \ - --with-swig-language-python3=yes \ - --with-postgresql=no \ - --with-wx=no \ - --with-swig-language-csharp=no \ - --with-swig-language-php7=no \ - --with-bpt=no \ - --disable-debug && \ - ln -s /usr/bin/python3 /usr/bin/python && \ - make && \ - make install diff --git a/scripts/docker/build-shebanq-db.sh b/scripts/docker/build-shebanq-db.sh new file mode 100644 index 00000000..7a68087d --- /dev/null +++ b/scripts/docker/build-shebanq-db.sh @@ -0,0 +1 @@ +docker build -t shebanq-db "$DOCKER_DIR/shebanq-db" diff --git a/scripts/docker/configtemplate.sh b/scripts/docker/configtemplate.sh new file mode 100644 index 00000000..433f7bb3 --- /dev/null +++ b/scripts/docker/configtemplate.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# BEFORE YOU START !! +# +# before running any docker script: +# +# - read docker.md +# - tweak the parameters in the next section +# - read and understand the script that you want to run + +# ALL_CAPS variables are used by the other scripts +# camelCase variables only occur in this file + +# TWEAKING PART ################################################# +# +# !!!!!!!! +# CAUTION: make sure you are not editing this content in its +# location scripts/maintenance/configtemplate.sh . +# Instead, create a copy _dockerconfig/config.sh and modify that. +# The directory _dockerconfig will not be pushed online. + +# Do not tweak the original files in the scripts/docker directory +# !!!!!!!! +# +# Adapt the following settings to your situation before +# running the maintenance scripts. +# +# +# Version of the Emdros software that is in use. +# see also https://emdros.org +# +emdrosVersion="3.7.3" +# +# +# Version of the Web2py software that is in use. +# see also https://github.com/web2py/web2py +# +web2pyVersion="2.21.1-stable" +# +# +# +# Versions of the ETCBC data that you want to install/update +# NB: the shebanq software has hardcoded references to these versions. +# +STATIC_VERSIONS="4 4b c 2017 2021" +# +# +# Where backups of the user-generated data of shebanq can be found +# When installing the shebanq on a new server with a +# new database, you should make a backup of this data from the current +# server with ./backup.sh which will copy it here. +# The provision script will copy that over to the server and +# import it in the new database. +# There is a separate setting for backups made on +# the test server, the new production server, and the new other server. +# So you will not inadvertently restore a non-production backup +# to the production server. +# +BACKUP_DIR=~/Documents/shebanq/backups # point to an existing dir +# +# +# Where your local github directory resided, under which +# shebanq has been cloned. +# +githubBase=~/github +# +# +# +# Where the Apache config files are on the server +APACHE_DIR="/etc/httpd/conf.d" +# +# +# Server specifications +# +# server is the server name as internet address +# +# serverUrl: the url of shebanq when served from this server; +# +# dbHost is the host server where the mysql database resides +# We assume mysql is served on an other server. +# There is are options whether to import the static and dynamic +# data into this database. +# In that case we do create database users, administer database grants, +# and fill database tables. +# We also assume that the grants of the database server +# are not host specific, so that when we access the database from a +# new server, the same grants apply as when we used the old server. +# +# We also configure mail settings: server and sender on behalf +# of which SHEBANQ sends emails to users (for password verification) +# Make sure that your server is set up so that it is permitted +# to send mail for this user. +# If you do not want the server to send mail, put an empty value +# in the mailSender... fields. +# +# Your username on the server +# +SERVER_USER="you" #replace by your user name on the server +# +# +SERVER="machine.server.edu" # replace by your own +SERVER_URL="shebanq.mydomain.org" # replace by your own +DB_HOST="mysql.server.edu" # replace by your own +MYSQL_SHEBANQ=xxx # obtain yourself +MYSQL_SHEBANQ_ADMIN=yyy # obtain yourself +CERT_FILE=/etc/pki/tls/certs/other_server_edu.cer +CERT_KEY=/etc/pki/tls/private/other_server_edu.key +CERT_CHAIN=/etc/pki/tls/certs/other_server_edu.cer +MAIL_SERVER=localhost +MAIL_SENDER=shebanq@mydomain.org +# +# END TWEAKING PART ################################################# diff --git a/scripts/docker/doconfig.sh b/scripts/docker/doconfig.sh new file mode 100644 index 00000000..3e1e07ed --- /dev/null +++ b/scripts/docker/doconfig.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +org="etcbc" +APP="shebanq" +REPO="shebanq" +REPO_URL="https://github.com/$org/$REPO" + +# All the following settings are by convention + +sourceOrg="$githubBase/$org" +SOURCE_REPO="$sourceOrg/$REPO" +SCRIPT_SRC_DIR="$SOURCE_REPO/scripts" +DOCKER_DIR="$SOURCE_REPO/_docker" +packageDir="$SCRIPT_SRC_DIR/packages" + +SERVER_APP_DIR="/opt/web-apps" +SERVER_CFG_DIR="/opt/cfg" +SERVER_OLD_CFG_DIR="/opt/emdros/cfg" +SERVER_EMDROS_DIR="/opt/emdros" +SERVER_MQL_DIR="$SERVER_EMDROS_DIR/bin" + +STATIC_SRC_DIR="$sourceOrg/bhsa/$APP" +STATIC_ETCBC="shebanq_etcbc" +STATIC_PASSAGE="shebanq_passage" + +DYNAMIC_WEB="shebanq_web" +DYNAMIC_NOTE="shebanq_note" + +MYSQL_USER="shebanq" +MYSQL_ADMIN="shebanq_admin" + +WEB2PY_BARE="web2py_src-$web2pyVersion" +WEB2PY_FILE="$WEB2PY_BARE.zip" +WEB2PY_PATH="$packageDir/$WEB2PY_FILE" + +EMDROS_BARE="emdros-$emdrosVersion" +EMDROS_FILE="$EMDROS_BARE.tar.gz" +EMDROS_PATH="$packageDir/$EMDROS_FILE" + +TEST_CONTROLLER="hebrew/text" + +TM="/usr/bin/time -f ElapsedTime=%E" diff --git a/scripts/docker/fill.py b/scripts/docker/fill.py new file mode 100644 index 00000000..9c374a65 --- /dev/null +++ b/scripts/docker/fill.py @@ -0,0 +1,37 @@ +import sys +import re + +varRe = re.compile(r"«([^»]*)»") + + +def varRepl(match): + k = match.group(1) + if k in keyValues: + value = keyValues[k] + else: + value = "" + print(f"Undefined parameter: {k}") + return value + + +inFile = sys.argv[1] +outFile = sys.argv[2] +data = sys.argv[3] + +keyValues = {} + +for line in data.strip().split("\n"): + parts = line.strip().split("=", 1) + if len(parts) < 2: + print(f"Missing value for {parts[0]}") + continue + (k, v) = parts + keyValues[k] = v + +with open(inFile) as f: + text = f.read() + text = varRe.sub(varRepl, text) + + +with open(outFile, "w") as f: + f.write(text) diff --git a/scripts/docker/functions.sh b/scripts/docker/functions.sh new file mode 100644 index 00000000..3e35676a --- /dev/null +++ b/scripts/docker/functions.sh @@ -0,0 +1,281 @@ +#!/bin/bash + +# give help if the user asks for it + +function showUsage { + if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "-?" ]]; then + echo "$2" + exit + fi +} + +# Set some variables that depend on the situation + +function setSituation { + KEYVALUES=" + SERVER=$SERVER + SERVER_CFG_DIR=$SERVER_CFG_DIR + SERVER_MQL_DIR=$SERVER_MQL_DIR + SERVER_URL=$SERVER_URL + STATIC_VERSIONS=$STATIC_VERSIONS + STATIC_PASSAGE=$STATIC_PASSAGE + STATIC_ETCBC=$STATIC_ETCBC + DB_HOST=$DB_HOST + MYSQL_ROOT=$MYSQL_ROOT + MYSQL_SHEBANQ=$MYSQL_SHEBANQ + MYSQL_SHEBANQ_ADMIN=$MYSQL_SHEBANQ_ADMIN + CERT_FILE=$CERT_FILE + CERT_KEY=$CERT_KEY + CERT_CHAIN=$CERT_CHAIN + MAIL_SERVER=$MAIL_SERVER + MAIL_SENDER=$MAIL_SENDER + " +} + +# make sure a directory exists + +function ensureDir { + if [[ -f "$1" ]]; then + rm -rf "$1" + fi + if [[ ! -d "$1" ]]; then + mkdir -p "$1" + fi +} + +# erase a directory if it exists + +function eraseDir { + if [[ -d "$1" ]]; then + rm -rf "$1" + fi +} + +# compile the python code in a web2py application + +function compileApp { + app="$1" + fresh="$2" + + echo "o-o-o - compile $app" + + if [[ "$app" == "$APP" ]]; then + parentDir="$SERVER_APP_DIR" + else + parentDir="$SERVER_APP_DIR/web2py/applications" + fi + + appPath="$parentDir/$app" + generatedDir="$appPath/compiled" + if [[ ! -e "$generatedDir" ]]; then + mkdir "$generatedDir" + chmod -R 775 "$generatedDir" + chmod 2775 "$generatedDir" + setfacl -d -m g::rw "$generatedDir" + chown "$SERVER_USER":shebanq "$generatedDir" + fi + cmd1="import gluon.compileapp;" + cmd2="gluon.compileapp.compile_application('applications/$app')" + + cd "$SERVER_APP_DIR/web2py" + python3 -c "$cmd1 $cmd2" > /dev/null + + if [[ "$fresh" == "v" ]]; then + chown -R "$SERVER_USER":shebanq "$generatedDir" + else + chown -R --from=$SERVER_USER "$SERVER_USER":shebanq "$generatedDir" + fi + + echo "- Compile modules of $app ..." + cd "$SERVER_APP_DIR/web2py/applications/$app" + python3 -m compileall modules > /dev/null + + generatedDir="$appPath/modules/__pycache__" + if [[ ! -e "$generatedDir" ]]; then + mkdir "$generatedDIr" + chmod -R 775 "$generatedDir" + chmod 2775 "$generatedDir" + setfacl -d -m g::rw "$generatedDir" + chown "$SERVER_USER":shebanq "$generatedDir" + fi + if [[ "$fresh" == "v" ]]; then + chown -R "$SERVER_USER":shebanq "$generatedDir" + else + chown -R --from=$SERVER_USER "$SERVER_USER":shebanq "$generatedDir" + fi + + echo "- Done compiling $app." +} + +# make sure the group shebanq exists and the users apache and you +# are members of it + +function setGroups { + groupadd -f shebanq + usermod -a -G shebanq apache + usermod -a -G shebanq $SERVER_USER +} + +# clone or pull shebanq into the installation directory +# i.e. ~/shebanq-install/shebanq + +# make certain directories in a web2py app writable + +function writableDirs { + app=$1 + fresh=$2 + parentDir="$SERVER_APP_DIR" + wDirs="languages log databases cache errors sessions private uploads" + + if [[ "$app" == "web2py" ]]; then + wDirs="logs deposit" + elif [[ "$app" == "$APP" ]]; then + : + else + parentDir="$SERVER_APP_DIR/web2py/applications" + fi + echo "o-o-o - make writable dirs in $app" + echo " $wDirs" + echo " under $parentDir/$app" + appPath="$parentDir/$app" + for dir in $wDirs + do + dirPath="$appPath/$dir" + if [[ ! -e "$dirPath" ]]; then + mkdir "$dirPath" + fi + chmod 2775 "$dirPath" + setfacl -d -m g::rw "$dirPath" + if [[ "$fresh" == "v" ]]; then + chown -R "$SERVER_USER":shebanq "$dirPath" + else + chown -R --from=$SERVER_USER "$SERVER_USER":shebanq "$dirPath" + fi + done +} + +# use possibly updated parameters_443.py and routes.py and wsgihandler.py + +function additionals { + shebanqScriptDir="$SERVER_APP_DIR/$APP/scripts" + web2pyDir="$SERVER_APP_DIR/web2py" + + for pyFile in parameters_443.py routes.py wsgihandler.py + do + skip="x" + if [[ "$pyFile" == "parameters_443.py" ]]; then + sourceFile="$SERVER_INSTALL_DIR/$pyFile" + if [[ ! -f "$sourceFile" ]]; then + skip="v" + echo "Warning: administrative interface. No file $pyFile provided." + echo "In order to use the administrative interface you need this file. + After installation you can generate it here and put it into your + $SERVER_INSTALL_DIR. + Even better, copy it to you local computer in your + _local directory in your clone of the shebanq repo. + From there it will be picked up by subsequent runs of the provision script. + " + fi + else + sourceFile="$shebanqScriptDir/$pyFile" + fi + + if [[ "$skip" == "x" ]]; then + cp "$sourceFile" "$web2pyDir" + pyPath="$web2pyDir/$pyFile" + chown "$SERVER_USER":shebanq "$pyPath" + fi + done +} + + +# put shebanq into place +# i.e. copy it from the install location to the web-apps location + +function installShebanq { + fresh="$1" + if [[ "$fresh" == "v" ]]; then + label="INSTALL" + else + label="UPDATE" + fi + echo "o-o-o $label SHEBANQ o-o-o" + + if [[ "$fresh" == "v" ]]; then + ensureDir "$SERVER_APP_DIR" + chmod 755 /opt + chmod 755 "$SERVER_APP_DIR" + fi + + cloneDir="$SERVER_INSTALL_DIR/$APP-clone" + shebanqDir="$SERVER_APP_DIR/$APP" + web2pyDir="$SERVER_APP_DIR/web2py" + + # in the install procedure we wipe out an earlier shebanq at this place + # so we remove errors, logs, uploads, etc + # in the update procedure we preserve the existing shebanq + if [[ "$fresh" == "v" ]]; then + eraseDir "$shebanqDir" + fi + ensureDir "$shebanqDir" + + # copy the SHEBANQ repo from the install dir to the webapps dir + # we do not copy the hidden files, such as the big .git directory + cp -R $cloneDir/* "$shebanqDir" + + # warming up + if [[ "$fresh" == "v" ]]; then + chown -R "$SERVER_USER":shebanq "$shebanqDir" + else + chown -R --from=$SERVER_USER "$SERVER_USER":shebanq "$shebanqDir" + fi + + if [[ -e "$web2pyDir" ]]; then + # if some configs have changed, pick them up + # and place them in the web2py dir + additionals + + compileApp $APP $fresh + fi + + # make sure the writable directories exists and have the right + # ownership and security context + writableDirs "$APP" "$fresh" + + ensureDir "$SERVER_CFG_DIR" + for file in mail.cfg host.cfg mql.cfg + do + cp -r "$SERVER_INSTALL_DIR/$file" "$SERVER_CFG_DIR" + done + chown -R "$SERVER_USER":shebanq "$SERVER_CFG_DIR" +} + +# run a controller of shebanq outside the apache context + +function testController { + echo "o-o-o TEST CONTROLLER o-o-o" + + cd "$SERVER_APP_DIR/web2py" + $TM python3 web2py.py -S $APP/hebrew/text -M > /dev/null +} + +# make a first visit to SHEBANQ by means of curl + +function firstVisit { + echo "o-o-o FIRST VISIT (this might take a minute) o-o-o" + echo "o-o-o An expensive index will be computed and cached" + echo "o-o-o This may take a minute ! ..." + + protocol="https" + flags="" + if [[ "$firstVisitHttp"=="v" ]]; then + protocol="http" + flags="-L" + elif [[ "firstVisitSloppy"=="v" ]]; then + flags="-k" + fi + fullUrl="${protocol}://$SERVER_URL/$TEST_CONTROLLER" + echo "curl $flags $fullUrl" + $TM curl $flags "$fullUrl" | tail + echo "o-o-o First visit done" +} diff --git a/scripts/docker/install.sh b/scripts/docker/install.sh new file mode 100755 index 00000000..1df51b28 --- /dev/null +++ b/scripts/docker/install.sh @@ -0,0 +1,467 @@ +#!/bin/bash + +# READ THIS FIRST: maintenance.md + +# Script to install a server. +# Run it on the server with root privileges. + +# Acknowledgements + +# Thanks to Elia Fiore for installing SHEBANQ on RHEL/Centos/AlmaLinux/RockyLinux +# and giving essential feedback. + +source ${0%/*}/config.sh +source ${0%/*}/doconfig.sh +source ${0%/*}/functions.sh + + +USAGE=" +Usage: ./$(basename $0) [Options] [version] + +Without options, it runs the complete installation process for a $APP server. +By means of options, you can select exactly one step to be performed. +The server must have been provisioned. + +Options: + --python: the python programming language + --mysqlinstall: install mariadb (drop-in replacement of mysql) + --emdros: the emdros software + --mysqlconfig: configure mysql + --dynamic: load dynamic data into mysql + --static: load static data into mysql + --fetch$APP: clone or pull $REPO + --$APP: install $REPO + --web2py: install web2py + --testcontroller: run a controller in the web2py shell for testing + --apache: setup apache (assume certificates are already in place) + --adminpwd: set a password for the web2py admin app + This step is skipped in the full procedure. + It is only executed if this flag is explicitly provided. + --firstvisit: run controller from shell, outside apache, to warm up cache + --firstvisit http: + same as first visit, but use http and allow curl to follow redirects (-L) + --firstvisit sloppy: + same as first visit, while using https, force curl to accept insecure certificates (-k) + +version: + a valid SHEBANQ data version, such as 4, 4b, c, 2017, 2021 + It will restrict the data provisioning to the databases + that belong to this version. + If left out, all versions will be done. + Especially relevant if --static is passed. + +CAUTION +Take care with installing the current production server. +It might damage the current installation. +" + +showUsage "$1" "$USAGE" + +setSituation "$HOSTNAME" "Installing" "$USAGE" + +ensureDir "$SERVER_UNPACK_DIR" + +doAll="v" +doPython="x" +doEmdros="x" +doMysqlinstall="x" +doMysqlConfig="x" +doStatic="x" +doDynamic="x" +doFetchShebanq="x" +doShebanq="x" +doWeb2py="x" +doTestController="x" +doApache="x" +doAdminPwd="x" +doFirstVisit="x" +firstVisitHttp="x" +firstVisitSloppy="x" + +if [[ "$1" == "--python" ]]; then + doAll="x" + doPython="v" + shift +elif [[ "$1" == "--emdros" ]]; then + doAll="x" + doEmdros="v" + shift +elif [[ "$1" == "--mysqlinstall" ]]; then + doAll="x" + doMysqlinstall="v" + shift +elif [[ "$1" == "--mysqlconfig" ]]; then + doAll="x" + doMysqlConfig="v" + shift +elif [[ "$1" == "--static" ]]; then + doAll="x" + doStatic="v" + shift +elif [[ "$1" == "--dynamic" ]]; then + doAll="x" + doDynamic="v" + shift +elif [[ "$1" == "--fetch$APP" ]]; then + doAll="x" + doFetchShebanq="v" + shift +elif [[ "$1" == "--$APP" ]]; then + doAll="x" + doShebanq="v" + shift +elif [[ "$1" == "--web2py" ]]; then + doAll="x" + doWeb2py="v" + shift +elif [[ "$1" == "--testcontroller" ]]; then + doAll="x" + doTestController="v" + shift +elif [[ "$1" == "--apache" ]]; then + doAll="x" + doApache="v" + shift +elif [[ "$1" == "--adminpwd" ]]; then + doAll="x" + doAdminPwd="v" + shift +elif [[ "$1" == "--firstvisit" ]]; then + doAll="x" + doFirstVisit="v" + shift + if [[ "$1" == "sloppy" ]]; then + firstVisitSloppy="v" + shift + elif [[ "$1" == "http" ]]; then + firstVisitHttp="v" + shift + fi +elif [[ "$1" == --* ]]; then + echo "Unrecognized switch: $1" + echo "Do ./$(basename $0) --help for available options" + exit +fi + +if [[ "$1" == "" ]]; then + versions="$STATIC_VERSIONS" +else + versions="$1" + shift +fi + +# stuff to get the emdros stuff working + +# needed for the mql command +PATH=$PATH:$HOME/.local/bin:$HOME/bin +EMDROS_HOME=/opt/emdros +export EMDROS_HOME +PATH=$EMDROS_HOME/bin:$PATH +export PATH + +# Install procedure + +# users and groups + +setGroups +# Python + +if [[ "$doAll" == "v" || "$doPython" == "v" ]]; then + echo "o-o-o INSTALL PYTHON o-o-o" + $TM yum -q -y install python36 + $TM yum -q -y install python36-devel + $TM yum -q -y install python3-markdown + # we need the python command (for emdros compilation) + alternatives --set python /usr/bin/python3 + + # mod_wsgi: use either one of the following two lines + # depending on your system + # The first one works under RedHat Fedora SELinux + # The second one works under RHEL/Centos/AlmaLinux/RockyLinux + $TM yum -q -y install mod_wsgi + #$TM yum -q -y install python3-mod_wsgi +fi + +# MariaDB +# * install mariadb + +if [[ "$doAll" == "v" || "$doMysqlinstall" == "v" ]]; then + echo "o-o-o INSTALL MARIADB o-o-o" + $TM yum -q -y install mariadb.x86_64 + $TM yum -q -y install mariadb-devel.x86_64 + $TM yum -q -y install mariadb-server.x86_64 + + service mariadb start +fi + +# Configure mysql databases, set users and grants + +skipUsers="x" +skipGrants="x" + +if [[ "$doAll" == "v" || "$doMysqlConfig" == "v" ]]; then + echo "o-o-o CONFIGURE MYSQL o-o-o" + + eraseDir "$SERVER_CFG_DIR" + ensureDir "$SERVER_CFG_DIR" + for file in mail.cfg host.cfg mql.cfg mqlimportopt mysqldumpopt user.sql + do + cp -r "$SERVER_INSTALL_DIR/$file" "$SERVER_CFG_DIR" + done + chown -R "$SERVER_USER":shebanq "$SERVER_CFG_DIR" + + cp "$SERVER_INSTALL_DIR/shebanq.cnf" /etc/my.cnf.d/ + + if [[ "$DB_HOST" == "localhost" ]]; then + if [[ "$skipUsers" != "v" ]]; then + echo "o-o-o - create users" + mysql -u root < "$SERVER_CFG_DIR/user.sql" + fi + if [[ "$skipGrants" != "v" ]]; then + echo "o-o-o - grant privileges" + mysql -u root < "$SERVER_INSTALL_DIR/grants.sql" + fi + fi + + setsebool -P httpd_can_network_connect 1 + setsebool -P httpd_can_network_connect_db 1 +fi + +# Emdros + +if [[ "$doAll" == "v" || "$doEmdros" == "v" ]]; then + echo "o-o-o INSTALL Emdros o-o-o" + + cd "$SERVER_INSTALL_DIR" + tar xvf "$EMDROS_FILE" > /dev/null + cd "$EMDROS_BARE" + + echo "o-o-o - Emdros CONFIGURE" + $TM ./configure --prefix=$SERVER_EMDROS_DIR --with-sqlite3=no --with-mysql=yes --with-swig-language-java=no --with-swig-language-python2=no --with-swig-language-python3=yes --with-postgresql=no --with-wx=no --with-swig-language-csharp=no --with-swig-language-php7=no --with-bpt=no --disable-debug > /dev/null + + echo "o-o-o - Emdros MAKE (may take 5-10 minutes)" + echo "There will be some alarming warnings, but that's ok" + $TM make > /dev/null + + echo "o-o-o - Emdros INSTALL" + echo "There will be some warnings, but that's ok" + $TM make install > /dev/null + chown -R "$SERVER_USER":shebanq "$SERVER_EMDROS_DIR" +fi + +# Import dynamic data: +# user-generated-content databases, previously saved in a backup + +if [[ "$DB_HOST" == "localhost" ]]; then + if [[ "$doAll" == "v" || "$doDynamic" == "v" ]]; then + echo "o-o-o LOAD DYNAMIC DATA start o-o-o" + for db in $DYNAMIC_NOTE $DYNAMIC_WEB + do + echo "o-o-o - clearing $db" + mysql --defaults-extra-file=$SERVER_CFG_DIR/mysqldumpopt -e "drop database if exists $db;" + mysql --defaults-extra-file=$SERVER_CFG_DIR/mysqldumpopt -e "create database $db;" + done + + for db in $DYNAMIC_WEB $DYNAMIC_NOTE + do + echo "o-o-o - unzipping $db" + cp "$SERVER_INSTALL_DIR/$db.sql.gz" "$SERVER_UNPACK_DIR" + $TM gunzip -f "$SERVER_UNPACK_DIR/$db.sql.gz" + + echo "o-o-o - loading $db" + echo "use $db" | cat - $SERVER_UNPACK_DIR/$db.sql | mysql --defaults-extra-file=$SERVER_CFG_DIR/mysqldumpopt + + rm "$SERVER_UNPACK_DIR/$db.sql" + done + echo "o-o-o LOAD DYNAMIC DATA end o-o-o" + fi +fi + +# Import static data: +# passage databases +# emdros databases + +skipPdb="x" +skipEdb="x" + +if [[ "$doAll" == "v" || "$doStatic" == "v" ]]; then + if [[ "$DB_HOST" == "localhost" || "$doAll" == "x" ]]; then + echo "o-o-o LOAD STATIC DATA start o-o-o" + + mysqlOpt="--defaults-extra-file=$SERVER_CFG_DIR/mysqldumpopt" + + for version in $versions + do + if [[ "$skipPdb" != "v" ]]; then + echo "o-o-o - VERSION $version ${STATIC_PASSAGE}" + db="$STATIC_PASSAGE$version" + echo "o-o-o - unzipping $db" + cp "$SERVER_INSTALL_DIR/$db.sql.gz" "$SERVER_UNPACK_DIR" + $TM gunzip -f "$SERVER_UNPACK_DIR/$db.sql.gz" + echo "o-o-o - loading $db (may take half a minute)" + $TM mysql $mysqlOpt < "$SERVER_UNPACK_DIR/$db.sql" + rm "$SERVER_UNPACK_DIR/$db.sql" + fi + + if [[ "$skipEdb" != "v" ]]; then + echo "o-o-o - VERSION $version ${STATIC_ETCBC}" + db="$STATIC_ETCBC$version" + echo "o-o-o - unzipping $db" + cp $SERVER_INSTALL_DIR/$db.mql.bz2 $SERVER_UNPACK_DIR + $TM bunzip2 -f $SERVER_UNPACK_DIR/$db.mql.bz2 + echo "o-o-o - dropping $db" + mysql $mysqlOpt -e "drop database if exists $db;" + echo "o-o-o - importing $db (may take a minute or two)" + mqlPwd=`cat $SERVER_CFG_DIR/mqlimportopt` + mqlOpt="-e UTF8 -n -b m -h "$DB_HOST" -u $MYSQL_ADMIN" + $TM $SERVER_MQL_DIR/mql $mqlOpt -p $mqlPwd < $SERVER_UNPACK_DIR/$db.mql + rm "$SERVER_UNPACK_DIR/$db.mql" + fi + done + echo "o-o-o LOAD STATIC DATA end o-o-o" + fi +fi + +# fetch Shebanq + +if [[ "$doAll" == "v" || "$doFetchShebanq" == "v" ]]; then + fetchShebanq +fi + +# install Shebanq + +if [[ "$doAll" == "v" || "$doShebanq" == "v" ]]; then + installShebanq "v" +fi + +# install web2py + +skipExtradirs="x" + +if [[ "$doAll" == "v" || "$doWeb2py" == "v" ]]; then + echo "o-o-o WEB2PY START o-o-o" + + # unpack Web2py + + echo "o-o-o - install" + ensureDir "$SERVER_APP_DIR" + chmod 755 /opt + chmod 755 "$SERVER_APP_DIR" + chown "$SERVER_USER":shebanq "$SERVER_APP_DIR" + + cd "$SERVER_APP_DIR" + cp "$SERVER_INSTALL_DIR/$WEB2PY_FILE" web2py.zip + if [[ -e web2py ]]; then + rm -rf web2py + fi + $TM unzip web2py.zip > /dev/null + chmod 2775 web2py + setfacl -d -m g::rw web2py + rm web2py.zip + + additionals + + writableDirs web2py v + compileApp admin v + + echo "o-o-o - Removing examples app" + rm -rf "$SERVER_APP_DIR/web2py/applications/examples" + + chown -R "$SERVER_USER":shebanq "$SERVER_APP_DIR/web2py" + chcon -R -t httpd_sys_content_t "$SERVER_APP_DIR/web2py" + + if [[ "$skipExtradirs" != "v" ]]; then + cd "$SERVER_APP_DIR/web2py/applications" + for app in welcome admin + do + writableDirs "$app" v + done + fi + + setsebool -P httpd_tmp_exec on + + # hook up SHEBANQ + + echo "o-o-o - hookup $APP" + + if [[ -e "$SERVER_APP_DIR/$APP" ]]; then + cd "$SERVER_APP_DIR/web2py/applications" + if [[ -e $APP ]]; then + rm -rf $APP + fi + ln -s "$SERVER_APP_DIR/$APP" "$APP" + chown -R "$SERVER_USER":shebanq "$APP" + + if [[ -e "$APP" ]]; then + compileApp $APP v + chown -R "$SERVER_USER":shebanq "$SERVER_APP_DIR/$APP" + chcon -R -t httpd_sys_content_t "$SERVER_APP_DIR/$APP" + fi + fi +fi + +# test controller to spot errors + +if [[ "$doAll" == "v" || "$doTestController" == "v" ]]; then + testController +fi + +# configure apache + +if [[ "$doAll" == "v" || "$doApache" == "v" ]]; then + echo "o-o-o APACHE setup o-o-o" + + if [[ -e "$APACHE_DIR/welcome.conf" ]]; then + mv "$APACHE_DIR/welcome.conf" "$APACHE_DIR/welcome.conf.disabled" + fi + if [[ "$SERVER_URL" != "$SERVER" && -e "${SERVER}.conf" ]]; then + # remove temporary conf based on the server name + # which is used before DNS has been updated to new server + rm -rf "$APACHE_DIR/${SERVER}.conf" + fi + for conf in "${SERVER_URL}.conf" wsgi.conf + do + cp "$SERVER_INSTALL_DIR/$conf" "$APACHE_DIR" + done + + service httpd restart +fi + +eraseDir "$SERVER_UNPACK_DIR" + +if [[ "$doAll" == "v" || "$doAdminPwd" == "v" ]]; then + paramFile="parameters_443.py" + paramGenerated="$SERVER_APP_DIR/web2py/$paramFile" + paramSaved="$SERVER_INSTALL_DIR/$paramFile" + + if [[ -f "$paramGenerated" ]]; then + echo "Found existing $paramFile with hash of admin password" + else + echo "No $paramFile with hash of admin password found" + if [[ "$doAdminPwd" == "x" ]]; then + echo "You can create one afterwards by running" + echo "sudo ./install.sh --adminpwd" + fi + fi + if [[ "$doAdminPwd" == "v" ]]; then + cd "$SERVER_APP_DIR/web2py" + python3 -c "from gluon.main import save_password; save_password(input('admin password: '),443)" + if [[ -f "$paramGenerated" ]]; then + chmod 775 "$paramGenerated" + chown -R $SERVER_USER:shebanq "$paramGenerated" + chcon -R -t httpd_sys_content_t "$paramGenerated" + cp "$paramGenerated" "$paramSaved" + chown $SERVER_USER:$SERVER_USER "$paramSaved" + echo "A new $paramFile file has been created and saved to $SERVER_INSTALL_DIR" + echo "Hint: on your local computer, fetch this file to your _local dir in your clone of shebanq." + echo 'cd ~/github/etcbc/shebanq/_local' + echo "scp $SERVER_USER@$SERVER:$SERVER_INSTALL_DIR/$paramFile ." + fi + fi +fi + +# first Visit to warm-up caches and to verify that the setup works + +if [[ "$doAll" == "v" || "$doFirstVisit" == "v" ]]; then + firstVisit +fi diff --git a/scripts/docker/localize.sh b/scripts/docker/localize.sh new file mode 100755 index 00000000..079524a1 --- /dev/null +++ b/scripts/docker/localize.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# copy data to _docker directory + +# read the config settings and define functions + +source ${0%/*}/configtemplate.sh +source ${0%/*}/doconfig.sh +source ${0%/*}/functions.sh + +dest="$DOCKER_DIR" +ensureDir "$dest" +for file in configtemplate.sh doconfig.sh functions.sh localize.sh provision.sh +do + cp "$SCRIPT_SRC_DIR/docker/$file" "$dest" +done + + +if [[ ! -e "$DOCKER_DIR/config.sh" ]]; then + echo " +Do not forget to: + +cd $DOCKER_DIR +mv configtemplate.sh config.sh +vim config.sh + +and adapt the contents to your situation. +" +fi diff --git a/scripts/docker/provision.sh b/scripts/docker/provision.sh new file mode 100755 index 00000000..563d7662 --- /dev/null +++ b/scripts/docker/provision.sh @@ -0,0 +1,171 @@ +#!/bin/bash + +# READ THIS FIRST: maintenance.md + +# Script to provision a server. +# Run it on your local computer. + +if [[ ! -e "${0%/*}/config.sh" ]]; then + echo "No config.sh found +Probably you are running this script from the maintenance directory. +You should run it from the _local directory where you have your +localized config.sh. + +Hint: do + +./localize.sh +cd ../../_local + " + exit +fi + +source ${0%/*}/config.sh +source ${0%/*}/doconfig.sh +source ${0%/*}/functions.sh + + +USAGE=" +Usage: ./$(basename $0) + +Provisions the docker build context with data to install it with. +" + +showUsage "$1" "$USAGE" + +setSituation + +# transfer data to build environment of the shebanq-db image + +dest="$DOCKER_DIR/shebanq-db" +ensureDir "$dest" +for file in grants.sql shebanq.cnf +do + cp "$SCRIPT_SRC_DIR/$file" "$dest" +done +for file in Dockerfile +do + cp "$SCRIPT_SRC_DIR/docker/shebanq-db/$file" "$dest" +done + +for template in mysqlrootopt user.sql +do + theTemplate="$SCRIPT_SRC_DIR/docker/templates/$template" + theFile="$DOCKER_DIR/shebanq-db/$template" + theFill="$SCRIPT_SRC_DIR/docker/fill.py" + if [[ -e "$theTemplate" ]]; then + python3 "$theFill" "$theTemplate" "$theFile" "$KEYVALUES" + else + good="x" + "Template not found: $theTemplate" + fi +done + +# transfer data to build environment of the shebanq-db image + +# - scripts, literally + +dest="$DOCKER_DIR/shebanq" +ensureDir "$dest" +for file in Dockerfile unconfigure.sql wsgi.conf +do + cp "$SCRIPT_SRC_DIR/docker/shebanq/$file" "$dest" +done +for file in routes.py wsgihandler.py +do + cp "$SCRIPT_SRC_DIR/$file" "$dest" +done + +# - scripts, as filled-in templates + +for template in apache.conf mail.cfg host.cfg mql.cfg mqlimportopt mysqldumpopt user.sql \ + staticdata.sh +do + theTemplate="$SCRIPT_SRC_DIR/docker/templates/$template" + theFile="$DOCKER_DIR/shebanq/$template" + theFill="$SCRIPT_SRC_DIR/docker/fill.py" + if [[ -e "$theTemplate" ]]; then + python3 "$theFill" "$theTemplate" "$theFile" "$KEYVALUES" + if [[ "$template" == "apache.conf" ]]; then + theConf="$DOCKER_DIR/shebanq/${SERVER_URL}.conf" + mv "$theFile" "$theConf" + fi + else + good="x" + "Template not found: $theTemplate" + fi +done + +# - dynamic data from latest backup + +latestBackup="$BACKUP_DIR/latest" + +echo "o-o-o previous dynamic data if any o-o-o" + +stampFile="$latestBackup/stamp" + +if [[ -f "$stampFile" ]]; then + stamp=`cat "$stampFile"` + echo "found backup made on $stamp" +fi +for db in $DYNAMIC_WEB $DYNAMIC_NOTE +do + dbFile="$latestBackup/${db}.sql.gz" + if [[ ! -f "$dbFile" ]]; then + echo "WARNING: no latest backup found for ($dbFile), we use an empty db" + dbFile="$SCRIPT_SRC_DIR/${db}.sql.gz" + fi + cp "$dbFile" "$DOCKER_DIR/shebanq" +done + +# - static data from local bhsa clones + +echo "o-o-o static data o-o-o" +for version in $versions +do + echo " o-o version $version ..." + src="$STATIC_SRC_DIR/$version" + staticEtcbc="${src}/${STATIC_ETCBC}$version.mql.bz2" + staticPassage="${src}/${STATIC_PASSAGE}$version.sql.gz" + for item in "$staticEtcbc" "$staticPassage" + do + cp "$item" "$DOCKER_DIR/shebanq" + done +done + +# - emdros zip + +echo "o-o-o emdros package o-o-o" +cp "$EMDROS_PATH" "$DOCKER_DIR/shebanq" + +# - web2py zip + +echo "o-o-o web2py package o-o-o" +cp "$WEB2PY_PATH" "$DOCKER_DIR/shebanq" +eraseDir "$DOCKER_DIR/shebanq/web2py" +unzip $WEB2PY_PATH -d "$DOCKER_DIR/shebanq" > /dev/null + + +for pyFile in routes.py wsgihandler.py +do + sourceFile="$DOCKER_DIR/shebanq/$pyFile" + destFile="$DOCKER_DIR/shebanq/web2py/$pyFile" + cp "$sourceFile" "$destFile" +done + +for pyFile in parameters_443.py +do + sourceFile="$DOCKER_DIR/$pyFile" + destFile="$DOCKER_DIR/shebanq/web2py/$pyFile" + cp "$sourceFile" "$destFile" +done + +# - shebanq webapp + +echo "o-o-o shebanq app o-o-o" + +eraseDir "$DOCKER_DIR/shebanq/shebanq" +ensureDir "$DOCKER_DIR/shebanq/shebanq" +for member in controllers cron models modules scripts static views +do + cp -r "$SOURCE_REPO/$member" "$DOCKER_DIR/shebanq/shebanq/$member" +done diff --git a/scripts/docker/shebanq-db/Dockerfile b/scripts/docker/shebanq-db/Dockerfile new file mode 100644 index 00000000..8e33ddc3 --- /dev/null +++ b/scripts/docker/shebanq-db/Dockerfile @@ -0,0 +1,9 @@ +FROM mysql:8 + +# docker run --detach --publish 3306:3306 --name shebanq-db -v ~/github/etcbc/shebanq/docker/shebanq-db/shebanq.cnf -e MYSQL_ALLOW_EMPTY_PASSWORD=1 -t shebanq-db +# docker run --detach --publish 3306:3306 --name shebanq-db -t shebanq-db -v ~/github/etcbc/shebanq/docker/shebanq-db/shebanq.cnf -e MYSQL_ROOT_PASSWORD=1 + +WORKDIR build +COPY user.sql grants.sql . +RUN mysql --defaults-extra-file=mysqlrootopt" < user.sql +RUN mysql --defaults-extra-file=mysqlrootopt" < grants.sql diff --git a/scripts/docker/shebanq-db/shebanq.cnf b/scripts/docker/shebanq-db/shebanq.cnf new file mode 100644 index 00000000..4fe8ab57 --- /dev/null +++ b/scripts/docker/shebanq-db/shebanq.cnf @@ -0,0 +1,11 @@ +[mysqld] +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci +innodb-buffer-pool-size=2G + +[client] +default-character-set = utf8mb4 + +[mysql] +default-character-set = utf8mb4 diff --git a/scripts/docker/shebanq/Dockerfile b/scripts/docker/shebanq/Dockerfile new file mode 100644 index 00000000..279f4245 --- /dev/null +++ b/scripts/docker/shebanq/Dockerfile @@ -0,0 +1,53 @@ +FROM ubuntu:20.04 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && \ + apt-get install -y \ + build-essential \ + python3 python3-dev python3-pip \ + libexpat1 apache2 apache2-utils ssl-cert \ + libapache2-mod-wsgi-py3 \ + libmysqlclient-dev \ + && \ + pip3 install markdown \ + && \ + ln -s /usr/bin/python3 /usr/bin/python + +ARG emdrosversion="3.7.3" +ARG emdrosdir="/opt/emdros" + +WORKDIR build +COPY emdros-${emdrosversion}.tar.gz . +RUN tar xf emdros-${emdrosversion}.tar.gz + +WORKDIR emdros-${emdrosversion} +RUN ./configure \ + --prefix=${emdrosdir} \ + --with-sqlite3=no \ + --with-mysql=yes \ + --with-swig-language-java=no \ + --with-swig-language-python2=no \ + --with-swig-language-python3=yes \ + --with-postgresql=no \ + --with-wx=no \ + --with-swig-language-csharp=no \ + --with-swig-language-php7=no \ + --with-bpt=no \ + --disable-debug && \ + make && \ + make install + +RUN apt-get update \ + && \ + apt-get install -y mysql-client + +WORKDIR .. + +ARG cfg=/opt/cfg +ARG db="shebanq_etcbc2021" +ARG datafile="${db}.mql" +COPY mail.cfg host.cfg host.cfg mql.cfg mqlimportopt mysqldumpopt ${cfg}/ +COPY filldb.sh ${datafile}.bz2 . + +ENTRYPOINT ./filldb.sh && bash diff --git a/scripts/docker/shebanq/grants.sql b/scripts/docker/shebanq/grants.sql new file mode 100644 index 00000000..186eb0d6 --- /dev/null +++ b/scripts/docker/shebanq/grants.sql @@ -0,0 +1,10 @@ +GRANT SELECT ON `shebanq\_etcbc%`.* TO 'shebanq'@'localhost'; +GRANT SELECT ON `shebanq\_passage%`.* TO 'shebanq'@'localhost'; + +GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER ON shebanq_web.* TO 'shebanq'@'localhost'; +GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER ON shebanq_note.* TO 'shebanq'@'localhost'; + +GRANT USAGE ON *.* TO 'shebanq_admin'@'localhost'; +GRANT ALL PRIVILEGES ON `shebanq%`.* TO 'shebanq_admin'@'localhost' WITH GRANT OPTION; + +FLUSH PRIVILEGES; diff --git a/scripts/docker/shebanq/unconfigure.sql b/scripts/docker/shebanq/unconfigure.sql new file mode 100644 index 00000000..8af3f8d8 --- /dev/null +++ b/scripts/docker/shebanq/unconfigure.sql @@ -0,0 +1,13 @@ +REVOKE SELECT ON `shebanq\_etcbc%`.* FROM 'shebanq'@'localhost'; +REVOKE SELECT ON `shebanq\_passage%`.* FROM 'shebanq'@'localhost'; + +REVOKE SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER ON shebanq_web.* FROM 'shebanq'@'localhost'; +REVOKE SELECT, INSERT, UPDATE, DELETE, CREATE, ALTER ON shebanq_note.* FROM 'shebanq'@'localhost'; + +REVOKE USAGE ON *.* FROM 'shebanq_admin'@'localhost'; +REVOKE ALL PRIVILEGES ON `shebanq%`.* FROM 'shebanq_admin'@'localhost'; + +FLUSH PRIVILEGES; + +DROP USER shebanq@localhost; +DROP USER shebanq_admin@localhost; diff --git a/scripts/docker/shebanq/wsgi.conf b/scripts/docker/shebanq/wsgi.conf new file mode 100644 index 00000000..00328442 --- /dev/null +++ b/scripts/docker/shebanq/wsgi.conf @@ -0,0 +1,17 @@ +# important notice +# The line with WSGIDaemonProcess should occur here +# so that all virtual hosts that need WSGI have it available. +# +# The way it is recommended by Web2py wreaks havoc with logging: +# There the line is included in the :80 host, and not in the :443 host. +# In that case, effectively all logging from python code goes to +# a data sink. +# +# SHEBANQ should run in one process with several threads, +# because there is a global, updatable index in the cache, which is +# in the RAM of the process. +# It is recommended to omit processes=1, since it is the default and +# stating processes=1 may have unwanted side effects. + +WSGISocketPrefix /var/run/wsgi +WSGIDaemonProcess web2py user=apache group=shebanq threads=5 diff --git a/scripts/docker/templates/apache.conf b/scripts/docker/templates/apache.conf new file mode 100644 index 00000000..7f835fa5 --- /dev/null +++ b/scripts/docker/templates/apache.conf @@ -0,0 +1,95 @@ +LogLevel info +ErrorLog /var/log/httpd/error_log +CustomLog /var/log/httpd/access_log common + + + ServerName «SERVER_URL» + WSGIProcessGroup web2py + WSGIScriptAlias / /opt/web-apps/web2py/wsgihandler.py + WSGIPassAuthorization On + + LogLevel warn + ErrorLog /var/log/httpd/shebanq_error + CustomLog /var/log/httpd/shebanq_access common + + + AllowOverride None + Require all denied + + Require all granted + + + + AliasMatch ^/([^/]+)/static/(?:_[\d]+.[\d]+.[\d]+/)?(.*) /opt/web-apps/web2py/applications/$1/static/$2 + + + Options -Indexes + ExpiresActive On + ExpiresDefault "access plus 1 hour" + Require all granted + + + + Deny from all + + + + Deny from all + + + + + + ServerName «SERVER_URL» + WSGIProcessGroup web2py + WSGIScriptAlias / /opt/web-apps/web2py/wsgihandler.py + WSGIPassAuthorization On + + LogLevel warn + ErrorLog /var/log/httpd/shebanq_error + CustomLog /var/log/httpd/shebanq_access common + + + AllowOverride None + Require all denied + + Require all granted + + + + AliasMatch ^/([^/]+)/static/(?:_[\d]+.[\d]+.[\d]+/)?(.*) /opt/web-apps/web2py/applications/$1/static/$2 + + + Options -Indexes + ExpiresActive On + ExpiresDefault "access plus 1 hour" + Require all granted + + + Header set Strict-Transport-Security "max-age=31536000; includeSubDomains" + + SSLEngine on + + SSLCertificateFile «CERT_FILE» + SSLCertificateKeyFile «CERT_KEY» + SSLCertificateChainFile «CERT_CHAIN» + + SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown downgrade-1.0 force-response-1.0 + + + SSLOptions +StdEnvVars + + + SSLOptions +StdEnvVars + + + BrowserMatch "MSIE [2-5]" \ + nokeepalive ssl-unclean-shutdown \ + downgrade-1.0 force-response-1.0 + + CustomLog logs/shebanq_request \ + "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + + + + diff --git a/scripts/docker/templates/host.cfg b/scripts/docker/templates/host.cfg new file mode 100644 index 00000000..107fd43c --- /dev/null +++ b/scripts/docker/templates/host.cfg @@ -0,0 +1 @@ +«DB_HOST» diff --git a/scripts/docker/templates/mail.cfg b/scripts/docker/templates/mail.cfg new file mode 100644 index 00000000..26efc71b --- /dev/null +++ b/scripts/docker/templates/mail.cfg @@ -0,0 +1,2 @@ +server = «MAIL_SERVER» +sender = «MAIL_SENDER» diff --git a/scripts/docker/templates/mql.cfg b/scripts/docker/templates/mql.cfg new file mode 100644 index 00000000..60584cc5 --- /dev/null +++ b/scripts/docker/templates/mql.cfg @@ -0,0 +1 @@ +«MYSQL_SHEBANQ» diff --git a/scripts/docker/templates/mqlimportopt b/scripts/docker/templates/mqlimportopt new file mode 100644 index 00000000..c431aff4 --- /dev/null +++ b/scripts/docker/templates/mqlimportopt @@ -0,0 +1 @@ +«MYSQL_SHEBANQ_ADMIN» diff --git a/scripts/docker/templates/mysqldumpopt b/scripts/docker/templates/mysqldumpopt new file mode 100644 index 00000000..ee641d4e --- /dev/null +++ b/scripts/docker/templates/mysqldumpopt @@ -0,0 +1,3 @@ +[mysql] +password = '«MYSQL_ROOT»' +user = root diff --git a/scripts/docker/templates/mysqlrootopt b/scripts/docker/templates/mysqlrootopt new file mode 100644 index 00000000..19a982f8 --- /dev/null +++ b/scripts/docker/templates/mysqlrootopt @@ -0,0 +1,9 @@ +[mysql] +password = '«MYSQL_SHEBANQ_ADMIN»' +user = shebanq_admin +host = «DB_HOST» + +[mysqldump] +password = '«MYSQL_SHEBANQ_ADMIN»' +user = shebanq_admin +host = «DB_HOST» diff --git a/scripts/docker/templates/staticdata.sh b/scripts/docker/templates/staticdata.sh new file mode 100755 index 00000000..8a3fbaab --- /dev/null +++ b/scripts/docker/templates/staticdata.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +mysql -h«DB_HOST» -uroot < «SERVER_CFG_DIR»/user.sql +mysql -h«DB_HOST» -uroot < «SERVER_CFG_DIR»/grants.sql + +mysqlOpt="--defaults-extra-file=«SERVER_CFG_DIR»/mysqldumpopt" + +for version in «STATIC_VERSIONS» +do + echo "o-o-o - VERSION $version «STATIC_PASSAGE»" + db="«STATIC_PASSAGE»$version" + datafile="${db}.sql" + echo "o-o-o - unzipping $db" + gunzip -f "${db}.gz" + echo "o-o-o - loading $db (may take half a minute)" + mysql ${mysqlOpt} < ${datafile} + rm ${datafile} + + echo "o-o-o - VERSION $version «STATIC_ETCBC»" + db="«STATIC_ETCBC»$version" + datafile="${db}.mql" + echo "o-o-o - unzipping $db" + bunzip2 -f ${datafile}.bz2 + mysql ${mysqlOpt} -h«DB_HOST» -e "drop database if exists ${db};" + «SERVER_MQL_DIR»/mql -e UTF8 -n -b m -h «DB_HOST» -u shebanq_admin -p uvw456 < ${datafile} + rm ${datafile} +done diff --git a/scripts/docker/templates/user.sql b/scripts/docker/templates/user.sql new file mode 100644 index 00000000..1fe02c42 --- /dev/null +++ b/scripts/docker/templates/user.sql @@ -0,0 +1,4 @@ +DROP USER IF EXISTS 'shebanq'@'%'; +CREATE USER 'shebanq'@'%' IDENTIFIED BY '«MYSQL_SHEBANQ»'; +DROP USER IF EXISTS 'shebanq_admin'@'%'; +CREATE USER 'shebanq_admin'@'%' IDENTIFIED BY '«MYSQL_SHEBANQ_ADMIN»';