diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index dcf98ac..03f0b8d 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -1,13 +1,14 @@ -name: Build and Push emoncms legacy Docker Image +name: Build and Push emoncms Docker Image on: + pull_request: workflow_dispatch: inputs: php_version: description: 'php_version' required: true type: string - default: '8.2.27' + default: '8.4' emoncms_src: description: 'emoncms_src' required: true @@ -19,38 +20,72 @@ on: type: string default: 'stable' +env: + PHP_VERSION: ${{ inputs.php_version || '8.4' }} + EMONCMS_SRC: ${{ inputs.emoncms_src || 'emoncms/emoncms' }} + EMONCMS_BRANCH: ${{ inputs.branch || 'stable' }} + jobs: build-and-push: runs-on: ubuntu-latest + permissions: + packages: write steps: - # Checkout the repository code - name: Checkout code uses: actions/checkout@v4 - # Log in to Docker Hub + - name: Check for Docker Hub credentials + id: check_dockerhub + run: | + if [ -n "${{ secrets.DOCKER_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_PASSWORD }}" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + - name: Log in to Docker Hub + if: steps.check_dockerhub.outputs.available == 'true' && github.event_name != 'pull_request' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - id: emoncms_version - name: get emoncms version + + - name: Log in to GitHub Container Registry + if: steps.check_dockerhub.outputs.available != 'true' || github.event_name == 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get emoncms version + id: emoncms_version run: | - wget https://raw.githubusercontent.com/${{ inputs.emoncms_src }}/${{ inputs.branch }}/version.json + wget https://raw.githubusercontent.com/${{ env.EMONCMS_SRC }}/${{ env.EMONCMS_BRANCH }}/version.json version=$(cat version.json | jq --raw-output '.version') echo $version echo "version=$version" >> "$GITHUB_OUTPUT" - # Build and push Docker image + - name: Set image tags + id: tags + run: | + VERSION="${{ steps.emoncms_version.outputs.version }}" + if [ "${{ github.event_name }}" = "pull_request" ]; then + TAGS="ghcr.io/${{ github.repository_owner }}/emoncms:pr-${{ github.event.number }}" + elif [ "${{ steps.check_dockerhub.outputs.available }}" = "true" ]; then + TAGS="openenergymonitor/emoncms:latest,openenergymonitor/emoncms:${VERSION}" + else + TAGS="ghcr.io/${{ github.repository_owner }}/emoncms:latest,ghcr.io/${{ github.repository_owner }}/emoncms:${VERSION}" + fi + echo "tags=$TAGS" >> "$GITHUB_OUTPUT" + - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: ./web build-args: | - "BUILD_FROM=php:${{ inputs.php_version }}-apache" - "EMONCMS_SRC=https://github.com/${{ inputs.emoncms_src }}" - "BRANCH=${{ inputs.branch }}" + "BUILD_FROM=php:${{ env.PHP_VERSION }}-apache" + "EMONCMS_SRC=https://github.com/${{ env.EMONCMS_SRC }}" + "BRANCH=${{ env.EMONCMS_BRANCH }}" push: true - tags: | - openenergymonitor/emoncms:latest - openenergymonitor/emoncms:${{ steps.emoncms_version.outputs.version }} + tags: ${{ steps.tags.outputs.tags }} diff --git a/default.docker-env b/default.docker-env index 0b4845b..129285d 100644 --- a/default.docker-env +++ b/default.docker-env @@ -1,6 +1,11 @@ # Default docker enviroment variables, change in production enviroment -# MySQL database +# General +TZ=Europe/London +EMONCMS_DATADIR=/var/opt/emoncms +EMONCMS_LOG_LEVEL=2 + +# MySQL database MYSQL_HOST=db MYSQL_PORT=3306 MYSQL_DATABASE=emoncms @@ -13,11 +18,10 @@ MYSQL_INITDB_SKIP_TZINFO=true REDIS_ENABLED=true REDIS_HOST=redis REDIS_PORT=6379 -# At the moment Docker doesn't honour the REDIS_AUTH variable, so it will always be passwordless -# REDIS_AUTH= REDIS_PREFIX='emoncms' +REDIS_BUFFER=1 -# MQTT (needs to be running elsewhere) +# MQTT MQTT_ENABLED=true MQTT_HOST=mqtt MQTT_CLIENTID=emoncmsmqtt diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 52b6e5f..92e8b23 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,6 +1,5 @@ # dev enviroment setup -version: '2' services: web: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b300623..7a24117 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,6 +1,5 @@ # production / deployment / clean testing enviroment setup -version: '2' services: web: diff --git a/docker-compose.yml b/docker-compose.yml index af2d232..4e98174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,20 @@ # Base docker compose, added to by docker-compose.override or docker-compose.prod. See Readme.md -version: '2' - services: - # PHP & apache container using offical Docker PHP iamge + # PHP & apache container with s6-overlay managing web server and background workers web: - # If pre-built image from docker hub exists then use that (docker pull openenergymonitor/emoncms:latest) if not build container see web/Dockerflile image: openenergymonitor/emoncms:latest build: web/. volumes: - # mount docker volumes persistant inside docker container - emon-phpfina:/var/opt/emoncms/phpfina - emon-phptimeseries:/var/opt/emoncms/phptimeseries links: - db - redis - mqtt + depends_on: + redis: + condition: service_healthy db: image: mariadb:11.0 @@ -34,14 +33,30 @@ services: image: redis:7.0 volumes: - emon-redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 5s logging: driver: json-file options: max-size: "10m" - + + # No local image build: extra layers on eclipse-mosquitto can hit BuildKit "max depth exceeded" + # on some Docker Desktop setups. Config + passwd are applied at container start (see mqtt/setup.sh). mqtt: image: eclipse-mosquitto:2.0 - build: mqtt/. + volumes: + - ./mqtt/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro + command: + - /bin/sh + - -c + - | + touch /mosquitto/config/passwd + /usr/bin/mosquitto_passwd -b /mosquitto/config/passwd emonpi emonpimqtt2016 + exec /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf restart: always logging: driver: json-file @@ -49,8 +64,6 @@ services: max-size: "10m" volumes: - emon-phpfiwa: - driver: local emon-phpfina: driver: local emon-phptimeseries: diff --git a/mqtt/Dockerfile b/mqtt/Dockerfile deleted file mode 100644 index 2d114aa..0000000 --- a/mqtt/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM eclipse-mosquitto:2.0 -COPY mosquitto.conf /mosquitto/config/mosquitto.conf -COPY setup.sh / -RUN chmod +x /setup.sh -RUN /setup.sh diff --git a/mqtt/setup.sh b/mqtt/setup.sh deleted file mode 100644 index e6196ca..0000000 --- a/mqtt/setup.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Create mosquitto password file -touch /mosquitto/config/passwd -mosquitto_passwd -b /mosquitto/config/passwd emonpi emonpimqtt2016 diff --git a/web/Dockerfile b/web/Dockerfile index b4d7d97..54c2b2f 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,9 +1,15 @@ -ARG BUILD_FROM=php:8.0-apache +ARG BUILD_FROM=php:8.4-apache -# Offical Docker PHP & Apache image https://hub.docker.com/_/php/ +# Official Docker PHP & Apache image https://hub.docker.com/_/php/ FROM $BUILD_FROM ARG \ + TARGETPLATFORM \ + S6_OVERLAY_VERSION=3.1.6.2 \ + S6_SRC=https://github.com/just-containers/s6-overlay/releases/download \ + S6_DIR=/etc/s6-overlay/s6-rc.d \ + PRIMOS="apache2" \ + SECONDOS="emoncms_mqtt service-runner feedwriter sync_upload" \ EMONCMS_SRC=https://github.com/emoncms/emoncms \ BRANCH=master \ DAEMON=www-data \ @@ -12,30 +18,39 @@ ARG \ EMONCMS_DIR=/opt/emoncms \ EMONCMS_LOG_LOCATION=/var/log/emoncms \ EMONCMS_DATADIR=/var/opt/emoncms - - -# Install deps -RUN apt-get update && apt-get install -y \ - iproute2 \ - libcurl4-gnutls-dev \ - libmcrypt-dev \ - gettext \ - nano \ - git-core \ - supervisor + +ENV \ + DAEMON=www-data \ + WWW=/var/www \ + OEM_DIR=/opt/openenergymonitor \ + EMONCMS_DIR=/opt/emoncms \ + EMONCMS_LOG_LOCATION=/var/log/emoncms + +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + iproute2 \ + libcurl4-gnutls-dev \ + libmosquitto-dev \ + gettext \ + nano \ + git-core \ + xz-utils \ + python3 \ + python3-redis \ + default-mysql-client \ + curl \ + && rm -rf /var/lib/apt/lists/* # Enable PHP modules RUN docker-php-ext-install -j$(nproc) mysqli gettext -# Install redis +# Install phpredis extension COPY install_redis.sh / -RUN chmod +x /install_redis.sh -RUN /install_redis.sh +RUN chmod +x /install_redis.sh && /install_redis.sh -# Install mosquitto +# Install Mosquitto-PHP extension COPY install_mosquitto.sh / -RUN chmod +x /install_mosquitto.sh -RUN /install_mosquitto.sh +RUN chmod +x /install_mosquitto.sh && /install_mosquitto.sh RUN a2enmod rewrite @@ -44,46 +59,139 @@ COPY config/php.ini /usr/local/etc/php/ # Add custom Apache config COPY config/emoncms.conf /etc/apache2/sites-available/emoncms.conf -RUN a2dissite 000-default.conf -RUN a2ensite emoncms - -# Clone in master Emoncms repo & modules - overwritten in development with local FS files -RUN \ - set -x;\ +RUN a2dissite 000-default.conf && a2ensite emoncms + +# Install s6-overlay +RUN set -x;\ + case ${TARGETPLATFORM} in \ + "linux/amd64") S6_ARCH="x86_64" ;; \ + "linux/arm/v7") S6_ARCH="arm" ;; \ + "linux/arm64") S6_ARCH="aarch64" ;; \ + *) S6_ARCH="x86_64" ;; \ + esac;\ + curl -fsSL -o /tmp/s6-overlay-${S6_ARCH}.tar.xz \ + ${S6_SRC}/v${S6_OVERLAY_VERSION}/s6-overlay-${S6_ARCH}.tar.xz;\ + curl -fsSL -o /tmp/s6-overlay-noarch.tar.xz \ + ${S6_SRC}/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz;\ + tar -C / -Jxpf /tmp/s6-overlay-${S6_ARCH}.tar.xz;\ + tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz;\ + rm -f /tmp/s6-overlay-*.tar.xz + +# Clone emoncms core and modules +RUN set -x;\ mkdir -p $OEM_DIR;\ + mkdir -p $EMONCMS_DIR/modules;\ cd $WWW && git clone -b $BRANCH $EMONCMS_SRC;\ cd $WWW/emoncms/Modules && git clone https://github.com/emoncms/dashboard;\ cd $WWW/emoncms/Modules && git clone https://github.com/emoncms/graph;\ cd $WWW/emoncms/Modules && git clone https://github.com/emoncms/app;\ cd $WWW/emoncms/Modules && git clone https://github.com/emoncms/device;\ - chown -R $DAEMON $WWW/emoncms + cd $EMONCMS_DIR/modules && git clone https://github.com/emoncms/sync;\ + cd $EMONCMS_DIR/modules && git clone https://github.com/emoncms/postprocess;\ + cd $EMONCMS_DIR/modules && git clone https://github.com/emoncms/backup;\ + ln -s $EMONCMS_DIR/modules/sync $WWW/emoncms/Modules/sync;\ + ln -s $EMONCMS_DIR/modules/postprocess $WWW/emoncms/Modules/postprocess;\ + ln -s $EMONCMS_DIR/modules/backup $WWW/emoncms/Modules/backup;\ + chown -R $DAEMON $WWW/emoncms;\ + chown -R $DAEMON $EMONCMS_DIR WORKDIR $OEM_DIR -RUN \ - set -x;\ +RUN set -x;\ git config --system --replace-all safe.directory '*';\ git clone https://github.com/openenergymonitor/EmonScripts;\ cp EmonScripts/install/emonsd.config.ini EmonScripts/install/config.ini COPY docker.settings.ini /var/www/emoncms/settings.ini - -# Create folders & set permissions for feed-engine data folders (mounted as docker volumes in docker-compose) -RUN \ - set -x;\ - mkdir $EMONCMS_DATADIR;\ - mkdir $EMONCMS_DATADIR/phpfina;\ - mkdir $EMONCMS_DATADIR/phptimeseries;\ - chown -R $DAEMON $EMONCMS_DATADIR - -# Create Emoncms logfile -RUN \ - set -x;\ +# Upstream service-runner uses redis.Redis() (localhost); Docker needs REDIS_HOST (see service-runner.py). +COPY service-runner.py $WWW/emoncms/scripts/services/service-runner/service-runner.py +RUN chown $DAEMON $WWW/emoncms/scripts/services/service-runner/service-runner.py + +# Create folders & set permissions for feed-engine data, logs, and worker lock files +RUN set -x;\ + mkdir -p $EMONCMS_DATADIR/phpfina;\ + mkdir -p $EMONCMS_DATADIR/phptimeseries;\ + chown -R $DAEMON $EMONCMS_DATADIR;\ mkdir -p $EMONCMS_LOG_LOCATION;\ chown $DAEMON $EMONCMS_LOG_LOCATION;\ touch $EMONCMS_LOG_LOCATION/emoncms.log;\ - chmod 666 $EMONCMS_LOG_LOCATION/emoncms.log - -# To start Apache and emoncms_mqtt from supervisord -COPY config/supervisord.conf /etc/supervisor/supervisord.conf -ENTRYPOINT [ "/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf" ] + chmod 666 $EMONCMS_LOG_LOCATION/emoncms.log;\ + chown $DAEMON /var/lock;\ + chown $DAEMON /tmp + +# Generate emoncmsdbupdate.php for schema initialisation +RUN set -x;\ + echo " $OEM_DIR/emoncmsdbupdate.php;\ + echo "\$applychanges = true;" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "define('EMONCMS_EXEC', 1);" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "chdir('$WWW/emoncms');" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "require 'process_settings.php';" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "require 'core.php';" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "\$mysqli = @new mysqli(" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo " \$settings['sql']['server']," >> $OEM_DIR/emoncmsdbupdate.php;\ + echo " \$settings['sql']['username']," >> $OEM_DIR/emoncmsdbupdate.php;\ + echo " \$settings['sql']['password']," >> $OEM_DIR/emoncmsdbupdate.php;\ + echo " \$settings['sql']['database']," >> $OEM_DIR/emoncmsdbupdate.php;\ + echo " \$settings['sql']['port']" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo ");" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "require_once 'Lib/dbschemasetup.php';" >> $OEM_DIR/emoncmsdbupdate.php;\ + echo "print json_encode(db_schema_setup(\$mysqli,load_db_schema(),\$applychanges)).'\n';" >> $OEM_DIR/emoncmsdbupdate.php + +COPY emoncms_pre.sh sql_ready.sh sync_upload-run.sh $OEM_DIR/ + +# s6-overlay: user bundle (primary services) +# sql_ready must run before Apache serves traffic (otherwise / hits user_controller with no users table). +# It was only registered in user2; in s6-overlay v3 the default target may not activate user2, so +# schema init never ran. sql_ready lives in user with emoncms_pre -> sql_ready -> apache2. +RUN set -x;\ + mkdir $S6_DIR/emoncms_pre;\ + mkdir $S6_DIR/emoncms_pre/dependencies.d;\ + touch $S6_DIR/emoncms_pre/dependencies.d/base;\ + echo "oneshot" > $S6_DIR/emoncms_pre/type;\ + echo "$OEM_DIR/emoncms_pre.sh" > $S6_DIR/emoncms_pre/up;\ + mkdir $S6_DIR/sql_ready;\ + mkdir $S6_DIR/sql_ready/dependencies.d;\ + touch $S6_DIR/sql_ready/dependencies.d/emoncms_pre;\ + echo "oneshot" > $S6_DIR/sql_ready/type;\ + echo "$OEM_DIR/sql_ready.sh" > $S6_DIR/sql_ready/up;\ + touch $S6_DIR/user/contents.d/emoncms_pre;\ + touch $S6_DIR/user/contents.d/sql_ready;\ + for i in $PRIMOS; do mkdir $S6_DIR/$i; done;\ + for i in $PRIMOS; do mkdir $S6_DIR/$i/dependencies.d; done;\ + for i in $PRIMOS; do touch $S6_DIR/$i/dependencies.d/sql_ready; done;\ + for i in $PRIMOS; do touch $S6_DIR/user/contents.d/$i; done;\ + for i in $PRIMOS; do echo "longrun" > $S6_DIR/$i/type; done;\ + echo "#!/command/with-contenv sh" > $S6_DIR/apache2/run;\ + echo "rm -f /var/run/apache2/apache2.pid" >> $S6_DIR/apache2/run;\ + echo "exec apache2-foreground" >> $S6_DIR/apache2/run + +# s6-overlay: user2 bundle (workers) +RUN set -x;\ + for i in $SECONDOS; do mkdir $S6_DIR/$i; done;\ + for i in $SECONDOS; do mkdir $S6_DIR/$i/dependencies.d; done;\ + for i in $SECONDOS; do touch $S6_DIR/$i/dependencies.d/sql_ready; done;\ + for i in $SECONDOS; do touch $S6_DIR/user2/contents.d/$i; done;\ + for i in $SECONDOS; do echo "longrun" > $S6_DIR/$i/type; done;\ + echo "#!/command/with-contenv sh" > $S6_DIR/emoncms_mqtt/run;\ + echo "exec s6-setuidgid $DAEMON php $WWW/emoncms/scripts/services/emoncms_mqtt/emoncms_mqtt.php" >> $S6_DIR/emoncms_mqtt/run;\ + echo "#!/command/with-contenv sh" > $S6_DIR/service-runner/run;\ + echo "exec s6-setuidgid $DAEMON python3 $WWW/emoncms/scripts/services/service-runner/service-runner.py" >> $S6_DIR/service-runner/run;\ + echo "#!/command/with-contenv sh" > $S6_DIR/feedwriter/run;\ + echo "if [ \"\${REDIS_BUFFER}\" -ne 1 ]; then" >> $S6_DIR/feedwriter/run;\ + echo " echo \"FEEDWRITER IS MUTED\"" >> $S6_DIR/feedwriter/run;\ + echo " s6-svc -O ." >> $S6_DIR/feedwriter/run;\ + echo " exit 0" >> $S6_DIR/feedwriter/run;\ + echo "fi" >> $S6_DIR/feedwriter/run;\ + echo "exec s6-setuidgid $DAEMON php $WWW/emoncms/scripts/feedwriter.php" >> $S6_DIR/feedwriter/run;\ + cp $OEM_DIR/sync_upload-run.sh $S6_DIR/sync_upload/run;\ + chmod +x $S6_DIR/sync_upload/run;\ + chmod +x $OEM_DIR/emoncms_pre.sh;\ + chmod +x $OEM_DIR/sql_ready.sh + +ENV \ + S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 \ + S6_SERVICES_GRACETIME=18000 + +EXPOSE 80 + +ENTRYPOINT ["/init"] diff --git a/web/README.md b/web/README.md index bc8f400..485f724 100644 --- a/web/README.md +++ b/web/README.md @@ -3,3 +3,9 @@ ``` docker build --build-arg="BUILD_FROM=php:8.2.27-apache" -t emoncms_legacy_docker . ``` + +## Database schema on first boot + +`sql_ready.sh` waits for MySQL, then runs `emoncmsdbupdate.php` (`db_schema_setup`) when the `users` table is missing. + +That oneshot must finish **before** Apache accepts HTTP traffic. It is registered in the **user** s6 bundle with `emoncms_pre → sql_ready → apache2`. (Previously `sql_ready` lived only under **user2**; with s6-overlay v3 the default target may not activate **user2**, so schema init never ran while Apache was already up.) diff --git a/web/config/supervisord.conf b/web/config/supervisord.conf deleted file mode 100644 index fab9785..0000000 --- a/web/config/supervisord.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file=/tmp/supervisor.sock - -[supervisord] -logfile=/var/log/supervisord.log -logfile_backups=0 -; log level; default info; others: debug,warn,trace -loglevel=info -pidfile=/tmp/supervisord.pid -nodaemon=true -user=root - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket - -[program:apache] -command=apache2-foreground -priority=998 -redirect_stderr=true - -[program:emoncms_mqtt] -command=php /var/www/emoncms/scripts/services/emoncms_mqtt/emoncms_mqtt.php -directory=/var/www/emoncms/scripts/services/emoncms_mqtt -priority=999 -redirect_stderr=true -user=www-data diff --git a/web/emoncms_pre.sh b/web/emoncms_pre.sh new file mode 100644 index 0000000..e17bda0 --- /dev/null +++ b/web/emoncms_pre.sh @@ -0,0 +1,20 @@ +#!/command/with-contenv sh + +if [ -n "$TZ" ] && [ -f "/usr/share/zoneinfo/$TZ" ]; then + cp "/usr/share/zoneinfo/$TZ" /etc/localtime + echo "$TZ" > /etc/timezone +fi + +if ! [ -d "$EMONCMS_DATADIR/phpfina" ]; then + echo "Creating timeseries directories" + mkdir -p "$EMONCMS_DATADIR/phpfina" + mkdir -p "$EMONCMS_DATADIR/phptimeseries" + mkdir -p "$EMONCMS_DATADIR/backup" + mkdir -p "$EMONCMS_DATADIR/backup/uploads" +else + echo "Using existing timeseries directories" +fi + +chown -R "$DAEMON" "$EMONCMS_DATADIR" + +echo "emoncms_pre init complete" diff --git a/web/install_mosquitto.sh b/web/install_mosquitto.sh index 3f43098..92f0096 100644 --- a/web/install_mosquitto.sh +++ b/web/install_mosquitto.sh @@ -1,8 +1,8 @@ #!/bin/bash -# Install Mosquitto +# Build and install Mosquitto-PHP extension +# Requires libmosquitto-dev to be pre-installed cd / -apt-get install -y libmosquitto-dev git clone https://github.com/openenergymonitor/Mosquitto-PHP cd Mosquitto-PHP/ phpize diff --git a/web/service-runner.py b/web/service-runner.py new file mode 100644 index 0000000..eae0865 --- /dev/null +++ b/web/service-runner.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +## Used to run arbitrary commands from the EmonCMS web interface +# EmonCMS submits commands to redis where this service picks them up +# Used in conjunction with: +# - Admin module to run service-runner-update.sh +# - Backup module +# - Others?? + +# Patched in emoncms-docker: default redis.Redis() uses localhost; Docker Compose uses hostname `redis`. + +import os +import subprocess +import time +import shlex +import redis + +KEYS = ["service-runner", "emoncms:service-runner"] + + +def _redis_client(): + return redis.Redis( + host=os.environ.get("REDIS_HOST", "127.0.0.1"), + port=int(os.environ.get("REDIS_PORT", "6379")), + ) + + +def connect_redis(): + while True: + try: + server = _redis_client() + if server.ping(): + print("Connected to redis server", flush=True) + return server + except redis.exceptions.ConnectionError: + print("Unable to connect to redis server, sleeping for 30s", flush=True) + time.sleep(30) + + +def main(): + print("Starting service-runner", flush=True) + server = connect_redis() + while True: + try: + # Get the next item from the 'service-runner' list, blocking until one exists + packed = server.blpop(KEYS) + if not packed: + continue + flag = packed[1].decode() + except redis.exceptions.ConnectionError: + print("Connection to redis server lost, attempting to reconnect", flush=True) + server = connect_redis() + continue + + print("Got flag:", flag, flush=True) + if ">" in flag: + script, logfile = flag.split(">") + print("STARTING:", script, '&>', logfile, flush=True) + # Got a cmdline, now run it. + with open(logfile, "w") as f: + try: + subprocess.call(shlex.split(script), stdout=f, stderr=f) + except Exception as exc: + # If an error occurs running the subprocess, add the error to + # the specified logfile + f.write("Error running [%s]" % script) + f.write("Exception occurred: %s" % exc) + continue + else: + script = flag + print("STARTING:", script, flush=True) + try: + subprocess.call(shlex.split(script), stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + except Exception as exc: + continue + + print("COMPLETE:", script, flush=True) + + +if __name__ == "__main__": + main() diff --git a/web/sql_ready.sh b/web/sql_ready.sh new file mode 100755 index 0000000..5ee8d9a --- /dev/null +++ b/web/sql_ready.sh @@ -0,0 +1,38 @@ +#!/command/with-contenv sh + +# Wait until MariaDB accepts TCP connections with our app credentials. +# mysqladmin ping can fail silently if mysqladmin is missing (stderr discarded) or misbehaves +# with remote hosts without --protocol=TCP; use mysql client like the checks below. +MYSQL_PORT="${MYSQL_PORT:-3306}" +# Client may default to TLS; DB container has no TLS. Use --skip-ssl (MariaDB client; --ssl-mode is not always available). +MYSQL_SSL="--skip-ssl" + +echo "Waiting for MySQL at $MYSQL_HOST:$MYSQL_PORT (user $MYSQL_USER, db $MYSQL_DATABASE)..." +RETRIES=0 +while ! mysql $MYSQL_SSL -h "$MYSQL_HOST" -P "$MYSQL_PORT" --protocol=TCP -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "USE \`$MYSQL_DATABASE\`" 2>/dev/null; do + RETRIES=$((RETRIES + 1)) + if [ "$RETRIES" -ge 30 ]; then + echo "ERROR: Could not authenticate as $MYSQL_USER after 30 attempts." + echo "If this is a fresh install, ensure MYSQL_USER, MYSQL_PASSWORD and" + echo "MYSQL_DATABASE are set in the db service environment." + echo "If reusing an old volume, try: docker compose down -v" + echo "Debug (this attempt only):" + echo "MYSQL_HOST=$MYSQL_HOST MYSQL_PORT=$MYSQL_PORT MYSQL_USER=$MYSQL_USER MYSQL_DATABASE=$MYSQL_DATABASE" + mysql $MYSQL_SSL -h "$MYSQL_HOST" -P "$MYSQL_PORT" --protocol=TCP -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "USE \`$MYSQL_DATABASE\`" 2>&1 || true + exit 1 + fi + sleep 1 +done +echo "MySQL server is up and credentials OK" + +TABLE_EXISTS=$(mysql $MYSQL_SSL -h "$MYSQL_HOST" -P "$MYSQL_PORT" --protocol=TCP -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DATABASE" \ + -sse "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='$MYSQL_DATABASE' AND table_name='users'" 2>/dev/null) + +if [ "$TABLE_EXISTS" = "0" ] || [ -z "$TABLE_EXISTS" ]; then + echo "New install detected - initialising emoncms database schema" + php "$OEM_DIR/emoncmsdbupdate.php" +else + echo "Existing emoncms database found" +fi + +echo "sql_ready complete, starting workers" diff --git a/web/sync_upload-run.sh b/web/sync_upload-run.sh new file mode 100755 index 0000000..1c0a8c0 --- /dev/null +++ b/web/sync_upload-run.sh @@ -0,0 +1,20 @@ +#!/command/with-contenv sh +# sync_upload.php expects cwd under $WWW/emoncms after Lib/load_emoncms.php runs; running +# php $EMONCMS_DIR/modules/sync/sync_upload.php breaks includes unless Modules/sync exists here. +WWW="${WWW:-/var/www}" +EMONCMS_DIR="${EMONCMS_DIR:-/opt/emoncms}" +DAEMON="${DAEMON:-www-data}" +MODULES_DIR="$WWW/emoncms/Modules" + +if [ ! -e "$MODULES_DIR/sync" ] && [ -f "$EMONCMS_DIR/modules/sync/sync_upload.php" ]; then + echo "sync_upload: linking $MODULES_DIR/sync -> $EMONCMS_DIR/modules/sync" + ln -snf "$EMONCMS_DIR/modules/sync" "$MODULES_DIR/sync" +fi + +if [ ! -f "$MODULES_DIR/sync/sync_model.php" ]; then + echo "sync_upload: sync module not installed; stopping service" + s6-svc -O . + exit 0 +fi + +cd "$WWW/emoncms" && exec s6-setuidgid "$DAEMON" php Modules/sync/sync_upload.php all bg