diff --git a/.gitignore b/.gitignore index e1d076434d15b8cd4d216782ecce38209cb91b08..d5c2afb87cac2876e24ddac0075c18aaf1676648 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -/nbproject/ /packages/ /public_html/inc_config.php +/public_html/packages/used_hashes.txt /shellscripts/getfile.sh.cfg +/static/* +/tests/hello.txt diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000000000000000000000000000000000000..95e9205208b0087ccac0d3df8dea85ea610831c2 --- /dev/null +++ b/docker/.env @@ -0,0 +1,16 @@ +# ====================================================================== +# +# GENERATED BY init.sh - template: ./templates/dot_env - e2cde05722688ff85d3a93e9cd55787e +# values to be used in docker-composer.yml +# +# ====================================================================== + +# ----- application +APP_NAME=ci-pkg + +# uid of www-data in the docker container +DOCKER_USER_UID=33 + +APP_PORT=8001 +WEBROOT=/var/www/ci-pkg/public_html + diff --git a/docker/containers/db-server/mariadb/my.cnf b/docker/containers/db-server/mariadb/my.cnf new file mode 100644 index 0000000000000000000000000000000000000000..cc3b80d295df31125a4e5c600a7301e2d44b9d2f --- /dev/null +++ b/docker/containers/db-server/mariadb/my.cnf @@ -0,0 +1,3 @@ +[mysqld] +; collation-server = utf8mb4_unicode_ci +; character-set-server = utf8mb4 \ No newline at end of file diff --git a/docker/containers/web-server/Dockerfile b/docker/containers/web-server/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0c170625f65e6eb10181b31ff7ac139e951cbb8f --- /dev/null +++ b/docker/containers/web-server/Dockerfile @@ -0,0 +1,14 @@ +# +# GENERATED BY init.sh - template: ./templates/web-server-Dockerfile - 42dce773c83597a7d05af398bdd66d15 +# +FROM php:8.2-apache + +# install packages +RUN apt-get update && apt-get install -y git unzip zip libapache2-mod-xsendfile + +# enable apache modules +RUN a2enmod xsendfile + +# install php packages +COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/ +RUN install-php-extensions diff --git a/docker/containers/web-server/apache/sites-enabled/vhost_app.conf b/docker/containers/web-server/apache/sites-enabled/vhost_app.conf new file mode 100644 index 0000000000000000000000000000000000000000..6d6e7fb93c99508aa2db13b821f858632966c90f --- /dev/null +++ b/docker/containers/web-server/apache/sites-enabled/vhost_app.conf @@ -0,0 +1,33 @@ +# +# GENERATED BY init.sh - template: ./templates/vhost_app.conf - 4dfd63417ad808a5ed00ffaf117464a8 +# +<VirtualHost *:80> + DocumentRoot /var/www/ci-pkg/public_html + <Directory /var/www/ci-pkg/public_html> + AllowOverride None + Order Allow,Deny + Allow from All + </Directory> + + # redirect requests to handle packages + <Location "/packages"> + + # for Php as php-fpm service: + # SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 + + RewriteEngine on + RewriteCond %{REQUEST_FILENAME} !-f + RewriteRule ^(.*)$ index.php [QSA,L] + + </Location> + + # download files are outside webroot + XSendFile On + XSendFilePath "/var/www/ci-pkg/example-packages/" + + # example to prevent access with http + <Location "/no-access"> + Require all denied + </Location> + +</VirtualHost> \ No newline at end of file diff --git a/docker/containers/web-server/php/extra-php-config.ini b/docker/containers/web-server/php/extra-php-config.ini new file mode 100644 index 0000000000000000000000000000000000000000..aa13bd779afa40bbfa25f10adef9baeae6d14f7d --- /dev/null +++ b/docker/containers/web-server/php/extra-php-config.ini @@ -0,0 +1,24 @@ +; +; GENERATED BY init.sh - template: ./templates/extra-php-config.ini - 9dce36d285d5b21d70e015c074c196c2 +; +[PHP] + +error_reporting=E_ALL +display_errors=1 + +; ---------------------------------------------------------------------- +; XDEBUG STUFF BELOW +; ---------------------------------------------------------------------- +; +; error_reporting=E_ALL +; +; [xdebug] +; xdebug.mode=develop,debug +; ; xdebug.client_host=localhost +; xdebug.start_with_request=yes +; ; xdebug.start_with_request=trigger +; +; xdebug.log=/tmp/xdebug.log +; xdebug.discover_client_host = 1 +; ; xdebug.client_port=9003 +; xdebug.idekey="netbeans-xdebug" \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..39e932e040f09a3d0ac65254cad4b6cf6c275885 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,44 @@ +# +# GENERATED BY init.sh - template: ./templates/docker-compose.yml - fc2f1d55926abdb9c54f65afd0571d7b +# +# ====================================================================== +# +# (1) see .env for set variables +# (2) run "docker-compose up" to startup +# +# ====================================================================== +version: '3.9' + +networks: + ci-pkg-network: + +services: + + # ----- apache httpd + php + ci-pkg-web-server: + build: + context: . + dockerfile: ./containers/web-server/Dockerfile + image: "php:8.2-apache" + container_name: 'ci-pkg-server' + ports: + - '${APP_PORT}:80' + + working_dir: ${WEBROOT} + + volumes: + - ../:/var/www/${APP_NAME} + - ./containers/web-server/apache/sites-enabled:/etc/apache2/sites-enabled + - ./containers/web-server/php/extra-php-config.ini:/usr/local/etc/php/conf.d/extra-php-config.ini + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost"] + interval: 10s + timeout: 3s + retries: 5 + # start_period: 40s + + networks: + - ci-pkg-network + + user: ${DOCKER_USER_UID} + diff --git a/docker/init.sh b/docker/init.sh new file mode 100755 index 0000000000000000000000000000000000000000..060db24942ad10e1bb0f3abca96443d85f55d289 --- /dev/null +++ b/docker/init.sh @@ -0,0 +1,340 @@ +#!/bin/bash +# ====================================================================== +# +# DOCKER PHP DEV ENVIRONMENT :: INIT +# +# ---------------------------------------------------------------------- +# 2021-11-nn v1.0 <axel.hahn@iml.unibe.ch> +# 2022-07-19 v1.1 <axel.hahn@iml.unibe.ch> support multiple dirs for setfacl +# 2022-11-16 v1.2 <www.axel-hahn.de> use docker-compose -p "$APP_NAME" +# 2022-12-18 v1.3 <www.axel-hahn.de> add -p "$APP_NAME" in other docker commands +# 2022-12-20 v1.4 <axel.hahn@unibe.ch> replace fgrep with grep -F +# 2023-03-06 v1.5 <www.axel-hahn.de> up with and without --build +# 2023-08-17 v1.6 <www.axel-hahn.de> menu selection with single key (without return) +# ====================================================================== + +cd $( dirname $0 ) +. $( basename $0 ).cfg + +# git@git-repo.iml.unibe.ch:iml-open-source/docker-php-starterkit.git +selfgitrepo="docker-php-starterkit.git" + +_version="1.6" + +# ---------------------------------------------------------------------- +# FUNCTIONS +# ---------------------------------------------------------------------- + +# draw a headline 2 +function h2(){ + echo + echo -e "\e[33m>>>>> $*\e[0m" +} + +# draw a headline 3 +function h3(){ + echo + echo -e "\e[34m----- $*\e[0m" +} + +# function _gitinstall(){ +# h2 "install/ update app from git repo ${gitrepo} in ${gittarget} ..." +# test -d ${gittarget} && ( cd ${gittarget} && git pull ) +# test -d ${gittarget} || git clone -b ${gitbranch} ${gitrepo} ${gittarget} +# } + +# set acl on local directory +function _setWritepermissions(){ + h2 "set write permissions on ${gittarget} ..." + + local _user=$( id -gn ) + typeset -i local _user_uid=0 + test -f /etc/subuid && _user_uid=$( grep $_user /etc/subuid 2>/dev/null | cut -f 2 -d ':' )-1 + typeset -i local DOCKER_USER_OUTSIDE=$_user_uid+$DOCKER_USER_UID + + set -vx + + for mywritedir in ${WRITABLEDIR} + do + + echo "--- ${mywritedir}" + # remove current acl + sudo setfacl -bR "${mywritedir}" + + # default permissions: both the host user and the user with UID 33 (www-data on many systems) are owners with rwx perms + sudo setfacl -dRm u:${DOCKER_USER_OUTSIDE}:rwx,${_user}:rwx "${mywritedir}" + + # permissions: make both the host user and the user with UID 33 owner with rwx perms for all existing files/directories + sudo setfacl -Rm u:${DOCKER_USER_OUTSIDE}:rwx,${_user}:rwx "${mywritedir}" + done + + set +vx +} + +# cleanup starterkit git data +function _removeGitdata(){ + h2 "Remove git data of starterkit" + echo -n "Current git remote url: " + git config --get remote.origin.url + git config --get remote.origin.url 2>/dev/null | grep $selfgitrepo >/dev/null + if [ $? -eq 0 ]; then + echo + echo -n "Delete local .git and .gitignore? [y/N] > " + read answer + test "$answer" = "y" && ( echo "Deleting ... " && rm -rf ../.git ../.gitignore ) + else + echo "It was done already - $selfgitrepo was not found." + fi + +} + +# helper function: cut a text file starting from database start marker +# see _generateFiles() +function _fix_no-db(){ + local _file=$1 + if [ $DB_ADD = false ]; then + typeset -i local iStart=$( cat ${_file} | grep -Fn "$CUTTER_NO_DATABASE" | cut -f 1 -d ':' )-1 + if [ $iStart -gt 0 ]; then + sed -ni "1,${iStart}p" ${_file} + fi + fi +} + +# loop over all files in templates subdir make replacements and generate +# a target file. +# It skips if +# - 1st line is not starting with "# TARGET: filename" +# - target file has no updated lines +function _generateFiles(){ + + # re-read config vars + . $( basename $0 ).cfg + + local _tmpfile=/tmp/newfilecontent$$.tmp + h2 "generate files from templates..." + for mytpl in $( ls -1 ./templates/* ) + do + # h3 $mytpl + local _doReplace=1 + + # fetch traget file from first line + target=$( head -1 $mytpl | grep "^# TARGET:" | cut -f 2- -d ":" | awk '{ print $1 }' ) + + if [ -z "$target" ]; then + echo SKIP: $mytpl - target was not found in 1st line + _doReplace=0 + fi + + # write generated files to target + if [ $_doReplace -eq 1 ]; then + + # write file from line 2 to a tmp file + sed -n '2,$p' $mytpl >$_tmpfile + + # add generator + # sed -i "s#{{generator}}#generated by $0 - template: $mytpl - $( date )#g" $_tmpfile + local _md5=$( md5sum $_tmpfile | awk '{ print $1 }' ) + sed -i "s#{{generator}}#GENERATED BY $( basename $0 ) - template: $mytpl - $_md5#g" $_tmpfile + + # loop over vars to make the replacement + grep "^[a-zA-Z]" $( basename $0 ).cfg | while read line + do + # echo replacement: $line + mykey=$( echo $line | cut -f 1 -d '=' ) + myvalue="$( eval echo \"\${$mykey}\" )" + # grep "{{$mykey}}" $_tmpfile + + # TODO: multiline values fail here in replacement with sed + sed -i "s#{{$mykey}}#${myvalue}#g" $_tmpfile + done + _fix_no-db $_tmpfile + + # echo "changes for $target:" + diff "../$target" "$_tmpfile" | grep -v "$_md5" | grep -v "^---" | grep . + if [ $? -eq 0 -o ! -f "../$target" ]; then + echo -n "$mytpl - changes detected - writing [$target] ... " + mkdir -p $( dirname "../$target" ) || exit 2 + mv "$_tmpfile" "../$target" || exit 2 + echo OK + else + rm -f $_tmpfile + echo "SKIP: $mytpl - Nothing to do." + fi + fi + echo + done + +} + +# loop over all files in templates subdir make replacements and generate +# a traget file. +function _removeGeneratedFiles(){ + h2 "remove generated files..." + for mytpl in $( ls -1 ./templates/* ) + do + h3 $mytpl + + # fetch traget file from first line + target=$( head -1 $mytpl | grep "^# TARGET:" | cut -f 2- -d ":" | awk '{ print $1 }' ) + + if [ ! -z "$target" -a -f "../$target" ]; then + echo -n "REMOVING " + ls -l "../$target" || exit 2 + rm -f "../$target" || exit 2 + echo OK + else + echo SKIP: $target + fi + + done +} + +function _showContainers(){ + local bLong=$1 + h2 CONTAINERS + if [ -z "$bLong" ]; then + docker-compose -p "$APP_NAME" ps + else + docker ps | grep $APP_NAME + fi +} + + +# a bit stupid ... i think I need to delete it. +function _showInfos(){ + _showContainers long + h2 INFO + + h3 "processes" + docker-compose top + + h3 "Check app port" + >/dev/tcp/localhost/${APP_PORT} 2>/dev/null && ( + echo "OK, app port ${APP_PORT} is reachable" + echo + echo "In a web browser open:" + echo " $frontendurl" + ) + h3 "Check database port" + >/dev/tcp/localhost/${DB_PORT} 2>/dev/null && ( + echo "OK, db port ${DB_PORT} is reachable" + echo + echo "In a local DB admin tool:" + echo " host : localhost" + echo " port : ${DB_PORT}" + echo " user : root" + echo " password: ${MYSQL_ROOT_PASS}" + ) + echo +} + +# helper for menu: print an inverted key +function _key(){ + printf "\e[4;7m ${1} \e[0m" +} + +# helper: wait for a return key +function _wait(){ + echo -n "... press RETURN > "; read -r +} + +# ---------------------------------------------------------------------- +# MAIN +# ---------------------------------------------------------------------- + +action=$1 + +while true; do + echo + echo -e "\e[32m===== INITIALIZER FOR DOCKER APP [$APP_NAME] v$_version ===== \e[0m\n\r" + + if [ -z "$action" ]; then + + _showContainers + + h2 MENU + echo " $( _key g ) - remove git data of starterkit" + echo + echo " $( _key i ) - init application: set permissions" + echo " $( _key t ) - generate files from templates" + echo " $( _key T ) - remove generated files" + echo + echo " $( _key u ) - startup containers docker-compose ... up -d" + echo " $( _key U ) - startup containers docker-compose ... up -d --build" + echo " $( _key s ) - shutdown containers docker-compose stop" + echo " $( _key r ) - remove containers docker-compose rm -f" + echo + echo " $( _key m ) - more infos" + echo " $( _key c ) - console (bash)" + echo + echo " $( _key q ) - quit" + echo + echo -n " select >" + read -rn 1 action + echo + fi + + case "$action" in + g) + _removeGitdata + ;; + i) + # _gitinstall + _setWritepermissions + ;; + t) + _generateFiles + ;; + T) + _removeGeneratedFiles + rm -rf containers + ;; + # not in the menu + # f) + # _removeGeneratedFiles + # _generateFiles + # _wait + # ;; + m) + _showInfos + _wait + ;; + u|U) + dockerUp="docker-compose -p "$APP_NAME" --verbose up -d --remove-orphans" + if [ "$action" = "U" ]; then + dockerUp+=" --build" + fi + if $dockerUp; then + echo "In a web browser:" + echo " $frontendurl" + else + echo "ERROR: docker-compose up failed :-/" + docker-compose -p "$APP_NAME" logs | tail + fi + echo + + _wait + ;; + s) + docker-compose -p "$APP_NAME" stop + ;; + r) + docker-compose -p "$APP_NAME" rm -f + ;; + c) + docker ps + echo -n "id or name >" + read dockerid + test -z "$dockerid" || docker exec -it $dockerid /bin/bash + ;; + q) + exit 0; + ;; + *) + test -n "$action" && ( echo " ACTION FOR [$action] NOT IMPLEMENTED."; sleep 1 ) + esac + action= +done + + +# ---------------------------------------------------------------------- diff --git a/docker/init.sh.cfg b/docker/init.sh.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b96727ecfc897540f1dbbe911cd3293f12d74558 --- /dev/null +++ b/docker/init.sh.cfg @@ -0,0 +1,68 @@ +# ====================================================================== +# +# settings for init.sh and base values for replacements in template files +# This script is sourced by init.sh ... this file is bash syntax +# +# ---------------------------------------------------------------------- +# 2021-12-17 <axel.hahn@iml.unibe.ch> +# ====================================================================== + +APP_NAME=ci-pkg + +# web port 80 in container is seen on localhost as ... +APP_PORT=8001 + +APP_APT_PACKAGES="git unzip zip libapache2-mod-xsendfile" + +#APP_APACHE_MODULES="rewrite" +APP_APACHE_MODULES="xsendfile" + +APP_PHP_VERSION=8.2 +# APP_PHP_MODULES="curl pdo_mysql mbstring xml zip xdebug" +APP_PHP_MODULES="" + +# optional exec command after container was started with init.sh script +# APP_ONSTARTUP="php /var/www/${APP_NAME}/public_html/myservice.php" +APP_ONSTARTUP="" + +# ---------------------------------------------------------------------- + +# add a container with database? +DB_ADD=false + +# ---------------------------------------------------------------------- +# for an optional database server + +DB_PORT=13306 + +# ----- database settings +MYSQL_IMAGE=mariadb:10.5.9 +MYSQL_RANDOM_ROOT_PASSWORD=0 +MYSQL_ALLOW_EMPTY_PASSWORD=0 +MYSQL_ROOT_PASS=12345678 +MYSQL_USER=${APP_NAME} +MYSQL_PASS=mypassword +MYSQL_DB=${APP_NAME} + + + +# ====================================================================== +# ignore things below + + +# where to set acl where local user and web user in container +# can write simultanously +WRITABLEDIR=../public_html + + +# web service user in container +DOCKER_USER_UID=33 + +# document root inside web-server container +WEBROOT=/var/www/${APP_NAME}/public_html + +CUTTER_NO_DATABASE="CUT-HERE-FOR-NO-DATABASE" + +frontendurl=http://localhost:${APP_PORT}/ + +# ---------------------------------------------------------------------- diff --git a/docker/templates/docker-compose.yml b/docker/templates/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..3e039a763699162c9f53e84e5e7a78ddfa34d717 --- /dev/null +++ b/docker/templates/docker-compose.yml @@ -0,0 +1,71 @@ +# TARGET: docker/docker-compose.yml +# +# {{generator}} +# +# ====================================================================== +# +# (1) see .env for set variables +# (2) run "docker-compose up" to startup +# +# ====================================================================== +version: '3.9' + +networks: + {{APP_NAME}}-network: + +services: + + # ----- apache httpd + php + {{APP_NAME}}-web-server: + build: + context: . + dockerfile: ./containers/web-server/Dockerfile + image: "php:{{APP_PHP_VERSION}}-apache" + container_name: '{{APP_NAME}}-server' + ports: + - '${APP_PORT}:80' + + working_dir: ${WEBROOT} + + volumes: + - ../:/var/www/${APP_NAME} + - ./containers/web-server/apache/sites-enabled:/etc/apache2/sites-enabled + - ./containers/web-server/php/extra-php-config.ini:/usr/local/etc/php/conf.d/extra-php-config.ini + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost"] + interval: 10s + timeout: 3s + retries: 5 + # start_period: 40s + + networks: + - {{APP_NAME}}-network + + user: ${DOCKER_USER_UID} + + # --- 8< --- {{CUTTER_NO_DATABASE}} --- 8< --- + + depends_on: + - {{APP_NAME}}-db-server + + # ----- mariadb + {{APP_NAME}}-db-server: + image: {{MYSQL_IMAGE}} + container_name: '${APP_NAME}-db' + # restart: always + ports: + - '${DB_PORT}:3306' + environment: + MYSQL_ROOT_PASSWORD: '${MYSQL_ROOT_PASS}' + MYSQL_USER: '${MYSQL_USER}' + MYSQL_PASSWORD: '${MYSQL_PASS}' + MYSQL_DATABASE: '${MYSQL_DB}' + volumes: + # - ./containers/db-server/db_data:/var/lib/mysql + - ./containers/db-server/mariadb/my.cnf:/etc/mysql/conf.d/my.cnf + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + retries: 5 + networks: + - {{APP_NAME}}-network diff --git a/docker/templates/dot_env b/docker/templates/dot_env new file mode 100644 index 0000000000000000000000000000000000000000..bc8af1d5372ca0731b4e2eebd0a3e95cc2d1c78b --- /dev/null +++ b/docker/templates/dot_env @@ -0,0 +1,28 @@ +# TARGET: docker/.env +# ====================================================================== +# +# {{generator}} +# values to be used in docker-composer.yml +# +# ====================================================================== + +# ----- application +APP_NAME={{APP_NAME}} + +# uid of www-data in the docker container +DOCKER_USER_UID={{DOCKER_USER_UID}} + +APP_PORT={{APP_PORT}} +WEBROOT={{WEBROOT}} + +# --- 8< --- {{CUTTER_NO_DATABASE}} --- 8< --- + +DB_PORT={{DB_PORT}} + +# ----- database settings +MYSQL_RANDOM_ROOT_PASSWORD={{MYSQL_RANDOM_ROOT_PASSWORD}} +MYSQL_ALLOW_EMPTY_PASSWORD={{MYSQL_ALLOW_EMPTY_PASSWORD}} +MYSQL_ROOT_PASS={{MYSQL_ROOT_PASS}} +MYSQL_USER={{APP_NAME}} +MYSQL_PASS={{MYSQL_PASS}} +MYSQL_DB={{APP_NAME}} diff --git a/docker/templates/extra-php-config.ini b/docker/templates/extra-php-config.ini new file mode 100644 index 0000000000000000000000000000000000000000..d3f53279bbc03ee66876efaeec97cfdd207f3c94 --- /dev/null +++ b/docker/templates/extra-php-config.ini @@ -0,0 +1,25 @@ +# TARGET: docker/containers/web-server/php/extra-php-config.ini +; +; {{generator}} +; +[PHP] + +error_reporting=E_ALL +display_errors=1 + +; ---------------------------------------------------------------------- +; XDEBUG STUFF BELOW +; ---------------------------------------------------------------------- +; +; error_reporting=E_ALL +; +; [xdebug] +; xdebug.mode=develop,debug +; ; xdebug.client_host=localhost +; xdebug.start_with_request=yes +; ; xdebug.start_with_request=trigger +; +; xdebug.log=/tmp/xdebug.log +; xdebug.discover_client_host = 1 +; ; xdebug.client_port=9003 +; xdebug.idekey="netbeans-xdebug" \ No newline at end of file diff --git a/docker/templates/my.cnf b/docker/templates/my.cnf new file mode 100644 index 0000000000000000000000000000000000000000..3692f17bf2799b45b3ce9746e31e3e8d1fcb5d64 --- /dev/null +++ b/docker/templates/my.cnf @@ -0,0 +1,4 @@ +# TARGET: docker/containers/db-server/mariadb/my.cnf +[mysqld] +; collation-server = utf8mb4_unicode_ci +; character-set-server = utf8mb4 \ No newline at end of file diff --git a/docker/templates/readme.md b/docker/templates/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..34c2c57c1ad0518f5cc00d12148cdddc0720187f --- /dev/null +++ b/docker/templates/readme.md @@ -0,0 +1,7 @@ +# Templates + +## Rules + +* in the first line must be a line `# TARGET: [name of target file]` to define the target file +* Placeholdrs have the syntax variable in double brackets, i.e. `{{VARNAME}}` +* variables to be replaced are those in docker/init.sh.cfg and `{{genrator}}` diff --git a/docker/templates/vhost_app.conf b/docker/templates/vhost_app.conf new file mode 100644 index 0000000000000000000000000000000000000000..7dcadadf78dda6ebabf4da613836f5fda115eb2c --- /dev/null +++ b/docker/templates/vhost_app.conf @@ -0,0 +1,34 @@ +# TARGET: docker/containers/web-server/apache/sites-enabled/vhost_app.conf +# +# {{generator}} +# +<VirtualHost *:80> + DocumentRoot {{WEBROOT}} + <Directory {{WEBROOT}}> + AllowOverride None + Order Allow,Deny + Allow from All + </Directory> + + # redirect requests to handle packages + <Location "/packages"> + + # for Php as php-fpm service: + # SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 + + RewriteEngine on + RewriteCond %{REQUEST_FILENAME} !-f + RewriteRule ^(.*)$ index.php [QSA,L] + + </Location> + + # download files are outside webroot + XSendFile On + XSendFilePath "/var/www/{{APP_NAME}}/example-packages/" + + # example to prevent access with http + <Location "/no-access"> + Require all denied + </Location> + +</VirtualHost> \ No newline at end of file diff --git a/docker/templates/web-server-Dockerfile b/docker/templates/web-server-Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d2443af6363a7d5ad2194d9c02349fb8fc3e218d --- /dev/null +++ b/docker/templates/web-server-Dockerfile @@ -0,0 +1,15 @@ +# TARGET: docker/containers/web-server/Dockerfile +# +# {{generator}} +# +FROM php:{{APP_PHP_VERSION}}-apache + +# install packages +RUN apt-get update && apt-get install -y {{APP_APT_PACKAGES}} + +# enable apache modules +RUN a2enmod {{APP_APACHE_MODULES}} + +# install php packages +COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/ +RUN install-php-extensions {{APP_PHP_MODULES}} diff --git a/docs/20_Installation.md b/docs/20_Installation.md index 79c8127a44a6a61d8908f4b28a0678e238834d02..97212ff5479c2a711699c9722b68535df6105582 100644 --- a/docs/20_Installation.md +++ b/docs/20_Installation.md @@ -1,10 +1,9 @@ # Installation on server - ## Receive data * Create an ssh user "deployment" to receive data -* Create a package directory - it can be outside webroot eg. /var/www/cipkg.example.com/packages/ with write permissions for user "deployment" and read persmissions for webserver. +* Create a package directory - it can be outside webroot eg. /var/www/cipkg.example.com/packages/ with write permissions for user "deployment" and read permissions for webserver. ```txt mkdir /var/www/cipkg.example.com/packages/ @@ -12,7 +11,11 @@ chown deployment:www-data /var/www/cipkg.example.com/packages/ chmod 750 /var/www/cipkg.example.com/packages/ ``` -* Configue the ci sever to rsync with ssh user "deployment" here +In the config of CI web server add a sync target. Use + +* the deployment user as ssh +* the fqdn as hostname +* the defined *packagedir* in your inc_config.php as target directory ## Xsentfile module @@ -29,7 +32,6 @@ path on your websever. Redirect all requests to /packages/[whatever] to /packages/index.php - Example snippet ```text @@ -39,6 +41,9 @@ Example snippet <Location "/packages"> + # for Php as php-fpm service: + SetEnvIf Authorization "(.*)" HTTP_AUTHORIZATION=$1 + RewriteEngine on RewriteCond %{REQUEST_FILENAME} !-f RewriteRule ^(.*)$ index.php [QSA,L] diff --git a/docs/30_Configuration.md b/docs/30_Configuration.md index 8088eb91c86ded58393776cd254d2ae6fce0cef4..be6ce5ced351832b9134b71f4276e697a9c90378 100644 --- a/docs/30_Configuration.md +++ b/docs/30_Configuration.md @@ -21,8 +21,6 @@ return array( 'maxage'=>60, // force that a hash can be used only once - // a side effect is that fast repeat or simultanius requests - // will be denied. 'onetimesecret'=>true, // filesize of lock file with stored hashed before starting garbage collection @@ -35,27 +33,20 @@ return array( // allow directory listing when accessing a path of a package // true is required to fetch all packages 'showdircontent'=>true, -); -``` - -## Prepare receive of packages - -* Create an deployment account package server that can be used to be connected - via SSH by the ci server -* add the public key of www-data of the ci server into - /home/deployment/.ssh/authorized keys -* Set permissions that the deployment user can write into - /var/www/cipkg.example.com/packages/ - and the user of the webeservice can read it - `chown deployment:apache /var/www/cipkg.example.com/packages/` and - `chmod 750 /var/www/cipkg.example.com/packages/` -## Ci server: add a sync target + // Enable for troubleshooting + 'debug'=>false, -TODO - -In the config of CI web server add a sync target. Use +); +``` -* the deployment user as ssh -* the fqdn as hostname -* the defined *packagedir* in your inc_config.php as target directory +| Key | Description | +|--- |--- | +| apikey | A secret for the server. A client that wants to fetch a package must use the same secret | +| packagedir | physical folder where to find the packages. To this folder you need to point XSendFilePath in your apache httpd vhost too. | +| maxage | max age of request ... client and server need to be in sync | +| onetimesecret | force that a hash can be used only once. There should be no reason to turn it off. | +| maxlockfilesize | filesize of lock file with stored hashed before starting garbage collection. 10.000 byte are reached after 114 req | +| tmpdir | tmp dir to store used hashes | +| showdircontent | allow directory listing when accessing a path of a package. true is required to fetch all packages by a single request | +| debug | enable debug output | diff --git a/docs/40_Usage.md b/docs/40_Usage.md index 561641bf0614788e4c2285b2dbff5df48eb48e39..3bdc5384d18f0bfcb647f6f2784c68196ef152bf 100644 --- a/docs/40_Usage.md +++ b/docs/40_Usage.md @@ -7,6 +7,11 @@ See deployment project <https://git-repo.iml.unibe.ch/iml-open-source/imldeploym The download script is bin/getfile.sh. +```text +wget -O getfile.sh "https://git-repo.iml.unibe.ch/iml-open-source/imldeployment-client/-/raw/master/bin/getfile.sh?ref_type=heads" +chmod 755 getfile.sh +``` + ## How does it work? TODO: needs to be completed. @@ -33,3 +38,57 @@ Possible GET requests are: If a valid request came in then the hash will be written to `[approot]/tmp/used_hashes.txt`. This file will be cleaned up if reaching the defined file size with value of *maxlockfilesize*. + +## Test package download + +If you use the docker environment for development: + +In your app root you there is a folder "example-packages". Inside the docker container it is available as /var/www/ci-pkg/example-packages/. + +* Below the package folder folders are subfolders for phases (preview, stage, live and "test"). +* below a phase are the folders with the project id +* inside the project folder are the files per project + +```text +example-packages/ +├── live +├── preview +├── stage +└── test + └── example-prj + └── hello.txt +``` + +In your app root go to the the "tests" folder. +This will download the "hello.txt" into the current folder: + +```txt +./getfile.sh -u http://localhost:8001 -s myapikey -e test -p example-prj -f hello.txt +-rw-r--r-- 1 axel axel 12 Sep 15 14:34 hello.txt +``` + +For less params with getfile.sh there is a config: + +```txt +cat getfile.sh.cfg +# for less params with getfile.sh +IMLCI_PKG_SECRET=myapikey +IMLCI_URL=http://localhost:8001 +IMLCI_PHASE=test +``` + +With it you can execute ``./getfile.sh -p example-prj -f hello.txt`` too. + +If you enabled the file listing you get a list of files: + +```txt +./getfile.sh -p example-prj +file:hello.txt +``` + +## Troubleshooting + +To have more output you have these possibilities: + +* in the command with ./getfile.sh add the flag ``-d`` to enable debugging for this script +* in public_html/inc_config.php set the key debug to enable the debugging on server (disable it as soon you can) diff --git a/example-packages/test/example-prj/hello.txt b/example-packages/test/example-prj/hello.txt new file mode 100644 index 0000000000000000000000000000000000000000..6769dd60bdf536a83c9353272157893043e9f7d0 --- /dev/null +++ b/example-packages/test/example-prj/hello.txt @@ -0,0 +1 @@ +Hello world! \ No newline at end of file diff --git a/public_html/inc_config.php.dist b/public_html/inc_config.php.dist index 58ac35d54112d2aab69d1f40e8c65c45235e06c5..0d163efdf9224ee8a273cf85b62d4e93e0182d9d 100644 --- a/public_html/inc_config.php.dist +++ b/public_html/inc_config.php.dist @@ -27,4 +27,7 @@ return array( // allow directory listing when accessing a path of a package // true is required to fetch all packages 'showdircontent'=>true, + + // Enable for troubleshooting + 'debug'=>false, ); \ No newline at end of file diff --git a/public_html/inc_functions.php b/public_html/inc_functions.php index 1756e2e095a5c88f8ace7dc89896173f9892358b..ea62f071a8534adce95a05f8fb99de5b6b26024b 100644 --- a/public_html/inc_functions.php +++ b/public_html/inc_functions.php @@ -25,7 +25,7 @@ function _checkAuth($sMySecret, $iMaxAge=60){ $sGotReq=$_SERVER['REQUEST_URI']; - $sMyData="${sGotMethod}\n${sGotReq}\n${sGotDate}\n"; + $sMyData="{$sGotMethod}\n{$sGotReq}\n{$sGotDate}\n"; $sMyHash= base64_encode(hash_hmac("sha1", $sMyData, $sMySecret)); _wd('Hash: '.$sGotHash.' -- from header'); @@ -164,7 +164,8 @@ function _sendHtml($sTitle, $sContent){ function _wd($s, $sLevel='info'){ global $bDebug; if ($bDebug){ - echo '<div class="debug debug-'.$sLevel.'">DEBUG: '.$s.'</div>'; + // echo '<div class="debug debug-'.$sLevel.'">DEBUG: '.$s.'</div>'; + echo "DEBUG[$sLevel]: $s<br>\n"; } return true; } diff --git a/public_html/packages/index.php b/public_html/packages/index.php index cc987e1c4e4e642a8888fe1797d61468671d2cb0..b72885a91ff6fd00defdc8702e390ef13e9222d9 100644 --- a/public_html/packages/index.php +++ b/public_html/packages/index.php @@ -6,17 +6,19 @@ * GET /packages/[phase]/[ID]/[filename] * * ---------------------------------------------------------------------- - * 2021-03-31 v0.0 <axel.hahn@iml.unibe.ch> init + * 2021-03-31 v1.0 <axel.hahn@iml.unibe.ch> init + * 2023-09-15 v1.1 <axel.hahn@unibe.ch> debug now driven by config * ====================================================================== */ - $bDebug=false; + ini_set('display_errors', 1); ini_set('display_startup_errors', 1); error_reporting(E_ALL); require_once('../inc_functions.php'); - $aConfig=require_once("../inc_config.php"); + $aConfig=require_once("../inc_config.php"); + $bDebug=(isset($aConfig['debug']) && $aConfig['debug']) ? true : false; $lockfile=$aConfig['tmpdir'].'/used_hashes.txt'; $iMaxAge=$aConfig['maxage']; @@ -26,9 +28,9 @@ // MAIN // ---------------------------------------------------------------------- - _wd('Start: '.date('Y-m-d H:i:s').'<style>body{background:#eee; color:#456;} - .debug{background:#ddd; margin-bottom: 2px;} - </style>'); + // _wd('Start: '.date('Y-m-d H:i:s').'<style>body{background:#eee; color:#456;} + // .debug{background:#ddd; margin-bottom: 2px;} + // </style>'); _wd('request uri is '.$_SERVER["REQUEST_URI"]); _wd('<pre>GET: '.print_r($_GET, 1).'</pre>'); @@ -77,7 +79,7 @@ if (!file_exists($sMyFile)){ _quit('File not found.', 404); } - + _wd('file exists send X-Sendfile header...'); // let the webserver deliver a given file header('X-Sendfile: ' . $sMyFile); diff --git a/tests/getfile.sh b/tests/getfile.sh new file mode 100755 index 0000000000000000000000000000000000000000..b4dcfdec5bbff702eba6a618e85a347f0aee70b6 --- /dev/null +++ b/tests/getfile.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash +# ====================================================================== +# +# API CLIENT :: GET A CI FILE FROM PACKAGE SERVER +# +# Source: https://git-repo.iml.unibe.ch/iml-open-source/imldeployment-client/ +# ---------------------------------------------------------------------- +# 2021-03-31 v1.0 <axel.hahn@iml.unibe.ch> init +# 2021-04-13 v1.1 <axel.hahn@iml.unibe.ch> add support for custom config +# 2021-04-15 v1.2 <axel.hahn@iml.unibe.ch> added debugging of curl request +# 2021-10-14 v1.3 <axel.hahn@iml.unibe.ch> add nanoseconds in hashed base data +# 2023-02-14 v1.4 <axel.hahn@unibe.ch> compatibility to openssl v3 +# ====================================================================== + +# ---------------------------------------------------------------------- +# CONFIG +# ---------------------------------------------------------------------- + +version="v1.4" +about="CI PACKAGE GETTER $version; +(c) 2021 Institute for Medical Education (IML); University of Bern; +GNU GPL 3.0" + +line="----------------------------------------------------------------------" +bDebug=0 +customconfig= + +. $0.cfg + +# ---------------------------------------------------------------------- +# FUNCTIONS +# ---------------------------------------------------------------------- + +function showhelp(){ +self=$( basename $0 ) +echo "$line +$about +$line + +Get packages from a software sattelite of IML ci server. + +SYNTAX: + + $self [OPTIONS] + +OPTIONS: + + -h Show this help + -v Show version + + -c CFGFILE load custom config file after defaults in $self.cfg + -d enable debug infos + -e PHASE phase; overrides env variable IMLCI_PHASE + -f FILE filename to get (without path); overrides env variable IMLCI_FILE + -l ITEM list + -o OUTFILE optional output file + -p PROJECT ci project id; overrides env variable IMLCI_PROJECT + -s SECRET override secret in IMLCI_PKG_SECRET + -u URL URL of iml ci server without trailing /; overrides env variable IMLCI_URL + +VALUES: + + CFGFILE custom config file. It is useful to handle files of different + projects on a server. + PHASE is a phase of the ci server; one of preview|stage|live + FILE is a filename without path that was created by ci server. + OUTFILE Output file. It can countain a path. If none is given the filename + will be taken from FILE and stored in current directory + PROJECT project id of the ci server + SECRET secret to access project data on package server. Your given secret + must match the secret on package server to get access to any url. + ITEM type what to list; one of phases|projects|files + To list projects a phase must be set. + To list files a phase and a project must be set. + +DEFAULTS: + + You don't need to set all values by command line. Use a config to set defaults + $0.cfg + +EXAMPLES: + + If url, secret, project and phase are set in the config you can operate by + setting the filename to request. + + $self -f FILE + downloads FILE to the current dir. + + $self -f FILE -o my-own-filename.tgz + downloads FILE as my-own-filename.tgz + + $self -f ALL + there is a special file ALL; it fetches all filenames by executing a directory + listing and then downloads all remote files with their original name + + $self -e preview -l projects + list existing projects in phase preview + + $self -l files + list existing files of current project + + Remark: The directory listing can be turned off on the package server and + results in a 403 status. +" +} + +# make an http request to fetch the software +# +# param string method; should be GET +# param string request url (without protocol and server) +# param string optional: filename for output data +# param string optional: secret; default: it will be generated +# +# global int bDebug (0|1) +# global string line string for a line with dashes +function makeRequest(){ + + local apiMethod=$1 + local apiRequest=$2 + local outfile=$3 + local secret=$4 + + # local outfile=$( mktemp ) + + if [ $bDebug = 1 ]; then + echo $line + echo "$apiMethod ${apiHost}${apiRequest}" + echo $line + fi + + if [ ! -z "$secret" ]; then + + # --- date in http format + LANG=en_EN + # export TZ=GMT + apiTS=$(date "+%a, %d %b %Y %H:%M:%S.%N %Z") + + +# --- generate data to hash: method + uri + timestamp; delimited with line break +data="${apiMethod} +${apiRequest} +${apiTS} +" + # these ase non critical data ... it does not show the ${secret} + if [ "$bDebug" = "1" ]; then + echo "RAW data for hashed secret:" + echo "$data" + fi + + # generate hash - split in 2 commands (piping "cut" sends additional line break) + myHash=$(echo -n "$data" | openssl dgst -sha1 -hex -hmac "${secret}" | cut -f 2 -d " ") + myHash=$(echo -n "$myHash" | base64) + + moreheaders="--fail" + test $bDebug = 1 && moreheaders="-i" + + tmpdownloadfile="${outfile}.downloading" + + curl \ + -H "Accept: application/json" -H "Content-Type: application/json" \ + -H "Date: ${apiTS}" \ + -H "Authorization: bash-client:${myHash}" \ + -X $apiMethod \ + -o "${tmpdownloadfile}" \ + $moreheaders \ + -s \ + ${IMLCI_URL}${apiRequest} + + rc=$? + if [ "$bDebug" = "1" ]; then + cat "${tmpdownloadfile}" + rm -f "${tmpdownloadfile}" + exit 0 + fi + + if [ $rc -eq 0 ]; then + # echo OK. + + # no outfile (= request to a directory) + if [ -z "$outfile" ]; then + # echo + # echo ----- RESPONSE BODY: + cat "${tmpdownloadfile}" + rm -f "${tmpdownloadfile}" + else + mv "${tmpdownloadfile}" "${outfile}" + ls -l "${outfile}" + fi + else + echo ERROR: Download failed. + exit 1 + fi + else + curl\ + -H "Accept: application/json" -H "Content-Type: application/json" \ + -X $apiMethod \ + -o "${tmpdownloadfile}" \ + ${IMLCI_URL}${apiRequest} + fi + +} + + +# ---------------------------------------------------------------------- +# MAIN +# ---------------------------------------------------------------------- + +if [ $# -lt 1 ]; then + showhelp + exit 1 +fi + + +while getopts "c:de:f:hl:o:p:s:u:v" option; do + case ${option} in + c) customconfig="$OPTARG" ;; + d) bDebug=1 ;; + e) export IMLCI_PHASE=$OPTARG ;; + f) export IMLCI_FILE=$OPTARG ;; + h) showhelp + exit 0 + ;; + l) case $OPTARG in + phases) + IMLCI_PHASE='' + IMLCI_PROJECT='' + IMLCI_FILE='' + ;; + projects) + IMLCI_PROJECT='' + IMLCI_FILE='' + ;; + files) + IMLCI_FILE='' + ;; + *) + echo ERROR: invalid value for option [-l] + echo + showhelp + exit 2 + esac + ;; + o) export IMLCI_OUTFILE=$OPTARG ;; + p) export IMLCI_PROJECT=$OPTARG ;; + s) export IMLCI_PKG_SECRET=$OPTARG ;; + u) export IMLCI_URL=$OPTARG ;; + v) echo $about; exit 0 ;; + *) + echo ERROR: invalid option [${option}] + echo + showhelp + exit 2 + esac +done + +if [ ! -z "$customconfig" ]; then + if [ -r "$customconfig" ]; then + . "$customconfig" || exit 2 + else + echo "ERROR: unable to read custom config [$customconfig]." + exit 2 + fi +fi + +test -z ${IMLCI_OUTFILE} && IMLCI_OUTFILE=$IMLCI_FILE + +if [ $bDebug = 1 ]; then + pre=">>>>>> " + echo $line + echo + echo DEBUG INFOS + echo + echo "${pre} defaults in $0.cfg" + cat $0.cfg 2>/dev/null + echo + if [ ! -z "$customconfig" ]; then + echo "${pre} custom config $customconfig" + cat "$customconfig" + echo + fi + echo "${pre} Params (override default values)" + echo $* + echo + echo "${pre} effective values" + echo "IMLCI_URL = $IMLCI_URL" + echo "IMLCI_PKG_SECRET = $IMLCI_PKG_SECRET" + echo "IMLCI_PROJECT = $IMLCI_PROJECT" + echo "IMLCI_PHASE = $IMLCI_PHASE" + echo "IMLCI_FILE = $IMLCI_FILE" + echo "IMLCI_OUTFILE = $IMLCI_OUTFILE" + + echo +fi + +if [ "$IMLCI_FILE" = "ALL" ]; then + # echo ALL files were requested ... + printf "%-30s" "get list of all files... " + tmpfilelist=$( mktemp ) + $0 -u "${IMLCI_URL}" \ + -p "${IMLCI_PROJECT}" \ + -e "${IMLCI_PHASE}" \ + -s "${IMLCI_PKG_SECRET}" \ + -l files \ + -o "${tmpfilelist}" + + # cat "${tmpfilelist}" + cat "${tmpfilelist}" | grep "^file:" | while read fileline + do + # echo $line + myfile=$( echo $fileline | cut -f 2- -d ':' ) + printf "%-30s" "GET $myfile... " + $0 -u "${IMLCI_URL}" \ + -p "${IMLCI_PROJECT}" \ + -e "${IMLCI_PHASE}" \ + -s "${IMLCI_PKG_SECRET}" \ + -f "${myfile}" + done + rm -f "${tmpfilelist}" +else + makeRequest GET "/packages/$IMLCI_PHASE/$IMLCI_PROJECT/$IMLCI_FILE" "$IMLCI_OUTFILE" "$IMLCI_PKG_SECRET" +fi diff --git a/tests/getfile.sh.cfg b/tests/getfile.sh.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8a323abb1a731573d4003fe15604d9fbcbd29c55 --- /dev/null +++ b/tests/getfile.sh.cfg @@ -0,0 +1,4 @@ +# for less params with getfile.sh +IMLCI_PKG_SECRET=myapikey +IMLCI_URL=http://localhost:8001 +IMLCI_PHASE=test diff --git a/tests/hello.txt b/tests/hello.txt new file mode 100644 index 0000000000000000000000000000000000000000..6769dd60bdf536a83c9353272157893043e9f7d0 --- /dev/null +++ b/tests/hello.txt @@ -0,0 +1 @@ +Hello world! \ No newline at end of file diff --git a/tests/readme.md b/tests/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..9d2cd996bfd3e3721c6ce2893e7dee433c558984 --- /dev/null +++ b/tests/readme.md @@ -0,0 +1,9 @@ +# Hints + +getfile.sh is part of the deployment clients that fatches the packages from package server. +To get/ update the script + +wget -O getfile.sh "https://git-repo.iml.unibe.ch/iml-open-source/imldeployment-client/-/raw/master/bin/getfile.sh?ref_type=heads" +chmod 755 getfile.sh + +see also ../docs/40_Usage.md.