From dd9b2ff7a90bddac0e4b92162a6ebf97b77763bc Mon Sep 17 00:00:00 2001
From: "Hahn Axel (hahn)" <axel.hahn@iml.unibe.ch>
Date: Thu, 1 Apr 2021 15:09:44 +0200
Subject: [PATCH] implemented file listing and donwload of all files
---
public_html/inc_config.php.dist | 8 ++-
public_html/packages/index.php | 19 ++++--
readme.md | 52 ++++++++++++----
shellscripts/getfile.sh | 102 ++++++++++++++++++++++++++-----
shellscripts/getfile.sh.cfg.dist | 1 -
5 files changed, 148 insertions(+), 34 deletions(-)
diff --git a/public_html/inc_config.php.dist b/public_html/inc_config.php.dist
index e00c2c8..080cff8 100644
--- a/public_html/inc_config.php.dist
+++ b/public_html/inc_config.php.dist
@@ -1,7 +1,13 @@
<?php
return array(
+
+ // define a secret aka pi key
'apikey'=>'our-package-server-secret',
- 'cutfromrequest'=>'^/packages',
+
+ // local directory of synched ci packages
'packagedir'=>dirname(__DIR__).'/packages',
+
+ // allow directory listing when accessing a path of a package
+ 'showdircontent'=>true,
);
\ No newline at end of file
diff --git a/public_html/packages/index.php b/public_html/packages/index.php
index 988a821..7f35bd4 100644
--- a/public_html/packages/index.php
+++ b/public_html/packages/index.php
@@ -1,7 +1,7 @@
<?php
/* ======================================================================
*
- * A P I F O R C I S E R V E R
+ * A P I F O R C I P A C K A G E S E R V E R
*
* GET /packages/[phase]/[ID]/[filename]
*
@@ -10,7 +10,7 @@
* ======================================================================
*/
- $bDebug=true;
+ $bDebug=false;
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
@@ -39,13 +39,24 @@
// ---------- SPLIT URL
- $sRelfile=preg_replace('#'.$aConfig['cutfromrequest'].'#', '', $_SERVER["REQUEST_URI"]);
+ $sRelfile=preg_replace('#^/packages#', '', $_SERVER["REQUEST_URI"]);
_wd('$sRelfile: '.$sRelfile);
-
+
$sMyFile=$aConfig['packagedir'].$sRelfile;
_wd('full path of file: '.$sMyFile);
+ if (is_dir($sMyFile)){
+ if(!$aConfig['showdircontent']){
+ _quit('Filelisting is denied by config.', 403);
+ } else {
+ foreach( array_diff(scandir($sMyFile), array('.', '..')) as $sEntry){
+ echo filetype($sMyFile.'/'.$sEntry).':'.str_replace($aConfig['packagedir'], '_', $sEntry).PHP_EOL;
+ }
+ die();
+ }
+ }
+
if (!file_exists($sMyFile)){
_quit('File not found.', 404);
}
diff --git a/readme.md b/readme.md
index 401c28b..96c5d36 100644
--- a/readme.md
+++ b/readme.md
@@ -21,7 +21,6 @@ GNU GPL 3.0
## Installation on server
-
### Xsentfile module
On Apache Webserver install xentfile module.
@@ -30,7 +29,6 @@ i.e. on CentOS
`yum install mod_xsendfile`
-
### Configuration of vhost
In the Apache vhost for cipkg server set the XSendFilePath - it is an absolute
@@ -56,7 +54,6 @@ Ecample snippet
</Location>
```
-
### Configuration of secret on server
if [approot]/public_html/ copy inc_config.php.dist to inc_config.php.dist.
@@ -100,15 +97,10 @@ directory.
Copy getfile.sh.cfg.dist to getfile.sh.cfg and setup values:
```text
-# ----------------------------------------------------------------------
-# defaults
-# ----------------------------------------------------------------------
-
IMLCI_PKG_SECRET=our-package-server-secret
IMLCI_URL=https://cipkg.example.com
IMLCI_PHASE=preview
IMLCI_PROJECT=myproject-id
-IMLCI_FILE=
```
Remark: using the cfg file is optional. It countains default values.
@@ -117,19 +109,55 @@ all values can be set by command line parameters.
Usage of getfile.sh:
```
SYNTAX:
+
+ getfile.sh [OPTIONS]
+
+OPTIONS:
+
-d enable debug infos
-e PHASE phase; overrides env variable IMLCI_PHASE
- -f FILE filename to get; overrides env variable IMLCI_FILE
- -o OUTFILE optional output file; default; given filename in current directory
+ -f FILE filename to get (without path); overrides env variable IMLCI_FILE
+ -o OUTFILE optional output file
-p PROJECT ci project id; overrides env variable IMLCI_PROJECT
-s SECRET override secret in IMLCI_PKG_SECRET
-u URL URL of iml ci server without trailing /; overrides env variable IMLCI_URL
+VALUES:
+
+ PHASE is a phase of the ci server; one of preview|stage|live
+ FILE is a filename without path that was created by ci server.
+ OUTFILE Output file. It can countain a path. If none is given the filename
+ will be taken from FILE and stored in current directory
+ PROJECT project id of the ci server
+ SECRET secret to access project data on package server. Your given secret
+ must match the secret on package server to get access to any url.
+
DEFAULTS:
+
You don't need to set all values by command line. Use a config to set defaults
- ./getfile.sh.cfg
+ /home/axel/data/htdocs/dev.cipkg.iml.unibe.ch/shellscripts/getfile.sh.cfg
EXAMPLES:
- getfile.sh -u https://ci.example.com -f [FILE]
+ If url, secret, project and phase are set in the config you can operate by
+ setting the filename to request.
+
+ getfile.sh -f FILE
+ downloads FILE to the current dir.
+
+ getfile.sh -f FILE -o my-own-filename.tgz
+ downloads FILE as my-own-filename.tgz
+
+ getfile.sh -f ALL
+ there is a special file ALL; it fetches all filenames by executing a directory
+ listing and then downloads all remote files with their original name
+
+ getfile.sh -f ''
+ empty file = directory listing of all your project files
+
+ getfile.sh -p ''
+ empty project = directory listing of all projects with current phase
+
+ Remark: The directory listing can be turned off on the package server and
+ results in a 403 status.
```
\ No newline at end of file
diff --git a/shellscripts/getfile.sh b/shellscripts/getfile.sh
index c3ae3b7..f7ecb33 100755
--- a/shellscripts/getfile.sh
+++ b/shellscripts/getfile.sh
@@ -21,21 +21,59 @@ bDebug=0
# ----------------------------------------------------------------------
function showhelp(){
+self=$( basename $0 )
echo "SYNTAX:
+
+ $self [OPTIONS]
+
+OPTIONS:
+
-d enable debug infos
-e PHASE phase; overrides env variable IMLCI_PHASE
- -f FILE filename to get; overrides env variable IMLCI_FILE
- -o OUTFILE optional output file; default; given filename in current directory
+ -f FILE filename to get (without path); overrides env variable IMLCI_FILE
+ -o OUTFILE optional output file
-p PROJECT ci project id; overrides env variable IMLCI_PROJECT
-s SECRET override secret in IMLCI_PKG_SECRET
-u URL URL of iml ci server without trailing /; overrides env variable IMLCI_URL
+VALUES:
+
+ PHASE is a phase of the ci server; one of preview|stage|live
+ FILE is a filename without path that was created by ci server.
+ OUTFILE Output file. It can countain a path. If none is given the filename
+ will be taken from FILE and stored in current directory
+ PROJECT project id of the ci server
+ SECRET secret to access project data on package server. Your given secret
+ must match the secret on package server to get access to any url.
+
DEFAULTS:
+
You don't need to set all values by command line. Use a config to set defaults
$0.cfg
EXAMPLES:
- `basename $0` -u https://ci.example.com -f [FILE]
+
+ If url, secret, project and phase are set in the config you can operate by
+ setting the filename to request.
+
+ $self -f FILE
+ downloads FILE to the current dir.
+
+ $self -f FILE -o my-own-filename.tgz
+ downloads FILE as my-own-filename.tgz
+
+ $self -f ALL
+ there is a special file ALL; it fetches all filenames by executing a directory
+ listing and then downloads all remote files with their original name
+
+ $self -f ''
+ empty file = directory listing of all your project files
+
+ $self -p ''
+ empty project = directory listing of all projects with current phase
+
+ Remark: The directory listing can be turned off on the package server and
+ results in a 403 status.
"
}
@@ -49,9 +87,11 @@ function makeRequest(){
# local outfile=$( mktemp )
- echo $line
- echo $apiMethod ${apiHost}${apiRequest}
- echo $line
+ if [ $bDebug = 1 ]; then
+ echo $line
+ echo $apiMethod ${apiHost}${apiRequest}
+ echo $line
+ fi
if [ ! -z "$secret" ]; then
@@ -71,7 +111,7 @@ ${apiTS}
myHash=`echo -n "$data" | openssl sha1 -hmac "${secret}" | cut -f 2 -d" "`
myHash=`echo -n "$myHash" | base64`
- curl\
+ curl \
-H "Accept: application/json" -H "Content-Type: application/json" \
-H "Date: ${apiTS}" \
-H "Authorization: demo-bash-client:${myHash}" \
@@ -81,11 +121,17 @@ ${apiTS}
${IMLCI_URL}${apiRequest}
if [ $? -eq 0 ]; then
echo Download OK.
- mv "${outfile}.downloading" "${outfile}"
- ls -l "${outfile}"
+ if [ -z "$outfile" ]; then
+ echo
+ echo ----- RESPONSE BODY:
+ cat "${outfile}.downloading"
+ rm -f "${outfile}.downloading"
+ else
+ mv "${outfile}.downloading" "${outfile}"
+ ls -l "${outfile}"
+ fi
else
echo ERROR: Download failed.
- # rm -f "${outfile}.downloading"
fi
else
curl\
@@ -112,8 +158,6 @@ if [ $# -lt 1 ]; then
fi
-
-
while getopts "de:f:o:p:s:u:" option; do
case ${option} in
d) bDebug=1 ;;
@@ -125,6 +169,8 @@ while getopts "de:f:o:p:s:u:" option; do
u) export IMLCI_URL=$OPTARG ;;
*)
echo ERROR: invalid option [${option}]
+ echo
+ showhelp
exit 2
esac
done
@@ -136,13 +182,13 @@ if [ $bDebug = 1 ]; then
echo
echo DEBUG INFOS
echo
- echo ----- content of $0.cfg
+ echo ----- defaults in $0.cfg
cat $0.cfg 2>/dev/null
echo
- echo '----- Params (override config values)'
+ echo '----- Params (override default values)'
echo $*
echo
- echo ----- used values
+ echo ----- effective values
echo "IMLCI_URL = $IMLCI_URL"
echo "IMLCI_PKG_SECRET = $IMLCI_PKG_SECRET"
echo "IMLCI_PROJECT = $IMLCI_PROJECT"
@@ -153,4 +199,28 @@ if [ $bDebug = 1 ]; then
echo
fi
-makeRequest GET "/packages/$IMLCI_PHASE/$IMLCI_PROJECT/$IMLCI_FILE" "$IMLCI_OUTFILE" "$IMLCI_PKG_SECRET"
+if [ "$IMLCI_FILE" = "ALL" ]; then
+ echo ALL files were requested ...
+ echo Get filelist ...
+ tmpfilelist=/tmp/myfilelist_gdgsdgadg
+ $0 -u "${IMLCI_URL}" \
+ -p "${IMLCI_PROJECT}" \
+ -e "${IMLCI_PHASE}" \
+ -s "${IMLCI_PKG_SECRET}" \
+ -o "${tmpfilelist}"
+
+ cat "${tmpfilelist}"
+ cat "${tmpfilelist}" | grep "^file:" | while read fileline
+ do
+ echo $line
+ myfile=$( echo $fileline | cut -f 2- -d ':' )
+ echo GET $myfile...
+ $0 -u "${IMLCI_URL}" \
+ -p "${IMLCI_PROJECT}" \
+ -e "${IMLCI_PHASE}" \
+ -s "${IMLCI_PKG_SECRET}" \
+ -f "${myfile}"
+ done
+else
+ makeRequest GET "/packages/$IMLCI_PHASE/$IMLCI_PROJECT/$IMLCI_FILE" "$IMLCI_OUTFILE" "$IMLCI_PKG_SECRET"
+fi
\ No newline at end of file
diff --git a/shellscripts/getfile.sh.cfg.dist b/shellscripts/getfile.sh.cfg.dist
index f1669cb..e649827 100644
--- a/shellscripts/getfile.sh.cfg.dist
+++ b/shellscripts/getfile.sh.cfg.dist
@@ -6,4 +6,3 @@ IMLCI_PKG_SECRET=our-package-server-secret
IMLCI_URL=https://cipkg.example.com
IMLCI_PHASE=preview
IMLCI_PROJECT=myproject-id
-IMLCI_FILE=
--
GitLab