implemented file listing and donwload of all files

dd9b2ff7 · Hahn Axel (hahn) · c044de71 · dd9b2ff7 · dd9b2ff7 · dd9b2ff7
Commit dd9b2ff7 authored 4 years ago by Hahn Axel (hahn)
--- a/public_html/inc_config.php.dist
+++ b/public_html/inc_config.php.dist
 <?php
 return array(
+    // define a secret aka pi key
    'apikey'=>'our-package-server-secret',
-    'cutfromrequest'=>'^/packages',
+    // local directory of synched ci packages
    'packagedir'=>dirname(__DIR__).'/packages',
+    // allow directory listing when accessing a path of a package
+    'showdircontent'=>true,
 );
\ No newline at end of file
--- a/public_html/packages/index.php
+++ b/public_html/packages/index.php
 <?php
 /* ======================================================================
 * 
- * A P I   F O R   C I   S E R V E R
+ * A P I   F O R   C I   P A C K A G E   S E R V E R
 * 
 * GET  /packages/[phase]/[ID]/[filename]
 * 
@@ -10,7 +10,7 @@
 * ======================================================================
 */
-    $bDebug=true;
+    $bDebug=false;
    ini_set('display_errors', 1);
    ini_set('display_startup_errors', 1);
    error_reporting(E_ALL);
@@ -39,13 +39,24 @@
    // ---------- SPLIT URL
-    $sRelfile=preg_replace('#'.$aConfig['cutfromrequest'].'#', '', $_SERVER["REQUEST_URI"]);
+    $sRelfile=preg_replace('#^/packages#', '', $_SERVER["REQUEST_URI"]);
    _wd('$sRelfile: '.$sRelfile);  
    $sMyFile=$aConfig['packagedir'].$sRelfile;
    _wd('full path of file: '.$sMyFile);
+    if (is_dir($sMyFile)){
+        if(!$aConfig['showdircontent']){
+            _quit('Filelisting is denied by config.', 403);
+        } else {
+            foreach( array_diff(scandir($sMyFile), array('.', '..')) as $sEntry){
+                echo filetype($sMyFile.'/'.$sEntry).':'.str_replace($aConfig['packagedir'], '_', $sEntry).PHP_EOL;
+            }
+            die();
+        }
+    }
    if (!file_exists($sMyFile)){
        _quit('File not found.', 404);
    }

--- a/readme.md
+++ b/readme.md
@@ -21,7 +21,6 @@ GNU GPL 3.0
 ## Installation on server
 ### Xsentfile module
 On Apache Webserver install xentfile module.
@@ -30,7 +29,6 @@ i.e. on CentOS
 `yum install mod_xsendfile`
 ### Configuration of vhost
 In the Apache vhost for cipkg server set the XSendFilePath - it is an absolute
@@ -56,7 +54,6 @@ Ecample snippet
    </Location>
 ```
 ### Configuration of secret on server
 if [approot]/public_html/ copy inc_config.php.dist to inc_config.php.dist.
@@ -100,15 +97,10 @@ directory.
 Copy getfile.sh.cfg.dist to getfile.sh.cfg and setup values:
 ```text
-# ----------------------------------------------------------------------
-# defaults 
-# ----------------------------------------------------------------------
 IMLCI_PKG_SECRET=our-package-server-secret
 IMLCI_URL=https://cipkg.example.com
 IMLCI_PHASE=preview
 IMLCI_PROJECT=myproject-id
-IMLCI_FILE=
 ```
 Remark: using the cfg file is optional. It countains default values.
@@ -117,19 +109,55 @@ all values can be set by command line parameters.
 Usage of getfile.sh:
 ```
 SYNTAX:
+  getfile.sh [OPTIONS]
+OPTIONS:
  -d          enable debug infos
  -e PHASE    phase; overrides env variable IMLCI_PHASE
-  -f FILE     filename to get; overrides env variable IMLCI_FILE
+  -f FILE     filename to get (without path); overrides env variable IMLCI_FILE
-  -o OUTFILE  optional output file; default; given filename in current directory
+  -o OUTFILE  optional output file
  -p PROJECT  ci project id; overrides env variable IMLCI_PROJECT
  -s SECRET   override secret in IMLCI_PKG_SECRET
  -u URL      URL of iml ci server without trailing /; overrides env variable IMLCI_URL
+VALUES:
+  PHASE       is a phase of the ci server; one of preview|stage|live
+  FILE        is a filename without path that was created by ci server.
+  OUTFILE     Output file. It can countain a path. If none is given the filename
+              will be taken from FILE and stored in current directory
+  PROJECT     project id of the ci server
+  SECRET      secret to access project data on package server. Your given secret
+              must match the secret on package server to get access to any url.
 DEFAULTS:
  You don't need to set all values by command line. Use a config to set defaults
-  ./getfile.sh.cfg
+  /home/axel/data/htdocs/dev.cipkg.iml.unibe.ch/shellscripts/getfile.sh.cfg
 EXAMPLES:
-  getfile.sh -u https://ci.example.com -f [FILE]
+  If url, secret, project and phase are set in the config you can operate by
+  setting the filename to request.
+  getfile.sh -f FILE 
+    downloads FILE to the current dir.
+  getfile.sh -f FILE -o my-own-filename.tgz 
+    downloads FILE as my-own-filename.tgz
+  getfile.sh -f ALL 
+    there is a special file ALL; it fetches all filenames by executing a directory 
+    listing and then downloads all remote files with their original name
+  getfile.sh -f '' 
+    empty file = directory listing of all your project files
+  getfile.sh -p '' 
+    empty project = directory listing of all projects with current phase
+  Remark: The directory listing can be turned off on the package server and
+  results in a 403 status.
 ```
\ No newline at end of file
--- a/shellscripts/getfile.sh
+++ b/shellscripts/getfile.sh
@@ -21,21 +21,59 @@ bDebug=0
 # ----------------------------------------------------------------------
 function showhelp(){
+self=$( basename $0 )
 echo "SYNTAX:
+  $self [OPTIONS]
+OPTIONS:
  -d          enable debug infos
  -e PHASE    phase; overrides env variable IMLCI_PHASE
-  -f FILE     filename to get; overrides env variable IMLCI_FILE
+  -f FILE     filename to get (without path); overrides env variable IMLCI_FILE
-  -o OUTFILE  optional output file; default; given filename in current directory
+  -o OUTFILE  optional output file
  -p PROJECT  ci project id; overrides env variable IMLCI_PROJECT
  -s SECRET   override secret in IMLCI_PKG_SECRET
  -u URL      URL of iml ci server without trailing /; overrides env variable IMLCI_URL
+VALUES:
+  PHASE       is a phase of the ci server; one of preview|stage|live
+  FILE        is a filename without path that was created by ci server.
+  OUTFILE     Output file. It can countain a path. If none is given the filename
+              will be taken from FILE and stored in current directory
+  PROJECT     project id of the ci server
+  SECRET      secret to access project data on package server. Your given secret
+              must match the secret on package server to get access to any url.
 DEFAULTS:
  You don't need to set all values by command line. Use a config to set defaults
  $0.cfg
 EXAMPLES:
-  `basename $0` -u https://ci.example.com -f [FILE]
+  If url, secret, project and phase are set in the config you can operate by
+  setting the filename to request.
+  $self -f FILE 
+    downloads FILE to the current dir.
+  $self -f FILE -o my-own-filename.tgz 
+    downloads FILE as my-own-filename.tgz
+  $self -f ALL 
+    there is a special file ALL; it fetches all filenames by executing a directory 
+    listing and then downloads all remote files with their original name
+  $self -f '' 
+    empty file = directory listing of all your project files
+  $self -p '' 
+    empty project = directory listing of all projects with current phase
+  Remark: The directory listing can be turned off on the package server and
+  results in a 403 status.
 "
 }
@@ -49,9 +87,11 @@ function makeRequest(){
  # local outfile=$( mktemp )
-  echo $line
+  if [ $bDebug = 1 ]; then
-  echo $apiMethod ${apiHost}${apiRequest}
+    echo $line
-  echo $line
+    echo $apiMethod ${apiHost}${apiRequest}
+    echo $line
+  fi
  if [ ! -z "$secret" ]; then
@@ -71,7 +111,7 @@ ${apiTS}
    myHash=`echo -n "$data" | openssl sha1 -hmac "${secret}" | cut -f 2 -d" "`
    myHash=`echo -n "$myHash" | base64`
-    curl\
+    curl \
      -H "Accept: application/json" -H "Content-Type: application/json" \
      -H "Date: ${apiTS}" \
      -H "Authorization: demo-bash-client:${myHash}" \
@@ -81,11 +121,17 @@ ${apiTS}
      ${IMLCI_URL}${apiRequest}
    if [ $? -eq 0 ]; then
        echo Download OK.
-        mv "${outfile}.downloading" "${outfile}"
+        if [ -z "$outfile" ]; then
-        ls -l "${outfile}"
+            echo
+            echo ----- RESPONSE BODY:
+            cat "${outfile}.downloading"
+            rm -f "${outfile}.downloading" 
+        else
+            mv "${outfile}.downloading" "${outfile}"
+            ls -l "${outfile}"
+        fi
    else
        echo ERROR: Download failed.
-        # rm -f "${outfile}.downloading"
    fi
  else
    curl\
@@ -112,8 +158,6 @@ if  [ $# -lt 1 ]; then
 fi
 while getopts "de:f:o:p:s:u:" option; do 
    case ${option} in
      d) bDebug=1 ;;
@@ -125,6 +169,8 @@ while getopts "de:f:o:p:s:u:" option; do
      u) export IMLCI_URL=$OPTARG ;;
      *)
        echo ERROR: invalid option [${option}]
+        echo
+        showhelp
        exit 2
    esac
 done
@@ -136,13 +182,13 @@ if [ $bDebug = 1 ]; then
    echo
    echo DEBUG INFOS
    echo
-    echo ----- content of $0.cfg
+    echo ----- defaults in $0.cfg
    cat $0.cfg 2>/dev/null
    echo
-    echo '----- Params (override config values)'
+    echo '----- Params (override default values)'
    echo $*
    echo
-    echo ----- used values
+    echo ----- effective values
    echo "IMLCI_URL = $IMLCI_URL"
    echo "IMLCI_PKG_SECRET = $IMLCI_PKG_SECRET"
    echo "IMLCI_PROJECT = $IMLCI_PROJECT"
@@ -153,4 +199,28 @@ if [ $bDebug = 1 ]; then
    echo
 fi
-makeRequest GET "/packages/$IMLCI_PHASE/$IMLCI_PROJECT/$IMLCI_FILE" "$IMLCI_OUTFILE" "$IMLCI_PKG_SECRET"
+if [ "$IMLCI_FILE" = "ALL" ]; then
+    echo ALL files were requested ...
+    echo Get filelist ...
+    tmpfilelist=/tmp/myfilelist_gdgsdgadg
+    $0 -u "${IMLCI_URL}" \
+        -p "${IMLCI_PROJECT}" \
+        -e "${IMLCI_PHASE}" \
+        -s "${IMLCI_PKG_SECRET}" \
+        -o "${tmpfilelist}"
+    cat "${tmpfilelist}"
+    cat "${tmpfilelist}" | grep "^file:" | while read fileline
+    do
+        echo $line
+        myfile=$( echo $fileline | cut -f 2- -d ':' )
+        echo GET $myfile...
+        $0 -u "${IMLCI_URL}" \
+            -p "${IMLCI_PROJECT}" \
+            -e "${IMLCI_PHASE}" \
+            -s "${IMLCI_PKG_SECRET}" \
+            -f "${myfile}"
+    done
+else 
+    makeRequest GET "/packages/$IMLCI_PHASE/$IMLCI_PROJECT/$IMLCI_FILE" "$IMLCI_OUTFILE" "$IMLCI_PKG_SECRET"
+fi
\ No newline at end of file
--- a/shellscripts/getfile.sh.cfg.dist
+++ b/shellscripts/getfile.sh.cfg.dist
@@ -6,4 +6,3 @@ IMLCI_PKG_SECRET=our-package-server-secret
 IMLCI_URL=https://cipkg.example.com
 IMLCI_PHASE=preview
 IMLCI_PROJECT=myproject-id
-IMLCI_FILE=