#!/bin/bash # ====================================================================== # # REST API CLIENT USING CURL # # REQUIREMENTS # - Bash (Linux or MS Windows i.e with Cygwin) # - curl # - sha1sum (optional; for export functionality with AUTOFILE only) # ---------------------------------------------------------------------- # License: GPL 3.0 # Source: <https://git-repo.iml.unibe.ch/iml-open-source/bash-rest-api-client> # Docs: <https://os-docs.iml.unibe.ch/bash-rest-api-client/> # ---------------------------------------------------------------------- # (1) source this script # (2) enter "http.help" to get a list of available commands # ---------------------------------------------------------------------- # 2020-02-07 v0.2 axel.hahn@iml.unibe.ch BETABETA # 2020-02-12 v0.4 axel.hahn@iml.unibe.ch Caching # 2020-03-02 v0.5 axel.hahn@iml.unibe.ch a few more response check functions # 2021-01-21 v0.6 axel.hahn@iml.unibe.ch add Content-type in request header # 2022-01-11 v0.7 axel.hahn@iml.unibe.ch fixes using shellcheck # 2024-10-09 v0.8 axel.hahn@unibe.ch add setAuthorization; customize accept header, add custom request headers # 2024-10-10 v0.9 axel.hahn@unibe.ch update docs # 2024-10-23 v0.10 axel.hahn@unibe.ch update help # 2024-11-20 v0.11 axel.hahn@unibe.ch no insecure requests by default; add setInsecure, setCA, addCurlparam # ====================================================================== http_cfg__about="Bash REST API client v0.11" typeset -i http_cfg__debug=0 typeset -i http_cfg__cacheTtl=0 http_cfg__cacheDir=/var/tmp/http-cache http_cfg__UA="${http_cfg__about}" http_cfg__prjurl="https://git-repo.iml.unibe.ch/iml-open-source/bash-rest-api-client" http_cfg__docsurl="https://os-docs.iml.unibe.ch/bash-rest-api-client/" # --- curl meta infos to collect # see variables in man curl --write-out param curlMeta="\ http_code \ http_connect \ local_ip \ local_port \ num_connects \ num_redirects \ redirect_url \ remote_ip \ remote_port \ size_download \ size_header \ size_request \ size_upload \ speed_download \ speed_upload \ ssl_verify_result \ time_appconnect \ time_connect \ time_namelookup \ time_pretransfer \ time_redirect \ time_starttransfer \ time_total \ url_effective \ " # ---------------------------------------------------------------------- # # functions # # ---------------------------------------------------------------------- # ...................................................................... # # Write a debug message to STDERR # Do no not change the prefix - is is read in inc_functions # # params strings output message # function http._wd(){ if [ $http_cfg__debug -gt 0 ]; then echo -e "\e[33m# RESTAPI::DEBUG $*\e[0m" >&2 fi } # ...................................................................... # # Write an error message to STDERR # Do no not change the prefix - is is read in inc_functions # # params strings output message # function http._we(){ echo -e "\e[31m# RESTAPI::ERROR $*\e[0m" >&2 } function http(){ cat <<EOH $http_cfg__about A REST API Client with curl Enter http.help to show all commands. EOH # $0 is not the current file if we source a script # grep "function http.[a-z]" $0 | sort } # ...................................................................... # # Initialize the client # # Initialize the client for a new request. Call this before any other # function. It will reset all variables. # function http.init(){ http._wd "${FUNCNAME[0]}()" which curl >/dev/null || http.quit # request vars http_req__auth= http_req__authorization= http_req__accept="application/json" http_req__headers=() http_req__curlparams=() http_req__body= http_req__method=GET http_req__url= http_req__fullurl= http_req__docs= http_req__cafile= http_req__insecure= http_req__dataprefix="RESTAPICLIENTMETADATA_$(date +%s)_$$" local writevar= for myvar in $curlMeta do writevar="${writevar}|${myvar}:%{${myvar}}" done http_curl__writeout="\\n${http_req__dataprefix}${writevar}\\n" # cache http_cfg__cacheTtl=0 http_cfg__cacheFile= # response http_resp__all= http_resp__neutral= mkdir ${http_cfg__cacheDir} 2>/dev/null chmod 777 ${http_cfg__cacheDir} 2>/dev/null } # ...................................................................... # # Execute the request # # param string optional: method; GET|POST|PUT|DELETE|... # param string optional: full url # param string optional: request body # # description: # # This function does the following: # # 1. Check if to use a cache # 2. If not cached, make the request # 3. If cached, use the cached result # function http.makeRequest(){ http._wd "${FUNCNAME[0]}()" # --- handle optional prams if [ $# -ne 0 ]; then if echo "$1" | grep -q "^[A-Z]*$"; then http.setMethod "$1" shift 1 fi http.setUrl "$1" http.setBody "$2" fi # test -z "$1" || http.setFullUrl "$1" # --- detect caching local useCache=0 local makeRequest=1 if [ $http_cfg__cacheTtl -gt 0 ] && [ "${http_req__method}" = "GET" ]; then useCache=1 test -z "${http_cfg__cacheFile}" && http_cfg__cacheFile=$(http._genOutfilename "${http_cfg__cacheDir}/AUTOFILE") if [ -f "${http_cfg__cacheFile}" ]; then http.responseImport "${http_cfg__cacheFile}" local iAge; typeset -i iAge iAge=$(http.getRequestAge) http._wd "INFO: Age of cache is $iAge sec - vs TTL $http_cfg__cacheTtl sec - file $http_cfg__cacheFile" if [ $iAge -gt 0 ] && [ $iAge -lt $http_cfg__cacheTtl ]; then http._wd "INFO: Using cache" makeRequest=0 else http._wd "INFO: Cache file will be updated after making the request" rm -f "${http_cfg__cacheFile}" 2>/dev/null fi fi fi # --- make the request if [ $makeRequest -eq 1 ]; then # build curl parameters local curl_params=( -s -i "${http_req__fullurl}" -X "${http_req__method}" -A "${http_cfg__UA}" ) test -n "$http_req__auth" && curl_params+=( -u "$http_req__auth" ) test -n "$http_req__authorization" && curl_params+=( -H "Authorization: $http_req__authorization" ) test -n "$http_req__accept" && curl_params+=( -H "Accept: $http_req__accept" ) test -n "$http_req__body" && curl_params+=( -d "${http_req__body}" ) test -n "$http_req__insecure" && curl_params+=( -k ) test -n "$http_req__cafile" && curl_params+=( --cacert "$http_req__cafile" ) curl_params+=( "${http_req__headers[@]}" ) curl_params+=( "${http_req__curlparams[@]}" ) http._wd "${FUNCNAME[0]}($1) ${http_req__method} ${http_req__fullurl}" http._wd "${FUNCNAME[0]}($1) ${curl_params[@]}" curl_params+=( -w "${http_curl__writeout}" ) http_resp__all=$( curl "${curl_params[@]}") || http.quit http._wd "OK - Curl finished the http request ... processing data" http_resp__neutral=$(http._fetchAllAndReformat) if [ $useCache -eq 1 ]; then http._wd "INFO: writing cache ..." http.responseExport "${http_cfg__cacheFile}" fi fi http._wd "Request function finished; Code $(http.getStatuscode)" } # ...................................................................... # # Show error message with last return code and quit with this exitcode # # This function is used to quit the script with a meaningful error message # and the exit code of the last command. # # The message is printed to STDERR and contains the return code. # If a documentation URL is known, it is printed as a hint. # # The exit code is the one of the last command. # To prevent the script from exiting when this function is called from a # sourced file, the exit is commented out. # # no params # function http.quit(){ rc=$? http._wd "${FUNCNAME[0]}()" echo >&2 echo -e "\e[31m# ERROR: command FAILED with rc $rc. \e[0m" >&2 if [ ! -z "${RestApiDocs}" ]; then echo "HINT: see ${RestApiDocs}" >&2 fi # dont make exit in a sourced file # exit $rc } # ...................................................................... # # Load a config file # # This function is marked as deprecated. # It will be removed in the future. # # param string config file name # # Sourcing that file will set the following vars # - RestApiUser # - RestApiPassword # - RestApiBaseUrl # - RestApiDocs # # The function will then set the "internal" vars # - http_req__auth # - http_req__fullurl # - http_req__docs # function http.loadcfg(){ http._wd "${FUNCNAME[0]}($1) !!! DEPRECATED !!!" # reset expected vars from config RestApiUser= RestApiPassword= RestApiBaseUrl= RestApiDocs= # source config file . "${1}" || http.quit # set "internal" vars if [ -z "$RestApiPassword" ]; then http.setAuth "$RestApiUser:$RestApiPassword" else http.setAuth fi http.setBaseUrl "${RestApiBaseUrl}" http.setDocs "${RestApiDocs}" } # ====================================================================== # GETTER # ====================================================================== # ...................................................................... # # Get the response header or response body # # param string what to return; one of header|body # # Return the response header or the response body. The output is the same # as that of http.getResponseHeader or http.getResponse. The difference is # the implementation # function http._fetchResponseHeaderOrBody(){ http._wd "${FUNCNAME[0]}($1)" local isheader=true # keep leading spaces IFS='' echo "${http_resp__all}" | grep -v "${http_req__dataprefix}" | while read -r line; do if $isheader; then if [[ $line = $'\r' ]]; then isheader=false else test "$1" = "header" && echo $line fi else # body="$body"$'\n'"$line" test "$1" = "body" && echo $line fi done } # ...................................................................... # # Get the response data # # Return the curl meta infos like http_code, http_connect, local_ip, ... # as key/ value pairs. Each line is a single key value pair. # # The data is extracted from the response header. The format is: # ${http_req__dataprefix}|key|value|key|value|... # function http._fetchResponseData(){ http._wd "${FUNCNAME[0]}()" echo "${http_resp__all}" | sed "s#${http_req__dataprefix}#\n${http_req__dataprefix}#" | grep "${http_req__dataprefix}" | tail -1 | cut -f 2- -d "|" | sed "s#|#\n#g" | grep -v "${http_req__dataprefix}" | while read -r line; do echo "$line" done } # ...................................................................... # # Generate the dump with request and response function http._fetchAllAndReformat(){ http._wd "${FUNCNAME[0]}()" IFS='' line="#------------------------------------------------------------" echo "#_META_|about:$http_cfg__about" echo "#_META_|host:$(hostname -f)" echo $line echo "#_REQUEST_|fullurl:$http_req__fullurl" echo "#_REQUEST_|method:$http_req__method" echo "#_REQUEST_|time:$(date)" echo "#_REQUEST_|timestamp:$(date +%s)" echo "#_REQUEST_|auth:$(echo $http_req__auth | sed 's#:.*#:xxxxxxxx#')" echo "#_REQUEST_|body:$http_req__body" echo "#_REQUEST_|baseurl:$http_req__baseurl" echo "#_REQUEST_|url:$http_req__url" echo "#_REQUEST_|docs:$http_req__docs" echo $line http._fetchResponseHeaderOrBody header | sed "s,^,#_HEADER_|,g" echo $line http._fetchResponseData | sed "s,^,#_DATA_|,g" echo $line http._fetchResponseHeaderOrBody body | sed "s,^,#_BODY_|,g" echo $line END } # ...................................................................... # # Get a section from dump data # # param string what to return; one of HEADER|DATA|BODY # # returns string the requested part of the response # function http._getFilteredResponse(){ http._wd "${FUNCNAME[0]}($1)" echo "${http_resp__neutral}" | grep "^#_${1}_|" | cut -f 2- -d "|" } # ---------- PUBLIC REQUEST GETTER # ...................................................................... # # Get timestamp of the response as a Unix timestamp. # # no param # # returns string the timestamp of the response # function http.getRequestTs(){ http._wd "${FUNCNAME[0]}()" http._getFilteredResponse REQUEST | grep "^timestamp" | cut -f 2 -d ":" } # ...................................................................... # # Get age of the response in sec. # It is especially useful after responseImport # # no param # # returns integer age of the response in sec # function http.getRequestAge(){ http._wd "${FUNCNAME[0]}()" local iAge; typeset -i iAge local iTs; typeset -i iTs iTs=$( http.getRequestTs ) iAge=$( date +%s )-${iTs} echo "$iAge" } # ---------- PUBLIC RESPONSE GETTER # ...................................................................... # # Get response body # # no param # # returns string the response body # function http.getResponse(){ http._wd "${FUNCNAME[0]}()" http._getFilteredResponse BODY } # ...................................................................... # # Get curl data of this request with status, transferred bytes, speed, ... # # no param # # returns string the response data # function http.getResponseData(){ http._wd "${FUNCNAME[0]}()" http._getFilteredResponse DATA } # ...................................................................... # # Get response header # # no param # # returns string the response header # function http.getResponseHeader(){ http._wd "${FUNCNAME[0]}()" http._getFilteredResponse HEADER } # ...................................................................... # # Get raw response (not available after import) # # no params # # no param # function http.getResponseRaw(){ http._wd "${FUNCNAME[0]}()" echo "${http_resp__all}" } # ...................................................................... # # Get Http status as string OK|Redirect|Error # # no params # function http.getStatus(){ http._wd "${FUNCNAME[0]}($1)" http.isOk >/dev/null && echo OK http.isRedirect >/dev/null && echo Redirect http.isError >/dev/null && echo Error } # ...................................................................... # # Get Http status code of the request as 3 digit number # # no params # function http.getStatuscode(){ http._wd "${FUNCNAME[0]}()" http.getResponseData | grep "^http_code:" | cut -f 2 -d ":" } # ...................................................................... # # Check: was response a 2xx status code? # output is a statuscode if it matches ... or empty # Additionally you can verify the return code # # $? -eq 0 means YES # $? -ne 0 means NO # # no params # function http.isOk(){ http._wd "${FUNCNAME[0]}()" http.getStatuscode | grep '2[0-9][0-9]' } # ...................................................................... # # Was the repsonse a redirect? # # no params # function http.isRedirect(){ http._wd "${FUNCNAME[0]}()" http.getStatuscode | grep '3[0-9][0-9]' } # ...................................................................... # # Was the repsonse a client error (4xx or 5xx) # # no params # function http.isError(){ http._wd "${FUNCNAME[0]}()" http.getStatuscode | grep '[45][0-9][0-9]' } # ...................................................................... # # Was the repsonse a client error (4xx) # # no params # function http.isClientError(){ http._wd "${FUNCNAME[0]}()" http.getStatuscode | grep '4[0-9][0-9]' } # ...................................................................... # # Was the repsonse a client error (5xx) # # no params # function http.isServerError(){ http._wd "${FUNCNAME[0]}()" http.getStatuscode | grep '5[0-9][0-9]' } # ...................................................................... # # Dump information about request and response # # no params # function http.dump(){ http._wd "${FUNCNAME[0]}()" http.responseExport } # ====================================================================== # Import/ Export # ====================================================================== # ...................................................................... # # Helper to replace "AUTOFILE" with something uniq using full url # # param string import or export filename # function http._genOutfilename(){ http._wd "${FUNCNAME[0]}($1)" if echo "$1" | grep -q "AUTOFILE"; then echo "$1" else local sum sum=$(echo ${http_req__fullurl} | sha1sum ) local autofile autofile=$(echo "${sum}__${http_req__fullurl}" | sed "s#[^a-z0-9]#_#g") echo "$1" | sed "s#AUTOFILE#${autofile}#" fi } # ...................................................................... # # Export response to a file # # param string optional: custom filename # function http.responseExport(){ http._wd "${FUNCNAME[0]}($1)" if [ -z "$1" ]; then echo "${http_resp__neutral}" else local outfile outfile=$(http._genOutfilename "$1") http._wd "${FUNCNAME[0]}($1) writing to outfile $outfile" echo "${http_resp__neutral}" >"$outfile" fi } # ...................................................................... # # Import a former response from a file # # param string filename with stored response # function http.responseImport(){ http._wd "${FUNCNAME[0]}($1)" local infile infile=$(http._genOutfilename "$1") if [ -r "${infile}" ]; then if grep -q "^#_META_|about:$http_cfg__about" "${infile}"; then http_resp__neutral=$(cat "${infile}") else echo "ERROR: Ooops [${infile}] does not seem to be an export dump." http.quit fi else echo "ERROR: Ooops the file [${infile}] is not readable." http.quit fi } # ...................................................................... # # Delete an exported file; this is especially useful if you use # AUTOFILE functionality # # param string filename with stored response # function http.responseDelete(){ http._wd "${FUNCNAME[0]}($1)" local infile infile=$(http._genOutfilename "$1") if [ -r "${infile}" ]; then if grep -q "^#_META_|about:$http_cfg__about" "${infile}"; then if rm -f "${infile}"; then http._wd "OK, ${infile} was deleted." else http._wd "ERROR: unable to delete existing ${infile}. Check permissions." fi else http._wd "SKIP: ${infile} is not an export file." fi else http._wd "SKIP: ${infile} is not readable." fi } # ====================================================================== # SETTER # ====================================================================== # ...................................................................... # # Add a line to the request header # # param string line to add, eg "Connection: keep-alive" # function http.addHeader(){ http._wd "${FUNCNAME[0]}($1)" http_req__headers+=( -H "$1") } # ...................................................................... # # Add an additional curl parameter # # param string line to add, eg "Connection: keep-alive" # function http.addCurlparam(){ http._wd "${FUNCNAME[0]}($1)" http_req__curlparams+=( "$1") } # ...................................................................... # # set Accept request header and override default # # param string accept header value, eg text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 # function http.setAccept(){ http._wd "${FUNCNAME[0]}($1)" if [ -z "$1" ]; then http_req__accept= else http_req__accept="$1" fi } # ...................................................................... # # Set basic authentication # Without given parameter, authentication is removed # # param string optional: USER:PASSWORD # function http.setAuth(){ http._wd "${FUNCNAME[0]}($1)" if [ -z "$1" ]; then http_req__auth= else http_req__auth="$1" fi } # ...................................................................... # # Set authentication via Athorization header # Without given parameter, authorization is removed # # param string optional: type, eg. Basic|Bearer|Negotiate # param string optional: token or encoded user + password # function http.setAuthorization(){ http._wd "${FUNCNAME[0]}($1 $2)" if [ -z "$1" ]; then http_req__authorization= else http_req__authorization="${1} ${2}" fi } # ...................................................................... # # Set body to send for PUTs and POSTs # # param string body # function http.setBody(){ http._wd "${FUNCNAME[0]}($1)" http_req__body=$1 } # ...................................................................... # # Set a base url of an API # Remark: Then use http.setUrl to complet the url to request # # param string url # function http.setBaseUrl(){ http._wd "${FUNCNAME[0]}($1)" http_req__baseurl=$1 http.setFullUrl "" } # ...................................................................... # # set and unset CA cert file to use # # param string optional: filename to use; no value to disable cafile # function http.setCA(){ http._wd "${FUNCNAME[0]}($1)" http_req__cafile="$1" } # ...................................................................... # # Enable or disable debug mode # # param integer 0|1 # function http.setDebug(){ http._wd "${FUNCNAME[0]}($1)" http_cfg__debug=$1 } function http.setDocs(){ http._wd "${FUNCNAME[0]}($1)" http_req__docs=$1 } # ...................................................................... # # Allow and disallow insecure connections # # param string optional: 1 to enable insecure flag; no value to disable insecure flag # function http.setInsecure(){ http._wd "${FUNCNAME[0]}($1)" http_req__insecure="$1" } # ...................................................................... # # Set the method to use; GET|POST|PUT|DELETE|... # # param string name of method # function http.setMethod(){ http._wd "${FUNCNAME[0]}($1)" http_req__method=$1 } # ...................................................................... # # Set a full url to request # # param string optional: url # function http.setFullUrl(){ http._wd "${FUNCNAME[0]}($1)" if [ -z "$1" ]; then http_req__fullurl=${http_req__baseurl}${http_req__url} else http_req__fullurl=$1 fi } # ...................................................................... # # Complete the base url # # param string url part behind base url # function http.setUrl(){ http._wd "${FUNCNAME[0]}($1)" http_req__url=$1 http.setFullUrl } # ----- caching # ...................................................................... # # Set cache ttl in seconds # # param integer ttl in seconds # function http.setCacheTtl(){ http._wd "${FUNCNAME[0]}($1)" http_cfg__cacheTtl=$1 } # ...................................................................... # # Set cache file # # param string filename # function http.setCacheFile(){ http._wd "${FUNCNAME[0]}($1)" http_cfg__cacheFile="$1" } # ...................................................................... # # Flush the cache # # no params # function http.flushCache(){ http._wd "${FUNCNAME[0]}($1)" rm -f ${http_cfg__cacheDir}/* } # ...................................................................... # # show a help text # # no params # function http.help(){ cat <<EOH $http_cfg__about This is a bash solution to script REST API calls. Source: <$http_cfg__prjurl> Docs: <$http_cfg__docsurl> License: GNU GPL 3 INSTRUCTION: - Source the file once - Then you can run functions starting with "http." http.init Start a new request. It resets internal vars of the last request (if there was one). http.setDebug 0|1 Enable or disable debugging infos during processing. It is written to STDERR. - initialize a request setAccept "<ACCEPTHEADER>" Set authentication with user and password for basic auth Default: $http_req__accept setAuth "<USER>:<PASSWORD>" Set authentication with user and password for basic auth Without given parameter, authentication is removed setAuthorization "<TYPE>" "<TOKEN|HASH>" Set authentication with Authorization header. As TYPE you can use Basic|Bearer|Negotiate|... 2nd param is the token or hashed user+password Without given parameter, authorization is removed http.setBody "<DATA>" set a body for POST/ PUT requests. http.setBaseUrl "<URL>" Set a base url to an api. renmark: Use http.setUrl to built a complete url. http.setCA "<FILE>" Set CA file to verify the server certificate. Default: [empty] = use system defaults Without parameter the cafile is removed http.setDocs "<URL>" Set a docs url. If set it will be shown as additional hint when a request fails. http.setInsecure 1 Set insecure flag by giving any non empty value. Default: [empty] = secure requests Without parameter the insecure flag is removed http.setMethod "<METHOD>" Set a http method. Use an uppercase string for GET|POST|PUT|DELETE|... http.setFullUrl "<URL>" Set a complete url for a request. http.setUrl "<REQUEST?QUERY>" Set a relative url for a request. This requires to use http.setBaseUrl before. http.addHeader "<HEADER_LINE>" Add a header line to the request. This command can be repeated multiple times to add multiple headers. http.addCurlparam "<CURL_PARAMETER>" Add any missing parameter for curl requestst. This command can be repeated multiple times to add multiple prameters. You also can add multiple parameters with one command. - caching functions http.setCacheTtl <SECONDS> Enable caching with values > 0 Remark: only GET requests will be cached. Default: 0 (no caching) http.setCacheFile "<FILENAME>" Set a file where to read/ store a request Default: empty; autogenerated file below $http_cfg__cacheDir http.flushCache Delete all files in $http_cfg__cacheDir - make the request http.makeRequest [[<METHOD>] ["<URL>"] ["<BODY>"]] The parameters are optional. Without parameter the request will be started with given data in http.set* functions described above. If minimum one param is given then they are handled: METHOD optional: set a method (must be uppercase) - see http.setMethod URL set a relative url - see http.setUrl BODY optional: set a body - see http.setBody The request will be skipped and uses a cached content if ... - METHOD is GET - http.setCacheTtl set a value > 0 - the cache file exists and is younger than the given TTL - handle response http.getResponse Get the Response Body http.getResponseData Get Meta infos from curl http.getResponseHeader Get The http reponse header - check http status code http.getStatus Get the http status as string Ok|Redirect|Error http.getStatuscode Get the http status code of a request as 3 digit integer http.isOk Check if the http response code is a 2xx http.isRedirect Check if the http response code is a 3xx http.isError Check if the http response code is a 4xx or 5xx http.isClientError Check if the http response code is a 4xx http.isServerError Check if the http response code is a 5xx http.getRequestAge Get the age of the request in seconds. Remark: This function is useful after an import see http.responseImport. http.getRequestTs Get the Unix timestamp of the request - import/ export http.responseExport ["<FILE>"] dump the response data Without parameter it is written on STDOUT. You can set a filename to write it to a file. The filename can contain "AUTOFILE" this string will be replaced with a uniq string. (requires sha1sum and a set url) Example: http.makeRequest "https://example.com/api/" http.responseExport /tmp/something_AUTOFILE_.txt http.responseImport "<FILE>" Import an export file. To use the AUTOFILE mechanism from export set the url first. Example: http.setFullUrl "https://example.com/api/" http.responseImport /tmp/something_AUTOFILE_.txt http.responseDelete "<FILE>" Delete a file after http.responseExport. It is useful if you use the AUTOFILE mechanism. EOH } # ---------------------------------------------------------------------- # # main # # ---------------------------------------------------------------------- http.init # ----------------------------------------------------------------------