Merge pull request 'include forgejo-curl.sh' (#48) from earl-warren/setup-forgejo:wip-forgejo-curl into main

Reviewed-on: https://code.forgejo.org/actions/setup-forgejo/pulls/48
Reviewed-by: dachary <dachary@noreply.code.forgejo.org>

.forgejo/workflows/forgejo-sh.yml

@@ -15,13 +15,14 @@ jobs:
       - uses: actions/checkout@v3
       - run: |
           set -x
-          ./forgejo-dependencies.sh install_docker
+          export PATH=$(pwd):$PATH
+          forgejo-dependencies.sh install_docker
           (
             echo codeberg.org/forgejo/forgejo 1.19.4-0
             echo codeberg.org/forgejo/forgejo 1.20.4-1
             echo codeberg.org/forgejo-experimental/forgejo 1.21.0-0-rc0
           ) | while read url version ; do
             echo "=========================== launching forgejo v$version =========="
-            ./forgejo.sh setup root admin1234 $url $version
+            forgejo.sh setup root admin1234 $url $version
-            ./forgejo.sh teardown
+            forgejo.sh teardown
           done

.forgejo/workflows/integration-nested.yml

@@ -7,7 +7,8 @@ jobs:
       - run: |
           set -x
           LXC_IP_PREFIX=10.0.9 ./forgejo-dependencies.sh
-          ./forgejo.sh setup root admin1234 codeberg.org/forgejo/forgejo 1.20
+          export PATH=$(pwd):$PATH
+          forgejo.sh setup root admin1234 codeberg.org/forgejo/forgejo 1.20
           #
           # Uncomment the following for a shortcut to debugging the Forgejo runner.
           # It will build the runner from a designated repository and branch instead of

.forgejo/workflows/integration.yml

@@ -24,7 +24,8 @@ jobs:
       - run: |
           set -x
           LXC_IP_PREFIX=10.0.10 ./forgejo-dependencies.sh
-          ./forgejo.sh setup root admin1234 codeberg.org/forgejo/forgejo 1.20
+          export PATH=$(pwd):$PATH
+          forgejo.sh setup root admin1234 codeberg.org/forgejo/forgejo 1.20
           #
           # Uncomment the following for a shortcut to debugging the Forgejo runner.
           # It will build the runner from a designated repository and branch instead of

README.md

@@ -10,19 +10,19 @@ The forgejo-test-helper.sh script is available to help test and debug actions.
 `forgejo=http://root:admin1234@${{ steps.forgejo.outputs.host-port }}`
 
 * `forgejo-test-helper.sh push_self_action $forgejo root myaction vTest`
-  Creates the repository `$forgejo/root/myaction` and populate it with the
+  Creates the repository `$forgejo/root/myaction` and populates it with the
   content of the repository under test, except for the `.forgejo` directory
-  (it would otherwise create an infinite recursion loop). The tag `vTest` is
+  (it would otherwise create an infinite loop). The tag `vTest` is
   set to the SHA under test.
 * `forgejo-test-helper.sh run_workflow testrepo $forgejo root testrepo myaction`
-  Create the repository `$forgejo/root/testrepo` and populate it with the
+  Creates the repository `$forgejo/root/testrepo` and populates it with the
   content of the `testrepo` directory. All occurrences of `SELF` in
   `testrepo/.forgejo/workflows/*.yml` are replaced with `$forgejo/root/myaction`.
 * `forgejo-test-helper.sh push testrepo $forgejo root testrepo`
-  Create the repository `$forgejo/root/testrepo` and populate it with the
+  Creates the repository `$forgejo/root/testrepo` and populates it with the
   content of the `testrepo` directory. The SHA of the tip of the repository
-  is in the output, starting with `sha=`.
+  is in the output, on a line starting with `sha=`.
-* `forgejo-test-helper.sh build_runner $forgejo/forgejo/runner v1.4.1`
+* `forgejo-test-helper.sh build_runner $forgejo/forgejo/runner v3.0.0`
   Builds the forgejo runner from source in `./forgejo-runner/forgejo-runner`.
   `export PATH=$(pwd)/forgejo-runner:$PATH` will ensure that calling `forgejo-runner.sh`
   will use this binary instead of downloading a released version of the runner.
@@ -31,6 +31,9 @@ The forgejo-test-helper.sh script is available to help test and debug actions.
 The combination of `push_self_action` and `run_workflow` allows to
 run Forgejo Actions workflows from `testrepo` that use the action
 under test (`myaction`) to verify it works as intended.
+
+The [forgejo-curl.sh](https://code.forgejo.org/forgejo/forgejo-curl#forgejo-curlsh)
+script is logged in the instance and ready to be used with web or api endpoints.
 <!-- action-docs-description -->
 
 It can only be run on the `self-hosted` platform, running on a host with LXC installed.

action.yml

@@ -9,19 +9,19 @@ description: |
   `forgejo=http://root:admin1234@${{ steps.forgejo.outputs.host-port }}`
 
   * `forgejo-test-helper.sh push_self_action $forgejo root myaction vTest`
-    Creates the repository `$forgejo/root/myaction` and populate it with the
+    Creates the repository `$forgejo/root/myaction` and populates it with the
     content of the repository under test, except for the `.forgejo` directory
-    (it would otherwise create an infinite recursion loop). The tag `vTest` is
+    (it would otherwise create an infinite loop). The tag `vTest` is
     set to the SHA under test.
   * `forgejo-test-helper.sh run_workflow testrepo $forgejo root testrepo myaction`
-    Create the repository `$forgejo/root/testrepo` and populate it with the
+    Creates the repository `$forgejo/root/testrepo` and populates it with the
     content of the `testrepo` directory. All occurrences of `SELF` in
     `testrepo/.forgejo/workflows/*.yml` are replaced with `$forgejo/root/myaction`.
   * `forgejo-test-helper.sh push testrepo $forgejo root testrepo`
-    Create the repository `$forgejo/root/testrepo` and populate it with the
+    Creates the repository `$forgejo/root/testrepo` and populates it with the
     content of the `testrepo` directory. The SHA of the tip of the repository
-    is in the output, starting with `sha=`.
+    is in the output, on a line starting with `sha=`.
-  * `forgejo-test-helper.sh build_runner $forgejo/forgejo/runner v1.4.1`
+  * `forgejo-test-helper.sh build_runner $forgejo/forgejo/runner v3.0.0`
     Builds the forgejo runner from source in `./forgejo-runner/forgejo-runner`.
     `export PATH=$(pwd)/forgejo-runner:$PATH` will ensure that calling `forgejo-runner.sh`
     will use this binary instead of downloading a released version of the runner.
@@ -31,6 +31,9 @@ description: |
   run Forgejo Actions workflows from `testrepo` that use the action
   under test (`myaction`) to verify it works as intended.
 
+  The [forgejo-curl.sh](https://code.forgejo.org/forgejo/forgejo-curl#forgejo-curlsh)
+  script is logged in the instance and ready to be used with web or api endpoints.
+
 inputs:
   image:
     description: 'Container image'
@@ -79,8 +82,6 @@ runs:
     - run: echo "${{ github.action_path }}" >> $GITHUB_PATH
       shell: bash
     - uses: actions/checkout@v3
-      with:
-        submodules: 'true'
     - id: forgejo
       run: |
         cd $(mktemp -d)

forgejo-curl.sh

@@ -0,0 +1,338 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+VERSION=1.0.0
+SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+VERBOSE=false
+DEBUG=false
+: ${EXIT_ON_ERROR:=true}
+: ${TOKEN_NAME:=forgejo-curl}
+: ${DOT:=$HOME/.forgejo-curl}
+
+function debug() {
+    DEBUG=true
+    set -x
+    PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
+}
+
+function verbose() {
+    VERBOSE=true
+}
+
+function log() {
+    echo "$@" >&2
+}
+
+function log_error() {
+    log "$@"
+}
+
+function log_verbose() {
+    if $VERBOSE ; then
+	log "$@"
+    fi
+}
+
+function log_info() {
+    log "$@"
+}
+
+function fatal_error() {
+    log_error "$@"
+    if $EXIT_ON_ERROR ; then
+	exit 1
+    else
+	return 1
+    fi
+}
+
+function dot_ensure() {
+    mkdir -p $DOT
+}
+
+HEADER_JSON='-H Content-Type:application/json'
+HEADER_TOKEN="-H @$DOT/header-token"
+HEADER_CSRF="-H @$DOT/header-csrf"
+
+function api() {
+    client $HEADER_TOKEN "$@"
+}
+
+function api_json() {
+    api $HEADER_JSON "$@"
+}
+
+function login_api() {
+    local user="$1" password="$2" token="$3" scopes="${4:-[\"all\"]}" url="$5"
+
+    dot_ensure
+    if test -s $DOT/token ; then
+	log_info "already logged in, ignored"
+	return
+    fi
+
+    if test -z "$token" ; then
+	log_verbose curl -sS -X DELETE --user "${user}:${password}" "${url}/api/v1/users/$user/tokens/${TOKEN_NAME}" -o /dev/null -w "%{http_code}"
+	local basic="${user:-unknown}:${password:-unknown}"
+	local status=$(curl -sS -X DELETE --user "${basic}" "${url}/api/v1/users/$user/tokens/${TOKEN_NAME}" -o /dev/null -w "%{http_code}")
+	if test "${status}" != 404 -a "${status}" != 204 ; then
+	    fatal_error permission denied, the user or password are probably incorrect, try again with --verbose
+	    return 1
+	fi
+	token=$(client $HEADER_JSON --user "${basic}" --data-raw '{"name":"'${TOKEN_NAME}'","scopes":'${scopes}'}' "${url}/api/v1/users/${user}/tokens"  | jq --raw-output .sha1)
+    fi
+    if [[ "$token" =~ ^@ ]] ; then
+	cp "${token##@}" $DOT/token
+    else
+	echo "$token" > $DOT/token
+    fi
+    ( echo -n "Authorization: token " ; cat $DOT/token ) > $DOT/header-token
+    #
+    # Verify it works
+    #
+    local status=$(api -w "%{http_code}" -o /dev/null "${url}/api/v1/user")
+    if test "${status}" != 200 ; then
+	fatal_error "${url}/api/v1/user returns status code '${status}', the token is invalid, $0 logout and login again"
+	return 1
+    fi
+}
+
+function client() {
+    log_verbose curl --cookie $DOT/cookies -f -sS "$@"
+    if ! curl --cookie $DOT/cookies -f -sS "$@" ; then
+	fatal_error
+    fi
+}
+
+function web() {
+    client $HEADER_CSRF "$@"
+}
+
+function client_update_cookies() {
+    log_verbose curl --cookie-jar $DOT/cookies --cookie $DOT/cookies -w "%{http_code}" -f -sS "$@"
+    local status=$(curl --cookie-jar $DOT/cookies --cookie $DOT/cookies -w "%{http_code}" -f -sS "$@")
+    if ! test "${status}" = 200 -o "${status}" = 303 ; then
+	fatal_error
+    fi
+}
+
+function login_client() {
+    local user="$1" password="$2" url="$3"
+
+    if test -z "$password" ; then
+	log_verbose "no password, web will not be authenticated"
+	return
+    fi
+
+    dot_ensure
+    #
+    # Get the CSRF required for login
+    #
+    client_update_cookies -o /dev/null "${url}/user/login"
+    #
+    # The login stores a cookie
+    #
+    client_update_cookies -X POST --data "user_name=${user}" --data "password=${password}" "${url}/user/login" -o $DOT/login.html
+    #
+    # Get the CSRF for reuse by other requests
+    #
+    client_update_cookies -o /dev/null "${url}/user/login"
+    local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DOT/cookies)
+    echo "X-Csrf-Token: $csrf" > $DOT/header-csrf
+    #
+    # Verify it works
+    #
+    local status=$(web -o /dev/null -w "%{http_code}" "${url}/user/settings")
+    if test "${status}" != 200 ; then
+	grep -C 1 flash-error $DOT/login.html
+	if ${DEBUG} ; then
+	    cat $DOT/login.html
+	fi
+	fatal_error login failed, the user or password are probably incorrect, try again with --verbose
+    fi
+}
+
+function login() {
+    local user="$1" password="$2" token="$3" scope="$4" url="$5"
+    login_client "${user}" "${password}" "${url}"
+    login_api "${user}" "${password}" "${token}" "${scope}" "${url}"
+}
+
+function logout() {
+    rm -f $DOT/*
+    if test -d $DOT ; then
+	rmdir $DOT
+    fi
+}
+
+
+function usage() {
+    cat >&2 <<EOF
+forgejo-curl.sh - thin curl wrapper that helps with Forgejo authentication
+
+COMMON OPTIONS
+
+  --verbose  display curl commands
+  --debug    equivalent to set -x
+
+LOGIN AND TOKEN
+
+  The API endpoints that require authentication will be given the
+  token provided with the --token argument. If not provided, it will
+  be generated (and named $TOKEN_NAME), using the --user and
+  --password credentials.
+
+  The web endpoints that require authentication will be given a cookie
+  and CSRF token created using the the --user and --password credentials
+
+  On a successful login the credentials are stored in the $DOT
+  directory to be used by the web, api, api_json commands. The logout
+  command removes the $DOT directory.
+
+  If the argument of --token starts with @, it is used as a filename
+  from which the token will be read.
+
+  forgejo-curl.sh [--verbose] [--debug]
+		  [--user <user>] [--password <password>]
+                  [--token {<token>|<@tokenfilename>}]
+		  [--scopes <scopes>] login URL
+  forgejo-curl.sh logout
+
+  OPTIONS
+
+    --user <user>                      username
+    --password <password>              password of <user>
+    --scopes <scopes>                  scopes of the token to be created (default ["all"])
+    --token {<token>|<@tokenfilename>} personal access token
+
+  EXAMPLES
+
+    forgejo-curl.sh --token ABCD \\
+		    login https://forgejo.example.com
+
+      web           is not authenticated
+      api, api_json use ABCD to authenticate
+
+    forgejo-curl.sh --token @/tmp/token \\
+		    login https://forgejo.example.com
+
+      web           is not authenticated
+      api, api_json use the content of /tmp/token to authenticate
+
+    forgejo-curl.sh --user joe --password passw0rd \\
+		    login https://forgejo.example.com
+
+      web           is authenticated
+      api, api_json use a newly generated token that belongs to user joe
+		    with scope ["all"] to authenticate
+
+    forgejo-curl.sh --user joe --password passw0rd --scopes '["write:package","write:issue"]' \\
+		    login https://forgejo.example.com
+
+      web           is authenticated
+      api, api_json use a newly generated token with write permission to packages and issues
+		    to authenticate
+
+forgejo-curl.sh [--verbose] [--debug] web [curl options]"
+
+  call curl using the CSRF token generated by the login command
+
+  EXAMPLES
+
+    forgejo-curl.sh web --form avatar=@avatar.png https://forgejo.example.com/settings/avatar
+
+      upload the file avatar.png and update the avatar of the logged in user
+
+forgejo-curl.sh [--verbose] [--debug] api|api_json [curl options]"
+
+  call curl using the token given to (or generated by) the login command. If called using
+  api_json, the Content-Type header is set to application/json.
+
+  EXAMPLES
+
+    forgejo-curl.sh api_json --data-raw '{"title":"TITLE"}' \\
+		    https://forgejo.example.com/api/v1/repos/joe/test/issues
+
+      create a new issue in the repository test
+
+    forgejo-curl.sh api --form name=image.png --form attachment=@image.png \\
+		    https://forgejo.example.com/api/v1/repos/joe/test/issues/1234/assets
+
+      add the image.png file as an attachment to the issue 1234 in the test repository
+
+forgejo-curl.sh --help - display help
+forgejo-curl.sh --version - show the version
+EOF
+}
+
+function main() {
+    local command=login user password token scopes
+
+    while true; do
+	case "$1" in
+	    --verbose)
+		shift
+		verbose
+		;;
+	    --debug)
+		shift
+		debug
+		;;
+	    --user)
+		shift
+		user="$1"
+		shift
+		;;
+	    --password)
+		shift
+		password="$1"
+		shift
+		;;
+	    --token)
+		shift
+		token="$1"
+		shift
+		;;
+	    --scopes)
+		shift
+		scopes="$1"
+		shift
+		;;
+	    login)
+		shift
+		login "$user" "$password" "$token" "$scopes" "$1"
+		return 0
+		;;
+	    logout)
+		shift
+		logout
+		return 0
+		;;
+	    web)
+		shift
+		web "$@"
+		return 0
+		;;
+	    api)
+		shift
+		api "$@"
+		return 0
+		;;
+	    api_json)
+		shift
+		api_json "$@"
+		return 0
+		;;
+	    --version)
+		echo "forgejo-curl.sh version $VERSION"
+		return 0
+		;;
+	    --help|*)
+		usage
+		return 1
+		;;
+	esac
+    done
+}
+
+${MAIN:-main} "${@}"

forgejo-test-helper.sh

@@ -42,25 +42,12 @@ function build_runner() {
     forgejo-runner --version
 }
 
-function api() {
-    method=$1
-    shift
-    url=$1
-    shift
-    path=$1
-    shift
-    token=$1
-    shift
-
-    curl --fail -X $method -sS -H "Content-Type: application/json" -H "Authorization: token $token" "$@" $url/api/v1/$path
-}
-
 function check_status() {
     local url="$1"
     local repo="$2"
     local sha="$3"
 
-    local state=$(api GET $url repos/$repo/commits/$sha/status | jq --raw-output .state)
+    local state=$(forgejo-curl.sh api_json $url/api/v1/repos/$repo/commits/$sha/status | jq --raw-output .state)
     echo $state
     test "$state" != "" -a "$state" != "pending" -a "$state" != "running" -a "$state" != "null"
 }
@@ -79,7 +66,7 @@ function wait_success() {
     done
     if ! test "$(check_status "$url" "$repo" "$sha")" = "success" ; then
 	test "$FORGEJO_RUNNER_LOGS" && cat $FORGEJO_RUNNER_LOGS
-	api GET $url repos/$repo/commits/$sha/status | jq .
+	forgejo-curl.sh api_json $url/api/v1/repos/$repo/commits/$sha/status | jq .
 	return 1
     fi
 }

forgejo.sh

@@ -3,24 +3,25 @@
 
 set -e
 
+DIR=/tmp
 if ${VERBOSE:-false}; then set -x; fi
 : ${CONTAINER:=forgejo}
 
-function wait_for() {
+function retry() {
-    rm -f /tmp/setup-forgejo.out
+    rm -f $DIR/retry.out
     success=false
-    for delay in 1 1 5 5 15 15 15 30 30 30 30 ; do
+    for delay in 1 1 5 5 15 30 ; do
-	if "$@" >> /tmp/setup-forgejo.out 2>&1 ; then
+        if "$@" >> $DIR/retry.out 2>&1 ; then
-	    success=true
+            success=true
-	    break
+            break
-	fi
+        fi
-	cat /tmp/setup-forgejo.out
+        cat $DIR/retry.out
-	echo waiting $delay
+        echo waiting $delay
-	sleep $delay
+        sleep $delay
     done
     if test $success = false ; then
-	cat /tmp/setup-forgejo.out
+        cat $DIR/retry.out
-	return 1
+        return 1
     fi
 }
 
@@ -61,9 +62,16 @@ function setup() {
     if docker exec --user 1000 ${CONTAINER} forgejo admin user list --admin | grep "$user" ; then
 	docker exec --user 1000 ${CONTAINER} forgejo admin user change-password --username "$user" --password "$password"
     else
-	wait_for docker exec --user 1000 ${CONTAINER} forgejo admin user create --admin --username "$user" --password "$password" --email "$user@example.com"
+	retry docker exec --user 1000 ${CONTAINER} forgejo admin user create --admin --username "$user" --password "$password" --email "$user@example.com"
     fi
+    #
+    # The 'sudo' scope was removed in Forgejo v1.20 and is ignored
+    #
     docker exec --user 1000 ${CONTAINER} forgejo admin user generate-access-token -u $user --raw --scopes 'all,sudo' > forgejo-token
+    retry forgejo-curl.sh --user "$user" --password "$password" --token @forgejo-token login http://$(cat forgejo-ip):3000
+    #
+    # Redundant with forgejo-curl.sh, kept around for backward compatibility 09/2023
+    #
     ( echo -n 'Authorization: token ' ; cat forgejo-token ) > forgejo-header
     ( echo "#!/bin/sh" ; echo 'curl -sS -H "Content-Type: application/json" -H @'$(pwd)/forgejo-header' "$@"' ) > forgejo-api && chmod +x forgejo-api
 }