From dd35a9f1a8eaa04373a83591a255e2cbe5bfbf4c Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Fri, 15 Mar 2024 13:29:20 +0700 Subject: [PATCH] lib: import generic helpers from upgrades --- forgejo/upgrades/test-upgrade.sh | 285 +------------------------------ {forgejo => lib}/build.sh | 2 +- lib/lib.sh | 285 +++++++++++++++++++++++++++++++ 3 files changed, 288 insertions(+), 284 deletions(-) rename {forgejo => lib}/build.sh (92%) diff --git a/forgejo/upgrades/test-upgrade.sh b/forgejo/upgrades/test-upgrade.sh index 7967454..25eea18 100755 --- a/forgejo/upgrades/test-upgrade.sh +++ b/forgejo/upgrades/test-upgrade.sh @@ -12,280 +12,18 @@ # SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SELF="${BASH_SOURCE[0]}" source $SELF_DIR/../../lib/lib.sh -# -# Forgejo releases for which a branch exists (7.0/forgejo etc.) -# -RELEASE_NUMBERS="7.0" - -PREFIX=============== -HOST_PORT=$IP:3000 STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" -DIR=/tmp/forgejo-upgrades -export DOT_FORGEJO_CURL=$DIR/forgejo-curl -export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0 -: ${FORGEJO_USER:=root} : ${FORGEJO_REPO:=fixture} -: ${FORGEJO_PASSWORD:=admin1234} -RELEASE_NUMBERS_AND_DEV="$(for r in $RELEASE_NUMBERS ; do echo -n $r $r-dev ; done)" source $SELF_DIR/fixtures.sh -function log_info() { - echo "$PREFIX $@" -} - -function dependencies() { - - if ! test -f /usr/local/bin/forgejo-curl.sh ; then - $SUDO curl --fail -sS https://code.forgejo.org/forgejo/forgejo-curl/raw/branch/main/forgejo-curl.sh -o /usr/local/bin/forgejo-curl.sh - $SUDO chmod +x /usr/local/bin/forgejo-curl.sh - fi - - if ! which curl daemon jq git-lfs > /dev/null ; then - $SUDO apt-get update -qq - $SUDO apt-get install -y -qq curl daemon git-lfs jq sqlite3 gettext-base - fi - - if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio ; then - $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc - $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/local/bin/minio - fi - if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio ; then - $SUDO chmod +x /usr/local/bin/mc - $SUDO chmod +x /usr/local/bin/minio - fi - - if ! test -f /usr/local/bin/garage > /dev/null ; then - $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage - fi - if ! test -x /usr/local/bin/garage > /dev/null ; then - $SUDO chmod +x /usr/local/bin/garage - fi -} - -function build_all() { - for dev in $RELEASE_NUMBERS ; do - log_info $dev-dev - $SELF_DIR/../build.sh $dev $DIR - done -} - -function retry() { - rm -f $DIR/wait-for.out - success=false - for delay in 1 1 5 5 15 ; do - if "$@" >> $DIR/wait-for.out 2>&1 ; then - success=true - break - fi - cat $DIR/wait-for.out - echo waiting $delay - sleep $delay - done - if test $success = false ; then - cat $DIR/wait-for.out - return 1 - fi -} - -function full_version() { - local version=$1 - local owner=$2 - - if [[ $version =~ ^[0-9]+\.[0-9]+$ ]] ; then - full_version=$(curl -sS https://codeberg.org/api/v1/repos/$owner/forgejo/releases | jq -r '.[] | .tag_name | select(startswith("v'$version'"))' | sort -r | head -1) - echo ${full_version#v} - else - echo $version - fi -} - -function download() { - local version=$1 - - if ! test -f $DIR/forgejo-$version ; then - mkdir -p $DIR - for owner in forgejo forgejo-experimental forgejo-integration ; do - full_version=$(full_version $version $owner) - if test "$full_version" = "" ; then - continue - fi - if wget -O $DIR/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then - break - fi - done - if test -s $DIR/forgejo-$version ; then - if test "$version" != "$full_version" ; then - log_info "downloaded $full_version for $version" - fi - else - echo unable to download Forgejo $version - return 1 - fi - chmod +x $DIR/forgejo-$version - fi -} - -function cleanup_logs() { - local work_path=$DIR/forgejo-work-path - - rm -f $DIR/*.log - rm -f $work_path/log/*.log -} - -function clobber() { - rm -fr /tmp/forgejo-upgrades -} - -function start_forgejo() { - local version=$1 - - download $version - local work_path=$DIR/forgejo-work-path - daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path - if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then - grep '' $DIR/*.log - grep '' $work_path/log/*.log 2> /dev/null - return 1 - fi - create_user_and_login $version -} - -function start_minio() { - mkdir -p $DIR/minio - daemon --chdir=$DIR --unsafe \ - --env="PATH=$PATH" \ - --env=MINIO_ROOT_USER=123456 \ - --env=MINIO_ROOT_PASSWORD=12345678 \ - --env=MINIO_VOLUMES=$DIR/minio \ - --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server - retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 -} - -function start_garage() { - mkdir -p $DIR/garage/{data,meta} - cat > $DIR/garage/garage.toml < $work_path/app.ini -} - -function reset_minio() { - rm -fr $DIR/minio -} - -function reset_garage() { - rm -fr $DIR/garage -} - function reset() { local config=$1 - reset_forgejo $config + reset_forgejo $SELF_DIR/$config-app.ini reset_minio reset_garage } @@ -347,25 +85,6 @@ function test_forgejo_database_version() { test "$expected_version" = "$actual_version" } -function run() { - local fun=$1 - shift - - echo Start running $fun - mkdir -p $DIR - > $DIR/$fun.out - tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" & - local pid=$! - if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then - kill $pid - cat $DIR/$fun.out - echo Failure running $fun - return 1 - fi - kill $pid - echo Success running $fun -} - source $SELF_DIR/test-upgrade-1.20-storage.sh source $SELF_DIR/test-upgrade-forgejo-database-v3.sh diff --git a/forgejo/build.sh b/lib/build.sh similarity index 92% rename from forgejo/build.sh rename to lib/build.sh index e0330e8..61f87c3 100755 --- a/forgejo/build.sh +++ b/lib/build.sh @@ -10,7 +10,7 @@ version=$1 DIR=$2 v=$(echo $version | sed -E -e 's/^([0-9]+\.[0-9]+).*/\1/') -read url ref semver < $SELF_DIR/sources/$v +read url ref semver < $SELF_DIR/../forgejo/sources/$v if ! test -d $DIR/src ; then mkdir -p $DIR/src diff --git a/lib/lib.sh b/lib/lib.sh index 4bacc59..78a9876 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -2,6 +2,8 @@ # Copyright 2024 The Forgejo Authors # SPDX-License-Identifier: MIT +LIB_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + if ${VERBOSE:-false} ; then set -ex PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' @@ -18,3 +20,286 @@ if test $(id -u) != 0 ; then fi IP=$(hostname -I | cut -f1 -d' ') + +# +# Forgejo releases for which a branch exists (7.0/forgejo etc.) +# +RELEASE_NUMBERS="7.0" + +PREFIX=============== +HOST_PORT=$IP:3000 +DIR=/tmp/forgejo-end-to-end +export DOT_FORGEJO_CURL=$DIR/forgejo-curl +export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0 +: ${FORGEJO_USER:=root} +: ${FORGEJO_PASSWORD:=admin1234} +RELEASE_NUMBERS_AND_DEV="$(for r in $RELEASE_NUMBERS ; do echo -n $r $r-dev ; done)" + +function log_info() { + echo "$PREFIX $@" +} + +function dependencies() { + + if ! test -f /usr/local/bin/forgejo-curl.sh ; then + $SUDO curl --fail -sS https://code.forgejo.org/forgejo/forgejo-curl/raw/branch/main/forgejo-curl.sh -o /usr/local/bin/forgejo-curl.sh + $SUDO chmod +x /usr/local/bin/forgejo-curl.sh + fi + + if ! which curl daemon jq git-lfs > /dev/null ; then + $SUDO apt-get update -qq + $SUDO apt-get install -y -qq curl daemon git-lfs jq sqlite3 gettext-base + fi + + if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio ; then + $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/local/bin/minio + fi + if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio ; then + $SUDO chmod +x /usr/local/bin/mc + $SUDO chmod +x /usr/local/bin/minio + fi + + if ! test -f /usr/local/bin/garage > /dev/null ; then + $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage + fi + if ! test -x /usr/local/bin/garage > /dev/null ; then + $SUDO chmod +x /usr/local/bin/garage + fi +} + +function build_all() { + for dev in $RELEASE_NUMBERS ; do + log_info $dev-dev + $LIB_DIR/build.sh $dev $DIR + done +} + +function retry() { + rm -f $DIR/wait-for.out + success=false + for delay in 1 1 5 5 15 ; do + if "$@" >> $DIR/wait-for.out 2>&1 ; then + success=true + break + fi + cat $DIR/wait-for.out + echo waiting $delay + sleep $delay + done + if test $success = false ; then + cat $DIR/wait-for.out + return 1 + fi +} + +function full_version() { + local version=$1 + local owner=$2 + + if [[ $version =~ ^[0-9]+\.[0-9]+$ ]] ; then + full_version=$(curl -sS https://codeberg.org/api/v1/repos/$owner/forgejo/releases | jq -r '.[] | .tag_name | select(startswith("v'$version'"))' | sort -r | head -1) + echo ${full_version#v} + else + echo $version + fi +} + +function download() { + local version=$1 + + if ! test -f $DIR/forgejo-$version ; then + mkdir -p $DIR + for owner in forgejo forgejo-experimental forgejo-integration ; do + full_version=$(full_version $version $owner) + if test "$full_version" = "" ; then + continue + fi + if wget -O $DIR/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then + break + fi + done + if test -s $DIR/forgejo-$version ; then + if test "$version" != "$full_version" ; then + log_info "downloaded $full_version for $version" + fi + else + echo unable to download Forgejo $version + return 1 + fi + chmod +x $DIR/forgejo-$version + fi +} + +function cleanup_logs() { + local work_path=$DIR/forgejo-work-path + + rm -f $DIR/*.log + rm -f $work_path/log/*.log +} + +function clobber() { + rm -fr /tmp/forgejo-end-to-end +} + +function start_forgejo() { + local version=$1 + + download $version + local work_path=$DIR/forgejo-work-path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then + grep '' $DIR/*.log + grep '' $work_path/log/*.log 2> /dev/null + return 1 + fi + create_user_and_login $version +} + +function start_minio() { + mkdir -p $DIR/minio + daemon --chdir=$DIR --unsafe \ + --env="PATH=$PATH" \ + --env=MINIO_ROOT_USER=123456 \ + --env=MINIO_ROOT_PASSWORD=12345678 \ + --env=MINIO_VOLUMES=$DIR/minio \ + --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server + retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 +} + +function start_garage() { + mkdir -p $DIR/garage/{data,meta} + cat > $DIR/garage/garage.toml < $work_path/app.ini +} + +function reset_minio() { + rm -fr $DIR/minio +} + +function reset_garage() { + rm -fr $DIR/garage +} + +function create_user_and_login() { + local version=$1 + + local work_path=$DIR/forgejo-work-path + local email="$FORGEJO_USER@example.com" + local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path" + if ! $cli admin user list | grep --quiet "$email" ; then + $cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email + fi + + forgejo-curl.sh logout + local scopes='--scopes ["all"]' + if echo $version | grep --quiet 1.18 ; then + scopes="" + fi + forgejo-curl.sh --user "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" $scopes login http://${HOST_PORT} +} + +function stop_daemon() { + local daemon=$1 + + if test -f $DIR/$daemon-pid ; then + local pid=$(cat $DIR/$daemon-pid) + kill -TERM $pid + pidwait $pid || true + for delay in 1 1 2 2 5 5 ; do + if ! test -f $DIR/$daemon-pid ; then + break + fi + sleep $delay + done + ! test -f $DIR/$daemon-pid + fi +} + +function stop() { + stop_daemon forgejo + stop_daemon minio + stop_daemon garage + + cleanup_logs +} + +function run() { + local fun=$1 + shift + + echo Start running $fun + mkdir -p $DIR + > $DIR/$fun.out + tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" & + local pid=$! + if ! VERBOSE=true $SELF $fun "$@" >& $DIR/$fun.out ; then + kill $pid + cat $DIR/$fun.out + echo Failure running $fun + return 1 + fi + kill $pid + echo Success running $fun +}