From bf689a247842c8683d7302afdfd460b0e3600c0f Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Sun, 17 Mar 2024 11:58:02 +0100 Subject: [PATCH 1/5] refactor end-to-end.yml workflow to use a top-level end-to-end.sh --- .forgejo/prepare-end-to-end/action.yml | 19 +++++ .forgejo/workflows/end-to-end.yml | 70 +++++++++++++++++++ .forgejo/workflows/upgrade.yml | 42 ----------- README.md | 4 +- end-to-end.sh | 21 ++++++ forgejo/upgrades/test-upgrade-1.20-storage.sh | 6 +- forgejo/upgrades/test-upgrade.sh | 17 ++--- lib/build.sh | 13 ++-- lib/lib.sh | 24 ++++--- 9 files changed, 142 insertions(+), 74 deletions(-) create mode 100644 .forgejo/prepare-end-to-end/action.yml create mode 100644 .forgejo/workflows/end-to-end.yml delete mode 100644 .forgejo/workflows/upgrade.yml create mode 100755 end-to-end.sh diff --git a/.forgejo/prepare-end-to-end/action.yml b/.forgejo/prepare-end-to-end/action.yml new file mode 100644 index 0000000..d94a09a --- /dev/null +++ b/.forgejo/prepare-end-to-end/action.yml @@ -0,0 +1,19 @@ +runs: + using: "composite" + steps: + - uses: https://code.forgejo.org/actions/setup-forgejo@v2 + with: + install-only: true + - run: forgejo-binary.sh ensure_user forgejo + - uses: actions/download-artifact@v3 + with: + name: forgejo-dev + path: /srv/forgejo-binaries + - name: chown/chmod /srv/forgejo-binaries + run: | + chmod +x /srv/forgejo-binaries/* + chown -R forgejo /srv/forgejo-binaries + - run: | + script=$(pwd)/end-to-end.sh + $script run dependencies + $script clobber diff --git a/.forgejo/workflows/end-to-end.yml b/.forgejo/workflows/end-to-end.yml new file mode 100644 index 0000000..4ad4303 --- /dev/null +++ b/.forgejo/workflows/end-to-end.yml @@ -0,0 +1,70 @@ +on: + pull_request: + push: + branches: + - 'main' + +jobs: + build: + runs-on: docker + container: + image: 'docker.io/node:20-bookworm' + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.21" + - name: lib/build.sh + run: | + set -x + + # + # SQLite needs gcc to be compiled + # + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get -q install -y -qq build-essential + + d=/tmp/forgejo-binaries + mkdir $d /tmp/forgejo-upload + + for version in $(ls forgejo/sources) ; do + forgejo=$d/forgejo-$version-dev + lib/build.sh $version $d + $forgejo --version + mv $forgejo /tmp/forgejo-upload + done + - uses: actions/upload-artifact@v3 + with: + name: forgejo-dev + path: /tmp/forgejo-upload + + packages: + needs: [build] + runs-on: lxc-bookworm + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: ./.forgejo/prepare-end-to-end + - run: su forgejo -c "./end-to-end.sh test_packages" + + upgrade: + needs: [build] + runs-on: lxc-bookworm + steps: + - name: cache S3 binaries + id: S3 + uses: https://code.forgejo.org/actions/cache@v3 + with: + path: | + /usr/local/bin/minio + /usr/local/bin/mc + /usr/local/bin/garage + key: S3 + + - name: skip if S3 cache hit + if: steps.S3.outputs.cache-hit != 'true' + run: echo no hit + + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: ./.forgejo/prepare-end-to-end + - run: su forgejo -c "./end-to-end.sh test_upgrades" diff --git a/.forgejo/workflows/upgrade.yml b/.forgejo/workflows/upgrade.yml deleted file mode 100644 index 5e8f7c5..0000000 --- a/.forgejo/workflows/upgrade.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: upgrade - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - upgrade: - runs-on: lxc-bookworm - steps: - - name: cache S3 binaries - id: S3 - uses: https://code.forgejo.org/actions/cache@v3 - with: - path: | - /usr/local/bin/minio - /usr/local/bin/mc - /usr/local/bin/garage - key: S3 - - - name: skip if S3 cache hit - if: steps.S3.outputs.cache-hit != 'true' - run: echo no hit - - - uses: https://code.forgejo.org/actions/checkout@v4 - - uses: https://code.forgejo.org/actions/setup-forgejo@v2 - with: - install-only: true - - uses: https://code.forgejo.org/actions/setup-go@v4 - with: - go-version: "1.21" - - run: | - set -x - forgejo-binary.sh ensure_user forgejo - apt-get -q install -y -qq build-essential - - run: | - script=$(pwd)/forgejo/upgrades/test-upgrade.sh - $script run dependencies - $script clobber - su forgejo -c "$script test_upgrades" diff --git a/README.md b/README.md index 6ca1311..7d3a95e 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ git clone https://code.forgejo.org/actions/setup-forgejo export PATH=$(pwd)/setup-forgejo:$PATH git clone https://code.forgejo.org/forgejo/end-to-end cd end-to-end -export DIR=/tmp/end-to-end -rm -fr /tmp/end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example +export DIR=/tmp/forgejo-end-to-end +rm -fr /tmp/forgejo-end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example ``` Run using Forgejo built from source. diff --git a/end-to-end.sh b/end-to-end.sh new file mode 100755 index 0000000..3202c23 --- /dev/null +++ b/end-to-end.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# +# Debug loop from the source tree: +# +# ./end-to-end.sh dependencies +# ./end-to-end.sh build_all +# VERBOSE=true ./end-to-end.sh test_downgrade_1.20.2_fails +# +# Everything happens in /tmp/forgejo-end-to-end +# + +SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SELF="${BASH_SOURCE[0]}" +source $SELF_DIR/lib/lib.sh + +source $SELF_DIR/forgejo/upgrades/test-upgrade.sh +source $SELF_DIR/packages/packages.sh + +"$@" diff --git a/forgejo/upgrades/test-upgrade-1.20-storage.sh b/forgejo/upgrades/test-upgrade-1.20-storage.sh index 6be1fbb..74bd0c9 100644 --- a/forgejo/upgrades/test-upgrade-1.20-storage.sh +++ b/forgejo/upgrades/test-upgrade-1.20-storage.sh @@ -12,7 +12,7 @@ function test_downgrade_1.20.2_fails() { start 1.20.3-0 stop download 1.20.2-0 - timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log return 1 @@ -41,7 +41,7 @@ function test_bug_storage_merged() { log_info "upgrading from 1.20.2-0 with [storage].PATH fails" download 1.20.3-0 - timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log return 1 @@ -206,7 +206,7 @@ function test_bug_storage_misplace() { log_info "upgrading from 1.20.2-0 with conflicting sections fails" download 1.20.3-0 - timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true for path in ${STORAGE_PATHS} ; do if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log diff --git a/forgejo/upgrades/test-upgrade.sh b/forgejo/upgrades/test-upgrade.sh index 721f219..c62e2a2 100755 --- a/forgejo/upgrades/test-upgrade.sh +++ b/forgejo/upgrades/test-upgrade.sh @@ -11,19 +11,17 @@ # Everything happens in /tmp/forgejo-upgrades # -SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -SELF="${BASH_SOURCE[0]}" -source $SELF_DIR/../../lib/lib.sh +UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" : ${FORGEJO_REPO:=fixture} -source $SELF_DIR/fixtures.sh +source $UPGRADE_DIR/fixtures.sh function reset() { local config=$1 - reset_forgejo $SELF_DIR/$config-app.ini + reset_forgejo $UPGRADE_DIR/$config-app.ini reset_minio reset_garage } @@ -86,17 +84,14 @@ function test_forgejo_database_version() { test "$expected_version" = "$actual_version" } -source $SELF_DIR/test-upgrade-1.20-storage.sh -source $SELF_DIR/test-upgrade-forgejo-database-v3.sh -source $SELF_DIR/../../packages/packages.sh +source $UPGRADE_DIR/test-upgrade-1.20-storage.sh +source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh function test_upgrades() { run stop run dependencies run build_all - test_packages - run test_successful_upgrades run test_storage_stable_s3 minio run test_storage_stable_s3 garage @@ -104,5 +99,3 @@ function test_upgrades() { test_upgrade_1_20_storage run test_forgejo_database_v3_upgrades } - -"$@" diff --git a/lib/build.sh b/lib/build.sh index 61f87c3..9ad74ec 100755 --- a/lib/build.sh +++ b/lib/build.sh @@ -7,18 +7,19 @@ set -ex SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" version=$1 -DIR=$2 +dir_binaries=$2 v=$(echo $version | sed -E -e 's/^([0-9]+\.[0-9]+).*/\1/') +src=$dir_binaries/src-$v read url ref semver < $SELF_DIR/../forgejo/sources/$v -if ! test -d $DIR/src ; then - mkdir -p $DIR/src - cd $DIR/src +if ! test -d $src ; then + mkdir -p $src + cd $src git init git remote add origin $url else - cd $DIR/src + cd $src fi if ! [[ "$ref" =~ ^refs/ ]] ; then @@ -30,4 +31,4 @@ git switch --force-create $v $ref export TAGS="bindata sqlite sqlite_unlock_notify" FORGEJO_VERSION=$semver make deps-backend backend make generate forgejo -cp -a forgejo $DIR/forgejo-$v-dev +cp -a forgejo $dir_binaries/forgejo-$v-dev diff --git a/lib/lib.sh b/lib/lib.sh index 78a9876..846e295 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -29,6 +29,7 @@ RELEASE_NUMBERS="7.0" PREFIX=============== HOST_PORT=$IP:3000 DIR=/tmp/forgejo-end-to-end +DIR_BINARIES=/srv/forgejo-binaries export DOT_FORGEJO_CURL=$DIR/forgejo-curl export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0 : ${FORGEJO_USER:=root} @@ -70,8 +71,13 @@ function dependencies() { function build_all() { for dev in $RELEASE_NUMBERS ; do - log_info $dev-dev - $LIB_DIR/build.sh $dev $DIR + local forgejo=$DIR_BINARIES/forgejo-$dev-dev + if test -f $forgejo ; then + log_info $dev already exists + else + $LIB_DIR/build.sh $dev $DIR_BINARIES + log_info $dev built from sources + fi done } @@ -108,18 +114,18 @@ function full_version() { function download() { local version=$1 - if ! test -f $DIR/forgejo-$version ; then - mkdir -p $DIR + if ! test -f $DIR_BINARIES/forgejo-$version ; then + mkdir -p $DIR_BINARIES for owner in forgejo forgejo-experimental forgejo-integration ; do full_version=$(full_version $version $owner) if test "$full_version" = "" ; then continue fi - if wget -O $DIR/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then + if wget -O $DIR_BINARIES/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then break fi done - if test -s $DIR/forgejo-$version ; then + if test -s $DIR_BINARIES/forgejo-$version ; then if test "$version" != "$full_version" ; then log_info "downloaded $full_version for $version" fi @@ -127,7 +133,7 @@ function download() { echo unable to download Forgejo $version return 1 fi - chmod +x $DIR/forgejo-$version + chmod +x $DIR_BINARIES/forgejo-$version fi } @@ -147,7 +153,7 @@ function start_forgejo() { download $version local work_path=$DIR/forgejo-work-path - daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then grep '' $DIR/*.log grep '' $work_path/log/*.log 2> /dev/null @@ -247,7 +253,7 @@ function create_user_and_login() { local work_path=$DIR/forgejo-work-path local email="$FORGEJO_USER@example.com" - local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path" + local cli="$DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path" if ! $cli admin user list | grep --quiet "$email" ; then $cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email fi From 554d2b48c0c4083c1389f585ae1dc86805e666e4 Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Sun, 17 Mar 2024 16:26:17 +0100 Subject: [PATCH 2/5] split storage & ugprade and separate fixtures --- .forgejo/workflows/end-to-end.yml | 2 + end-to-end.sh | 4 +- forgejo/fixtures.sh | 5 + .../fixtures.sh => fixtures/storage.sh} | 1 - forgejo/upgrades/test-upgrade.sh | 101 ------------------ {forgejo/upgrades => storage}/default-app.ini | 0 .../legagy-relative-app.ini | 0 {forgejo/upgrades => storage}/merged-app.ini | 0 .../upgrades => storage}/misplace-app.ini | 0 .../upgrades => storage}/misplace-s3-app.ini | 0 .../misplace-s3-two-app.ini | 0 .../upgrades => storage}/relative-app.ini | 0 .../upgrades => storage}/specific-app.ini | 0 .../upgrades => storage}/stable-s3-app.ini | 0 .../storage-relative-app.ini | 0 storage/storage.sh | 58 ++++++++++ .../test-upgrade-1.20-storage.sh | 26 ++--- upgrade/default-app.ini | 30 ++++++ .../test-upgrade-forgejo-database-v3.sh | 11 +- upgrade/upgrade.sh | 50 +++++++++ 20 files changed, 170 insertions(+), 118 deletions(-) create mode 100644 forgejo/fixtures.sh rename forgejo/{upgrades/fixtures.sh => fixtures/storage.sh} (99%) delete mode 100755 forgejo/upgrades/test-upgrade.sh rename {forgejo/upgrades => storage}/default-app.ini (100%) rename {forgejo/upgrades => storage}/legagy-relative-app.ini (100%) rename {forgejo/upgrades => storage}/merged-app.ini (100%) rename {forgejo/upgrades => storage}/misplace-app.ini (100%) rename {forgejo/upgrades => storage}/misplace-s3-app.ini (100%) rename {forgejo/upgrades => storage}/misplace-s3-two-app.ini (100%) rename {forgejo/upgrades => storage}/relative-app.ini (100%) rename {forgejo/upgrades => storage}/specific-app.ini (100%) rename {forgejo/upgrades => storage}/stable-s3-app.ini (100%) rename {forgejo/upgrades => storage}/storage-relative-app.ini (100%) create mode 100755 storage/storage.sh rename {forgejo/upgrades => storage}/test-upgrade-1.20-storage.sh (93%) create mode 100644 upgrade/default-app.ini rename {forgejo/upgrades => upgrade}/test-upgrade-forgejo-database-v3.sh (68%) create mode 100755 upgrade/upgrade.sh diff --git a/.forgejo/workflows/end-to-end.yml b/.forgejo/workflows/end-to-end.yml index 4ad4303..04072cd 100644 --- a/.forgejo/workflows/end-to-end.yml +++ b/.forgejo/workflows/end-to-end.yml @@ -48,6 +48,7 @@ jobs: - run: su forgejo -c "./end-to-end.sh test_packages" upgrade: + name: upgrade and storage needs: [build] runs-on: lxc-bookworm steps: @@ -68,3 +69,4 @@ jobs: - uses: https://code.forgejo.org/actions/checkout@v4 - uses: ./.forgejo/prepare-end-to-end - run: su forgejo -c "./end-to-end.sh test_upgrades" + - run: su forgejo -c "./end-to-end.sh test_storage" diff --git a/end-to-end.sh b/end-to-end.sh index 3202c23..6157d3f 100755 --- a/end-to-end.sh +++ b/end-to-end.sh @@ -15,7 +15,9 @@ SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" SELF="${BASH_SOURCE[0]}" source $SELF_DIR/lib/lib.sh -source $SELF_DIR/forgejo/upgrades/test-upgrade.sh +source $SELF_DIR/forgejo/fixtures.sh +source $SELF_DIR/storage/storage.sh +source $SELF_DIR/upgrade/upgrade.sh source $SELF_DIR/packages/packages.sh "$@" diff --git a/forgejo/fixtures.sh b/forgejo/fixtures.sh new file mode 100644 index 0000000..9c609f5 --- /dev/null +++ b/forgejo/fixtures.sh @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: MIT + +FIXTURES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +source $FIXTURES_DIR/fixtures/storage.sh diff --git a/forgejo/upgrades/fixtures.sh b/forgejo/fixtures/storage.sh similarity index 99% rename from forgejo/upgrades/fixtures.sh rename to forgejo/fixtures/storage.sh index abad7e3..1efffe1 100644 --- a/forgejo/upgrades/fixtures.sh +++ b/forgejo/fixtures/storage.sh @@ -1,4 +1,3 @@ -#!/bin/bash # SPDX-License-Identifier: MIT #ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" diff --git a/forgejo/upgrades/test-upgrade.sh b/forgejo/upgrades/test-upgrade.sh deleted file mode 100755 index c62e2a2..0000000 --- a/forgejo/upgrades/test-upgrade.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: MIT - -# -# Debug loop from the source tree: -# -# ./forgejo/upgrades/test-upgrade.sh dependencies -# ./forgejo/upgrades/test-upgrade.sh build_all -# VERBOSE=true ./forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails -# -# Everything happens in /tmp/forgejo-upgrades -# - -UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" -STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" -: ${FORGEJO_REPO:=fixture} - -source $UPGRADE_DIR/fixtures.sh - -function reset() { - local config=$1 - reset_forgejo $UPGRADE_DIR/$config-app.ini - reset_minio - reset_garage -} - -function verify_storage() { - local work_path=$DIR/forgejo-work-path - - for path in ${STORAGE_PATHS} ; do - test -d $work_path/data/$path - done -} - -function cleanup_storage() { - local work_path=$DIR/forgejo-work-path - - for path in ${STORAGE_PATHS} ; do - rm -fr $work_path/data/$path - done -} - -function test_storage_stable_s3() { - local work_path=$DIR/forgejo-work-path - local s3_backend=${1:-minio} - - log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" - - for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do - log_info "Forgejo $version & $s3_backend" - stop - reset stable-s3 - start $version $s3_backend - fixture_create - for fun in ${STORAGE_FUN} ; do - fixture_${fun}_assert_s3 - done - done -} - -function test_successful_upgrades() { - stop - for config in default specific ; do - log_info "using $config app.ini" - reset $config - - for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do - log_info "run $version" - cleanup_storage - start $version - verify_storage - stop - done - done -} - -function test_forgejo_database_version() { - local expected_version=$1 - local work_path=$DIR/forgejo-work-path - - actual_version=$(sqlite3 $work_path/forgejo.db "select version from forgejo_version") - test "$expected_version" = "$actual_version" -} - -source $UPGRADE_DIR/test-upgrade-1.20-storage.sh -source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh - -function test_upgrades() { - run stop - run dependencies - run build_all - - run test_successful_upgrades - run test_storage_stable_s3 minio - run test_storage_stable_s3 garage - - test_upgrade_1_20_storage - run test_forgejo_database_v3_upgrades -} diff --git a/forgejo/upgrades/default-app.ini b/storage/default-app.ini similarity index 100% rename from forgejo/upgrades/default-app.ini rename to storage/default-app.ini diff --git a/forgejo/upgrades/legagy-relative-app.ini b/storage/legagy-relative-app.ini similarity index 100% rename from forgejo/upgrades/legagy-relative-app.ini rename to storage/legagy-relative-app.ini diff --git a/forgejo/upgrades/merged-app.ini b/storage/merged-app.ini similarity index 100% rename from forgejo/upgrades/merged-app.ini rename to storage/merged-app.ini diff --git a/forgejo/upgrades/misplace-app.ini b/storage/misplace-app.ini similarity index 100% rename from forgejo/upgrades/misplace-app.ini rename to storage/misplace-app.ini diff --git a/forgejo/upgrades/misplace-s3-app.ini b/storage/misplace-s3-app.ini similarity index 100% rename from forgejo/upgrades/misplace-s3-app.ini rename to storage/misplace-s3-app.ini diff --git a/forgejo/upgrades/misplace-s3-two-app.ini b/storage/misplace-s3-two-app.ini similarity index 100% rename from forgejo/upgrades/misplace-s3-two-app.ini rename to storage/misplace-s3-two-app.ini diff --git a/forgejo/upgrades/relative-app.ini b/storage/relative-app.ini similarity index 100% rename from forgejo/upgrades/relative-app.ini rename to storage/relative-app.ini diff --git a/forgejo/upgrades/specific-app.ini b/storage/specific-app.ini similarity index 100% rename from forgejo/upgrades/specific-app.ini rename to storage/specific-app.ini diff --git a/forgejo/upgrades/stable-s3-app.ini b/storage/stable-s3-app.ini similarity index 100% rename from forgejo/upgrades/stable-s3-app.ini rename to storage/stable-s3-app.ini diff --git a/forgejo/upgrades/storage-relative-app.ini b/storage/storage-relative-app.ini similarity index 100% rename from forgejo/upgrades/storage-relative-app.ini rename to storage/storage-relative-app.ini diff --git a/storage/storage.sh b/storage/storage.sh new file mode 100755 index 0000000..dda1307 --- /dev/null +++ b/storage/storage.sh @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: MIT + +STORAGE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" +STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" +: ${FORGEJO_REPO:=fixture} + +function storage_reset() { + local config=$1 + reset_forgejo $STORAGE_DIR/$config-app.ini + reset_minio + reset_garage +} + +function verify_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/$path + done +} + +function cleanup_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + rm -fr $work_path/data/$path + done +} + +function test_storage_stable_s3() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${1:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do + log_info "Forgejo $version & $s3_backend" + stop + storage_reset stable-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done +} + +source $STORAGE_DIR/test-upgrade-1.20-storage.sh + +function test_storage() { + run test_storage_stable_s3 minio + run test_storage_stable_s3 garage + + test_upgrade_1_20_storage + run test_forgejo_database_v3_upgrades +} diff --git a/forgejo/upgrades/test-upgrade-1.20-storage.sh b/storage/test-upgrade-1.20-storage.sh similarity index 93% rename from forgejo/upgrades/test-upgrade-1.20-storage.sh rename to storage/test-upgrade-1.20-storage.sh index 74bd0c9..063b56a 100644 --- a/forgejo/upgrades/test-upgrade-1.20-storage.sh +++ b/storage/test-upgrade-1.20-storage.sh @@ -8,7 +8,7 @@ function test_downgrade_1.20.2_fails() { log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails" stop - reset default + storage_reset default start 1.20.3-0 stop download 1.20.2-0 @@ -27,7 +27,7 @@ function test_bug_storage_merged() { log_info "using < 1.20.3-0 and [storage].PATH merge all storage" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do stop - reset merged + storage_reset merged start $version for path in ${STORAGE_PATHS} ; do ! test -d $work_path/data/$path @@ -54,7 +54,7 @@ function test_bug_storage_relative_path() { log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do stop - reset legagy-relative + storage_reset legagy-relative start $version test -d $work_path/relative-lfs test -d $work_path/relative-avatars @@ -64,7 +64,7 @@ function test_bug_storage_relative_path() { log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-5-rc2 ; do stop - reset legagy-relative + storage_reset legagy-relative start $version test -d $work_path/data/relative-lfs test -d $work_path/data/relative-avatars @@ -74,7 +74,7 @@ function test_bug_storage_relative_path() { log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-5-rc2 ; do stop - reset storage-relative + storage_reset storage-relative start $version for path in ${STORAGE_PATHS} ; do test -d $work_path/data/relative-$path @@ -84,7 +84,7 @@ function test_bug_storage_relative_path() { log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" for version in 1.20.2-0 ; do stop - reset storage-relative + storage_reset storage-relative start $version test -d $work_path/data/packages test -d $work_path/relative-repo-archive @@ -97,7 +97,7 @@ function test_bug_storage_relative_path() { log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent" for version in 1.18.5-0 1.19.4-0 ; do stop - reset storage-relative + storage_reset storage-relative start $version test -d $work_path/relative-packages test -d $work_path/relative-repo-archive @@ -110,7 +110,7 @@ function test_bug_storage_relative_path() { log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do stop - reset relative + storage_reset relative start $version for path in ${STORAGE_PATHS} ; do test -d $work_path/relative-$path @@ -120,7 +120,7 @@ function test_bug_storage_relative_path() { log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-5-rc2 ; do stop - reset relative + storage_reset relative start $version for path in ${STORAGE_PATHS} ; do test -d $work_path/data/relative-$path @@ -139,7 +139,7 @@ function test_bug_storage_s3_misplace() { for version in 1.20.2-0 1.20.3-0 ; do log_info "Forgejo $version & $s3_backend" stop - reset misplace-s3 + storage_reset misplace-s3 start $version $s3_backend fixture_create for fun in ${STORAGE_FUN} ; do @@ -150,7 +150,7 @@ function test_bug_storage_s3_misplace() { for version in 1.18.5-0 1.19.4-0 ; do log_info "Forgejo $version & $s3_backend" stop - reset misplace-s3 + storage_reset misplace-s3 start $version $s3_backend fixture_create # @@ -176,7 +176,7 @@ function test_bug_storage_misplace() { log_info "using < 1.20 and conflicting sections misplace storage" for version in 1.18.5-0 1.19.4-0 ; do stop - reset misplace + storage_reset misplace start $version # # some storage are where they should be @@ -195,7 +195,7 @@ function test_bug_storage_misplace() { log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" for version in 1.20.2-0 ; do stop - reset misplace + storage_reset misplace start $version for path in ${STORAGE_PATHS} ; do test -d $work_path/elsewhere/$path diff --git a/upgrade/default-app.ini b/upgrade/default-app.ini new file mode 100644 index 0000000..a51290a --- /dev/null +++ b/upgrade/default-app.ini @@ -0,0 +1,30 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = trace +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true diff --git a/forgejo/upgrades/test-upgrade-forgejo-database-v3.sh b/upgrade/test-upgrade-forgejo-database-v3.sh similarity index 68% rename from forgejo/upgrades/test-upgrade-forgejo-database-v3.sh rename to upgrade/test-upgrade-forgejo-database-v3.sh index db1e9d4..f15a4f3 100644 --- a/forgejo/upgrades/test-upgrade-forgejo-database-v3.sh +++ b/upgrade/test-upgrade-forgejo-database-v3.sh @@ -1,6 +1,13 @@ -#!/bin/bash # SPDX-License-Identifier: MIT +function test_forgejo_database_version() { + local expected_version=$1 + local work_path=$DIR/forgejo-work-path + + actual_version=$(sqlite3 $work_path/forgejo.db "select version from forgejo_version") + test "$expected_version" = "$actual_version" +} + function test_forgejo_database_v3_upgrades_list_table() { local table=$1 local work_path=$DIR/forgejo-work-path @@ -13,7 +20,7 @@ function test_forgejo_database_v3_upgrades() { stop - reset default + upgrade_reset default log_info "run 1.20.4-1" start 1.20.4-1 stop diff --git a/upgrade/upgrade.sh b/upgrade/upgrade.sh new file mode 100755 index 0000000..fa40d76 --- /dev/null +++ b/upgrade/upgrade.sh @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: MIT + +UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function upgrade_reset() { + local config=$1 + reset_forgejo $UPGRADE_DIR/$config-app.ini + reset_minio +} + +function verify_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/$path + done +} + +function cleanup_storage() { + local work_path=$DIR/forgejo-work-path + + for path in ${STORAGE_PATHS} ; do + rm -fr $work_path/data/$path + done +} + +function test_successful_upgrades() { + for config in default ; do + log_info "using $config app.ini" + upgrade_reset $config + + for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do + log_info "run $version" + cleanup_storage + start $version + verify_storage + stop + done + done +} + +source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh + +function test_upgrades() { + run dependencies + run build_all + + run test_successful_upgrades + run test_forgejo_database_v3_upgrades +} From 51f7263dee25235921dc39471f5f81400dd9a3e5 Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Sun, 17 Mar 2024 17:36:05 +0100 Subject: [PATCH 3/5] hide expected errors so they are not confused with actual errors --- lib/lib.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lib.sh b/lib/lib.sh index 846e295..5b07c86 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -154,7 +154,7 @@ function start_forgejo() { download $version local work_path=$DIR/forgejo-work-path daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path - if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then + if ! retry grep --no-messages --quiet 'Starting server on' $work_path/log/forgejo.log ; then grep '' $DIR/*.log grep '' $work_path/log/*.log 2> /dev/null return 1 @@ -170,7 +170,8 @@ function start_minio() { --env=MINIO_ROOT_PASSWORD=12345678 \ --env=MINIO_VOLUMES=$DIR/minio \ --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server - retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 + retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 >& /dev/null + mc alias set testS3 http://127.0.0.1:9000 123456 12345678 } function start_garage() { From 21920b10d97d5404cda48fb3911ce392bcd41063 Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Sun, 17 Mar 2024 17:22:44 +0100 Subject: [PATCH 4/5] upgrades: add LFS assert --- forgejo/fixtures/storage.sh | 21 +++++++++++++++++++++ lib/lib.sh | 14 ++++++++++---- storage/storage.sh | 2 -- upgrade/upgrade.sh | 14 +++++++++++--- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/forgejo/fixtures/storage.sh b/forgejo/fixtures/storage.sh index 1efffe1..0de79b5 100644 --- a/forgejo/fixtures/storage.sh +++ b/forgejo/fixtures/storage.sh @@ -7,6 +7,9 @@ # ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg==" +STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" +: ${FORGEJO_REPO:=fixture} + function fixture_get_paths_s3() { local path=$1 @@ -81,6 +84,18 @@ function fixture_lfs_create() { ) } +function fixture_lfs_assert() { + local d=$(mktemp -d) + ( + git clone http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO} $d/${FORGEJO_REPO} + cd $d/${FORGEJO_REPO} + rm file.txt + git-lfs checkout file.txt + test -f file.txt + ) + rm -fr $d +} + function fixture_lfs_assert_s3() { local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) test "$content" = CONTENT @@ -200,3 +215,9 @@ function fixture_create() { fixture_${fun}_create done } + +function fixture_assert() { + for fun in lfs ; do + fixture_${fun}_assert + done +} diff --git a/lib/lib.sh b/lib/lib.sh index 5b07c86..fa6375f 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -249,14 +249,20 @@ function reset_garage() { rm -fr $DIR/garage } +function forgejo_cli() { + local version="$1" + shift + local work_path=$DIR/forgejo-work-path + + $DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path "$@" +} + function create_user_and_login() { local version=$1 - local work_path=$DIR/forgejo-work-path local email="$FORGEJO_USER@example.com" - local cli="$DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path" - if ! $cli admin user list | grep --quiet "$email" ; then - $cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email + if ! forgejo_cli $version admin user list | grep --quiet "$email" ; then + forgejo_cli $version admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email fi forgejo-curl.sh logout diff --git a/storage/storage.sh b/storage/storage.sh index dda1307..b08e6bc 100755 --- a/storage/storage.sh +++ b/storage/storage.sh @@ -3,8 +3,6 @@ STORAGE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" -STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" -: ${FORGEJO_REPO:=fixture} function storage_reset() { local config=$1 diff --git a/upgrade/upgrade.sh b/upgrade/upgrade.sh index fa40d76..5ad4415 100755 --- a/upgrade/upgrade.sh +++ b/upgrade/upgrade.sh @@ -25,16 +25,24 @@ function cleanup_storage() { } function test_successful_upgrades() { + stop for config in default ; do log_info "using $config app.ini" upgrade_reset $config - for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do + version=1.18 + log_info "run $version" + cleanup_storage + start $version + fixture_create + fixture_assert + + for version in 1.19 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do + stop log_info "run $version" - cleanup_storage start $version verify_storage - stop + fixture_assert done done } From 6bb5e947fb0d2ef90465f4563692f8fe44172ba3 Mon Sep 17 00:00:00 2001 From: Twenty Panda Date: Sun, 17 Mar 2024 18:12:41 +0100 Subject: [PATCH 5/5] upgrades: run doctor check --all after each upgrade --- forgejo/fixtures.sh | 1 + forgejo/fixtures/doctor.sh | 7 +++++++ upgrade/default-app.ini | 3 +++ upgrade/upgrade.sh | 2 ++ 4 files changed, 13 insertions(+) create mode 100644 forgejo/fixtures/doctor.sh diff --git a/forgejo/fixtures.sh b/forgejo/fixtures.sh index 9c609f5..9e7f5ca 100644 --- a/forgejo/fixtures.sh +++ b/forgejo/fixtures.sh @@ -3,3 +3,4 @@ FIXTURES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $FIXTURES_DIR/fixtures/storage.sh +source $FIXTURES_DIR/fixtures/doctor.sh diff --git a/forgejo/fixtures/doctor.sh b/forgejo/fixtures/doctor.sh new file mode 100644 index 0000000..9064699 --- /dev/null +++ b/forgejo/fixtures/doctor.sh @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: MIT + +function doctor_run() { + local version=$1 + + forgejo_cli $version doctor check --all # --log-file - +} diff --git a/upgrade/default-app.ini b/upgrade/default-app.ini index a51290a..af8f6e3 100644 --- a/upgrade/default-app.ini +++ b/upgrade/default-app.ini @@ -7,6 +7,9 @@ HTTP_PORT = 3000 SSH_LISTEN_PORT = 2222 LFS_START_SERVER = true +[queue] +TYPE = immediate + [database] DB_TYPE = sqlite3 PATH = ${WORK_PATH}/forgejo.db diff --git a/upgrade/upgrade.sh b/upgrade/upgrade.sh index 5ad4415..5f69abc 100755 --- a/upgrade/upgrade.sh +++ b/upgrade/upgrade.sh @@ -36,6 +36,7 @@ function test_successful_upgrades() { start $version fixture_create fixture_assert + doctor_run $version for version in 1.19 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do stop @@ -43,6 +44,7 @@ function test_successful_upgrades() { start $version verify_storage fixture_assert + doctor_run $version done done }