diff --git a/.forgejo/prepare-end-to-end/action.yml b/.forgejo/prepare-end-to-end/action.yml new file mode 100644 index 0000000..d94a09a --- /dev/null +++ b/.forgejo/prepare-end-to-end/action.yml @@ -0,0 +1,19 @@ +runs: + using: "composite" + steps: + - uses: https://code.forgejo.org/actions/setup-forgejo@v2 + with: + install-only: true + - run: forgejo-binary.sh ensure_user forgejo + - uses: actions/download-artifact@v3 + with: + name: forgejo-dev + path: /srv/forgejo-binaries + - name: chown/chmod /srv/forgejo-binaries + run: | + chmod +x /srv/forgejo-binaries/* + chown -R forgejo /srv/forgejo-binaries + - run: | + script=$(pwd)/end-to-end.sh + $script run dependencies + $script clobber diff --git a/.forgejo/workflows/end-to-end.yml b/.forgejo/workflows/end-to-end.yml new file mode 100644 index 0000000..4ad4303 --- /dev/null +++ b/.forgejo/workflows/end-to-end.yml @@ -0,0 +1,70 @@ +on: + pull_request: + push: + branches: + - 'main' + +jobs: + build: + runs-on: docker + container: + image: 'docker.io/node:20-bookworm' + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: "1.21" + - name: lib/build.sh + run: | + set -x + + # + # SQLite needs gcc to be compiled + # + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get -q install -y -qq build-essential + + d=/tmp/forgejo-binaries + mkdir $d /tmp/forgejo-upload + + for version in $(ls forgejo/sources) ; do + forgejo=$d/forgejo-$version-dev + lib/build.sh $version $d + $forgejo --version + mv $forgejo /tmp/forgejo-upload + done + - uses: actions/upload-artifact@v3 + with: + name: forgejo-dev + path: /tmp/forgejo-upload + + packages: + needs: [build] + runs-on: lxc-bookworm + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: ./.forgejo/prepare-end-to-end + - run: su forgejo -c "./end-to-end.sh test_packages" + + upgrade: + needs: [build] + runs-on: lxc-bookworm + steps: + - name: cache S3 binaries + id: S3 + uses: https://code.forgejo.org/actions/cache@v3 + with: + path: | + /usr/local/bin/minio + /usr/local/bin/mc + /usr/local/bin/garage + key: S3 + + - name: skip if S3 cache hit + if: steps.S3.outputs.cache-hit != 'true' + run: echo no hit + + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: ./.forgejo/prepare-end-to-end + - run: su forgejo -c "./end-to-end.sh test_upgrades" diff --git a/.forgejo/workflows/upgrade.yml b/.forgejo/workflows/upgrade.yml deleted file mode 100644 index 5e8f7c5..0000000 --- a/.forgejo/workflows/upgrade.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: upgrade - -on: - pull_request: - push: - branches: - - 'main' - -jobs: - upgrade: - runs-on: lxc-bookworm - steps: - - name: cache S3 binaries - id: S3 - uses: https://code.forgejo.org/actions/cache@v3 - with: - path: | - /usr/local/bin/minio - /usr/local/bin/mc - /usr/local/bin/garage - key: S3 - - - name: skip if S3 cache hit - if: steps.S3.outputs.cache-hit != 'true' - run: echo no hit - - - uses: https://code.forgejo.org/actions/checkout@v4 - - uses: https://code.forgejo.org/actions/setup-forgejo@v2 - with: - install-only: true - - uses: https://code.forgejo.org/actions/setup-go@v4 - with: - go-version: "1.21" - - run: | - set -x - forgejo-binary.sh ensure_user forgejo - apt-get -q install -y -qq build-essential - - run: | - script=$(pwd)/forgejo/upgrades/test-upgrade.sh - $script run dependencies - $script clobber - su forgejo -c "$script test_upgrades" diff --git a/README.md b/README.md index 6ca1311..7d3a95e 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ git clone https://code.forgejo.org/actions/setup-forgejo export PATH=$(pwd)/setup-forgejo:$PATH git clone https://code.forgejo.org/forgejo/end-to-end cd end-to-end -export DIR=/tmp/end-to-end -rm -fr /tmp/end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example +export DIR=/tmp/forgejo-end-to-end +rm -fr /tmp/forgejo-end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example ``` Run using Forgejo built from source. diff --git a/end-to-end.sh b/end-to-end.sh new file mode 100755 index 0000000..3202c23 --- /dev/null +++ b/end-to-end.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# +# Debug loop from the source tree: +# +# ./end-to-end.sh dependencies +# ./end-to-end.sh build_all +# VERBOSE=true ./end-to-end.sh test_downgrade_1.20.2_fails +# +# Everything happens in /tmp/forgejo-end-to-end +# + +SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SELF="${BASH_SOURCE[0]}" +source $SELF_DIR/lib/lib.sh + +source $SELF_DIR/forgejo/upgrades/test-upgrade.sh +source $SELF_DIR/packages/packages.sh + +"$@" diff --git a/forgejo/upgrades/test-upgrade-1.20-storage.sh b/forgejo/upgrades/test-upgrade-1.20-storage.sh index 6be1fbb..74bd0c9 100644 --- a/forgejo/upgrades/test-upgrade-1.20-storage.sh +++ b/forgejo/upgrades/test-upgrade-1.20-storage.sh @@ -12,7 +12,7 @@ function test_downgrade_1.20.2_fails() { start 1.20.3-0 stop download 1.20.2-0 - timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log return 1 @@ -41,7 +41,7 @@ function test_bug_storage_merged() { log_info "upgrading from 1.20.2-0 with [storage].PATH fails" download 1.20.3-0 - timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log return 1 @@ -206,7 +206,7 @@ function test_bug_storage_misplace() { log_info "upgrading from 1.20.2-0 with conflicting sections fails" download 1.20.3-0 - timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true + timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true for path in ${STORAGE_PATHS} ; do if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then cat $work_path/log/forgejo.log diff --git a/forgejo/upgrades/test-upgrade.sh b/forgejo/upgrades/test-upgrade.sh index 721f219..c62e2a2 100755 --- a/forgejo/upgrades/test-upgrade.sh +++ b/forgejo/upgrades/test-upgrade.sh @@ -11,19 +11,17 @@ # Everything happens in /tmp/forgejo-upgrades # -SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -SELF="${BASH_SOURCE[0]}" -source $SELF_DIR/../../lib/lib.sh +UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" : ${FORGEJO_REPO:=fixture} -source $SELF_DIR/fixtures.sh +source $UPGRADE_DIR/fixtures.sh function reset() { local config=$1 - reset_forgejo $SELF_DIR/$config-app.ini + reset_forgejo $UPGRADE_DIR/$config-app.ini reset_minio reset_garage } @@ -86,17 +84,14 @@ function test_forgejo_database_version() { test "$expected_version" = "$actual_version" } -source $SELF_DIR/test-upgrade-1.20-storage.sh -source $SELF_DIR/test-upgrade-forgejo-database-v3.sh -source $SELF_DIR/../../packages/packages.sh +source $UPGRADE_DIR/test-upgrade-1.20-storage.sh +source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh function test_upgrades() { run stop run dependencies run build_all - test_packages - run test_successful_upgrades run test_storage_stable_s3 minio run test_storage_stable_s3 garage @@ -104,5 +99,3 @@ function test_upgrades() { test_upgrade_1_20_storage run test_forgejo_database_v3_upgrades } - -"$@" diff --git a/lib/build.sh b/lib/build.sh index 61f87c3..9ad74ec 100755 --- a/lib/build.sh +++ b/lib/build.sh @@ -7,18 +7,19 @@ set -ex SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" version=$1 -DIR=$2 +dir_binaries=$2 v=$(echo $version | sed -E -e 's/^([0-9]+\.[0-9]+).*/\1/') +src=$dir_binaries/src-$v read url ref semver < $SELF_DIR/../forgejo/sources/$v -if ! test -d $DIR/src ; then - mkdir -p $DIR/src - cd $DIR/src +if ! test -d $src ; then + mkdir -p $src + cd $src git init git remote add origin $url else - cd $DIR/src + cd $src fi if ! [[ "$ref" =~ ^refs/ ]] ; then @@ -30,4 +31,4 @@ git switch --force-create $v $ref export TAGS="bindata sqlite sqlite_unlock_notify" FORGEJO_VERSION=$semver make deps-backend backend make generate forgejo -cp -a forgejo $DIR/forgejo-$v-dev +cp -a forgejo $dir_binaries/forgejo-$v-dev diff --git a/lib/lib.sh b/lib/lib.sh index 78a9876..846e295 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -29,6 +29,7 @@ RELEASE_NUMBERS="7.0" PREFIX=============== HOST_PORT=$IP:3000 DIR=/tmp/forgejo-end-to-end +DIR_BINARIES=/srv/forgejo-binaries export DOT_FORGEJO_CURL=$DIR/forgejo-curl export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0 : ${FORGEJO_USER:=root} @@ -70,8 +71,13 @@ function dependencies() { function build_all() { for dev in $RELEASE_NUMBERS ; do - log_info $dev-dev - $LIB_DIR/build.sh $dev $DIR + local forgejo=$DIR_BINARIES/forgejo-$dev-dev + if test -f $forgejo ; then + log_info $dev already exists + else + $LIB_DIR/build.sh $dev $DIR_BINARIES + log_info $dev built from sources + fi done } @@ -108,18 +114,18 @@ function full_version() { function download() { local version=$1 - if ! test -f $DIR/forgejo-$version ; then - mkdir -p $DIR + if ! test -f $DIR_BINARIES/forgejo-$version ; then + mkdir -p $DIR_BINARIES for owner in forgejo forgejo-experimental forgejo-integration ; do full_version=$(full_version $version $owner) if test "$full_version" = "" ; then continue fi - if wget -O $DIR/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then + if wget -O $DIR_BINARIES/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then break fi done - if test -s $DIR/forgejo-$version ; then + if test -s $DIR_BINARIES/forgejo-$version ; then if test "$version" != "$full_version" ; then log_info "downloaded $full_version for $version" fi @@ -127,7 +133,7 @@ function download() { echo unable to download Forgejo $version return 1 fi - chmod +x $DIR/forgejo-$version + chmod +x $DIR_BINARIES/forgejo-$version fi } @@ -147,7 +153,7 @@ function start_forgejo() { download $version local work_path=$DIR/forgejo-work-path - daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then grep '' $DIR/*.log grep '' $work_path/log/*.log 2> /dev/null @@ -247,7 +253,7 @@ function create_user_and_login() { local work_path=$DIR/forgejo-work-path local email="$FORGEJO_USER@example.com" - local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path" + local cli="$DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path" if ! $cli admin user list | grep --quiet "$email" ; then $cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email fi