split storage & ugprade and separate fixtures

.forgejo/workflows/end-to-end.yml

@@ -48,6 +48,7 @@ jobs:
       - run: su forgejo -c "./end-to-end.sh test_packages"
 
   upgrade:
+    name: upgrade and storage
     needs: [build]
     runs-on: lxc-bookworm
     steps:
@@ -68,3 +69,4 @@ jobs:
       - uses: https://code.forgejo.org/actions/checkout@v4
       - uses: ./.forgejo/prepare-end-to-end
       - run: su forgejo -c "./end-to-end.sh test_upgrades"
+      - run: su forgejo -c "./end-to-end.sh test_storage"

end-to-end.sh

@@ -15,7 +15,9 @@ SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SELF="${BASH_SOURCE[0]}"
 source $SELF_DIR/lib/lib.sh
 
-source $SELF_DIR/forgejo/upgrades/test-upgrade.sh
+source $SELF_DIR/forgejo/fixtures.sh
+source $SELF_DIR/storage/storage.sh
+source $SELF_DIR/upgrade/upgrade.sh
 source $SELF_DIR/packages/packages.sh
 
 "$@"

forgejo/fixtures.sh

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: MIT
+
+FIXTURES_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source $FIXTURES_DIR/fixtures/storage.sh

forgejo/fixtures/storage.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # SPDX-License-Identifier: MIT
 
 #ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="

forgejo/upgrades/test-upgrade.sh

@@ -1,101 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: MIT
-
-#
-# Debug loop from the source tree:
-#
-# ./forgejo/upgrades/test-upgrade.sh dependencies
-# ./forgejo/upgrades/test-upgrade.sh build_all
-# VERBOSE=true ./forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails
-#
-# Everything happens in /tmp/forgejo-upgrades
-#
-
-UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars"
-STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars"
-: ${FORGEJO_REPO:=fixture}
-
-source $UPGRADE_DIR/fixtures.sh
-
-function reset() {
-    local config=$1
-    reset_forgejo $UPGRADE_DIR/$config-app.ini
-    reset_minio
-    reset_garage
-}
-
-function verify_storage() {
-    local work_path=$DIR/forgejo-work-path
-
-    for path in ${STORAGE_PATHS} ; do
-        test -d $work_path/data/$path
-    done
-}
-
-function cleanup_storage() {
-    local work_path=$DIR/forgejo-work-path
-
-    for path in ${STORAGE_PATHS} ; do
-        rm -fr $work_path/data/$path
-    done
-}
-
-function test_storage_stable_s3() {
-    local work_path=$DIR/forgejo-work-path
-    local s3_backend=${1:-minio}
-
-    log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
-
-    for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do
-        log_info "Forgejo $version & $s3_backend"
-        stop
-        reset stable-s3
-        start $version $s3_backend
-        fixture_create
-        for fun in ${STORAGE_FUN} ; do
-            fixture_${fun}_assert_s3
-        done
-    done
-}
-
-function test_successful_upgrades() {
-    stop
-    for config in default specific ; do
-        log_info "using $config app.ini"
-        reset $config
-
-        for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do
-            log_info "run $version"
-            cleanup_storage
-            start $version
-            verify_storage
-            stop
-        done
-    done
-}
-
-function test_forgejo_database_version() {
-    local expected_version=$1
-    local work_path=$DIR/forgejo-work-path
-
-    actual_version=$(sqlite3 $work_path/forgejo.db "select version from forgejo_version")
-    test "$expected_version" = "$actual_version"
-}
-
-source $UPGRADE_DIR/test-upgrade-1.20-storage.sh
-source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh
-
-function test_upgrades() {
-    run stop
-    run dependencies
-    run build_all
-
-    run test_successful_upgrades
-    run test_storage_stable_s3 minio
-    run test_storage_stable_s3 garage
-
-    test_upgrade_1_20_storage
-    run test_forgejo_database_v3_upgrades
-}

storage/storage.sh

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: MIT
+
+STORAGE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars"
+STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars"
+: ${FORGEJO_REPO:=fixture}
+
+function storage_reset() {
+    local config=$1
+    reset_forgejo $STORAGE_DIR/$config-app.ini
+    reset_minio
+    reset_garage
+}
+
+function verify_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        test -d $work_path/data/$path
+    done
+}
+
+function cleanup_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        rm -fr $work_path/data/$path
+    done
+}
+
+function test_storage_stable_s3() {
+    local work_path=$DIR/forgejo-work-path
+    local s3_backend=${1:-minio}
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
+
+    for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do
+        log_info "Forgejo $version & $s3_backend"
+        stop
+        storage_reset stable-s3
+        start $version $s3_backend
+        fixture_create
+        for fun in ${STORAGE_FUN} ; do
+            fixture_${fun}_assert_s3
+        done
+    done
+}
+
+source $STORAGE_DIR/test-upgrade-1.20-storage.sh
+
+function test_storage() {
+    run test_storage_stable_s3 minio
+    run test_storage_stable_s3 garage
+
+    test_upgrade_1_20_storage
+    run test_forgejo_database_v3_upgrades
+}

storage/test-upgrade-1.20-storage.sh

@@ -8,7 +8,7 @@ function test_downgrade_1.20.2_fails() {
 
     log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails"
     stop
-    reset default
+    storage_reset default
     start 1.20.3-0
     stop
     download 1.20.2-0
@@ -27,7 +27,7 @@ function test_bug_storage_merged() {
     log_info "using < 1.20.3-0 and [storage].PATH merge all storage"
     for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
         stop
-        reset merged
+        storage_reset merged
         start $version
         for path in ${STORAGE_PATHS} ; do
             ! test -d $work_path/data/$path
@@ -54,7 +54,7 @@ function test_bug_storage_relative_path() {
     log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH"
     for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
         stop
-        reset legagy-relative
+        storage_reset legagy-relative
         start $version
         test -d $work_path/relative-lfs
         test -d $work_path/relative-avatars
@@ -64,7 +64,7 @@ function test_bug_storage_relative_path() {
     log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH"
     for version in 1.20.3-0 1.21.0-5-rc2 ; do
         stop
-        reset legagy-relative
+        storage_reset legagy-relative
         start $version
         test -d $work_path/data/relative-lfs
         test -d $work_path/data/relative-avatars
@@ -74,7 +74,7 @@ function test_bug_storage_relative_path() {
     log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH"
     for version in 1.20.3-0 1.21.0-5-rc2 ; do
         stop
-        reset storage-relative
+        storage_reset storage-relative
         start $version
         for path in ${STORAGE_PATHS} ; do
             test -d $work_path/data/relative-$path
@@ -84,7 +84,7 @@ function test_bug_storage_relative_path() {
     log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent"
     for version in 1.20.2-0 ; do
         stop
-        reset storage-relative
+        storage_reset storage-relative
         start $version
         test -d $work_path/data/packages
         test -d $work_path/relative-repo-archive
@@ -97,7 +97,7 @@ function test_bug_storage_relative_path() {
     log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent"
     for version in 1.18.5-0 1.19.4-0 ; do
         stop
-        reset storage-relative
+        storage_reset storage-relative
         start $version
         test -d $work_path/relative-packages
         test -d $work_path/relative-repo-archive
@@ -110,7 +110,7 @@ function test_bug_storage_relative_path() {
     log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH"
     for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
         stop
-        reset relative
+        storage_reset relative
         start $version
         for path in ${STORAGE_PATHS} ; do
             test -d $work_path/relative-$path
@@ -120,7 +120,7 @@ function test_bug_storage_relative_path() {
     log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH"
     for version in 1.20.3-0 1.21.0-5-rc2 ; do
         stop
-        reset relative
+        storage_reset relative
         start $version
         for path in ${STORAGE_PATHS} ; do
             test -d $work_path/data/relative-$path
@@ -139,7 +139,7 @@ function test_bug_storage_s3_misplace() {
     for version in 1.20.2-0 1.20.3-0 ; do
         log_info "Forgejo $version & $s3_backend"
         stop
-        reset misplace-s3
+        storage_reset misplace-s3
         start $version $s3_backend
         fixture_create
         for fun in ${STORAGE_FUN} ; do
@@ -150,7 +150,7 @@ function test_bug_storage_s3_misplace() {
     for version in 1.18.5-0 1.19.4-0 ; do
         log_info "Forgejo $version & $s3_backend"
         stop
-        reset misplace-s3
+        storage_reset misplace-s3
         start $version $s3_backend
         fixture_create
         #
@@ -176,7 +176,7 @@ function test_bug_storage_misplace() {
     log_info "using < 1.20 and conflicting sections misplace storage"
     for version in 1.18.5-0 1.19.4-0 ; do
         stop
-        reset misplace
+        storage_reset misplace
         start $version
         #
         # some storage are where they should be
@@ -195,7 +195,7 @@ function test_bug_storage_misplace() {
     log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]"
     for version in 1.20.2-0 ; do
         stop
-        reset misplace
+        storage_reset misplace
         start $version
         for path in ${STORAGE_PATHS} ; do
             test -d $work_path/elsewhere/$path

upgrade/default-app.ini

@@ -0,0 +1,30 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+PATH = ${WORK_PATH}/forgejo.db
+
+[log]
+MODE = file
+LEVEL = trace
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true

upgrade/test-upgrade-forgejo-database-v3.sh

@@ -1,6 +1,13 @@
-#!/bin/bash
 # SPDX-License-Identifier: MIT
 
+function test_forgejo_database_version() {
+    local expected_version=$1
+    local work_path=$DIR/forgejo-work-path
+
+    actual_version=$(sqlite3 $work_path/forgejo.db "select version from forgejo_version")
+    test "$expected_version" = "$actual_version"
+}
+
 function test_forgejo_database_v3_upgrades_list_table() {
     local table=$1
     local work_path=$DIR/forgejo-work-path
@@ -13,7 +20,7 @@ function test_forgejo_database_v3_upgrades() {
 
     stop
 
-    reset default
+    upgrade_reset default
     log_info "run 1.20.4-1"
     start 1.20.4-1
     stop

upgrade/upgrade.sh

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: MIT
+
+UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function upgrade_reset() {
+    local config=$1
+    reset_forgejo $UPGRADE_DIR/$config-app.ini
+    reset_minio
+}
+
+function verify_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        test -d $work_path/data/$path
+    done
+}
+
+function cleanup_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        rm -fr $work_path/data/$path
+    done
+}
+
+function test_successful_upgrades() {
+    for config in default ; do
+        log_info "using $config app.ini"
+        upgrade_reset $config
+
+        for version in 1.18 1.19 1.20.2-0 1.20.3-0 1.20 1.21 $RELEASE_NUMBERS_AND_DEV ; do
+            log_info "run $version"
+            cleanup_storage
+            start $version
+            verify_storage
+            stop
+        done
+    done
+}
+
+source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh
+
+function test_upgrades() {
+    run dependencies
+    run build_all
+
+    run test_successful_upgrades
+    run test_forgejo_database_v3_upgrades
+}