magic_rb/forgejo
Template
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo synced 2024-11-25 03:06:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

[TESTS] upgrade tests for storage

Browse source 
(cherry picked from commit 884ca63738)
(cherry picked from commit 0a45d9c37b)

[TESTS] upgrade tests for storage (squash) relative paths

(cherry picked from commit 3bb19285f3)
(cherry picked from commit c640c09e61)
(cherry picked from commit 40ffe2d226)

[UPGRADE] S3 storage and fixtures

(cherry picked from commit c466c9c657)
(cherry picked from commit e80abbe2cd)
(cherry picked from commit 50a47df1d1)

[UPGRADE]  add sanity checks for [storage*] (squash) speedup upgrade tests

(cherry picked from commit f578279cfe)
(cherry picked from commit 6dd677f151)
This commit is contained in:
Earl Warren 2023-08-11 19:12:44 +02:00
parent 4e63a01a8b
commit ee7e7eca27
No known key found for this signature in database
GPG key ID: 0579CB2928A78A00
12 changed files with 1238 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
.forgejo/upgrades/default-app.ini Normal file
View file

@ -0,0 +1,30 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/data
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
PATH = ${WORK_PATH}/forgejo.db
[log]
MODE = file
LEVEL = trace
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true

187
.forgejo/upgrades/fixtures.sh Normal file
View file

@ -0,0 +1,187 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
#ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
#
# one pixel scaled to 290x290 because that's what versions lower or equal to v1.19.4-0 want
# by default and any other size will be transformed which make it difficult to compare.
#
ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg=="
function fixture_get_paths_s3() {
local path=$1
(
echo -n $path/
mc ls --quiet --recursive testS3/$path | sed -e 's/.* //'
) > $DIR/path
}
function fixture_get_paths_local() {
local path=$1
local work_path=$DIR/forgejo-work-path
( cd $work_path ; find $path -type f) > $DIR/path
}
function fixture_get_one_path() {
local storage=$1
local path=$2
fixture_get_paths_$storage $path
if test $(wc -l < $DIR/path) != 1 ; then
echo expected one path but got
cat $DIR/path
return 1
fi
cat $DIR/path
}
function fixture_repo_archive_create() {
retry curl -f -sS http://${HOST_PORT}/root/fixture/archive/main.zip -o /dev/null
}
function fixture_repo_archive_assert_s3() {
mc ls --recursive testS3/forgejo/repo-archive | grep --quiet '.zip$'
}
function fixture_repo_archive_assert_local() {
local path=$1
local work_path=$DIR/forgejo-work-path
find $work_path/$path | grep --quiet '.zip$'
}
function fixture_lfs_create() {
(
cd $DIR/fixture
git lfs track "*.txt"
echo CONTENT > file.txt
git add .
git commit -m 'lfs files'
git push
)
}
function fixture_lfs_assert_s3() {
local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7)
test "$content" = CONTENT
}
function fixture_lfs_assert_local() {
local path=$1
local work_path=$DIR/forgejo-work-path
local content=$(mc cat $work_path/$path/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7)
test "$content" = CONTENT
}
function fixture_packages_create() {
echo PACKAGE_CONTENT > $DIR/fixture/package
$work_path/forgejo-api -X DELETE http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt || true
$work_path/forgejo-api --upload-file $DIR/fixture/package http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt
}
function fixture_packages_assert_s3() {
local content=$(mc cat testS3/forgejo/packages/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17)
test "$content" = PACKAGE_CONTENT
}
function fixture_packages_assert_local() {
local path=$1
local content=$(cat $work_path/$path/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17)
test "$content" = PACKAGE_CONTENT
}
function fixture_avatars_create() {
echo -n $ONEPIXEL | base64 --decode > $DIR/avatar.png
$work_path/forgejo-client --form avatar=@$DIR/avatar.png http://${HOST_PORT}/user/settings/avatar
}
function fixture_avatars_assert_s3() {
local filename=$(fixture_get_one_path s3 forgejo/avatars)
local content=$(mc cat testS3/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_avatars_assert_local() {
local path=$1
local filename=$(fixture_get_one_path local $path)
local content=$(cat $work_path/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_repo_avatars_create() {
echo -n $ONEPIXEL | base64 --decode > $DIR/repo-avatar.png
$work_path/forgejo-client --form avatar=@$DIR/repo-avatar.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/settings/avatar
# v1.21 only
#$work_path/forgejo-api -X POST --data-raw '{"body":"'$avatar'"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/avatar
}
function fixture_repo_avatars_assert_s3() {
local filename=$(fixture_get_one_path s3 forgejo/repo-avatars)
local content=$(mc cat testS3/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_repo_avatars_assert_local() {
local path=$1
local filename=$(fixture_get_one_path local $path)
local content=$(cat $work_path/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_attachments_create_1_18() {
echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png
$work_path/forgejo-client --trace-ascii - --form file=@$DIR/attachment.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/issues/attachments
}
function fixture_attachments_create() {
if $work_path/forgejo-api http://${HOST_PORT}/api/v1/version | grep --quiet --fixed-strings 1.18. ; then
fixture_attachments_create_1_18
return
fi
id=$($work_path/forgejo-api --data-raw '{"title":"TITLE"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues | jq .id)
echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png
$work_path/forgejo-client -H @$DIR/forgejo-work-path/forgejo-header --form name=attachment.png --form attachment=@$DIR/attachment.png http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues/$id/assets
}
function fixture_attachments_assert_s3() {
local filename=$(fixture_get_one_path s3 forgejo/attachments)
local content=$(mc cat testS3/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_attachments_assert_local() {
local path=$1
local filename=$(fixture_get_one_path local $path)
local content=$(cat $work_path/$filename | base64 -w0)
test "$content" = "$ONEPIXEL"
}
function fixture_create() {
local work_path=$DIR/forgejo-work-path
rm -fr $DIR/fixture
mkdir -p $DIR/fixture
(
cd $DIR/fixture
git init
git checkout -b main
git remote add origin http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}
git config user.email root@example.com
git config user.name username
echo SOMETHING > README
git add README
git commit -m 'initial commit'
git push --set-upstream --force origin main
)
for fun in ${STORAGE_FUN} ; do
fixture_${fun}_create
done
}

32
.forgejo/upgrades/legagy-relative-app.ini Normal file
View file

@ -0,0 +1,32 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/data
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
LFS_CONTENT_PATH = relative-lfs
[database]
DB_TYPE = sqlite3
PATH = ${WORK_PATH}/forgejo.db
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[picture]
AVATAR_UPLOAD_PATH = relative-avatars
REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars

32
.forgejo/upgrades/merged-app.ini Normal file
View file

@ -0,0 +1,32 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/data
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true
[storage]
PATH = ${WORK_PATH}/merged

59
.forgejo/upgrades/misplace-app.ini Normal file
View file

@ -0,0 +1,59 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/elsewhere
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true
[attachment]
[storage.attachments]
PATH = ${WORK_PATH}/data/attachments
[lfs]
[storage.lfs]
PATH = ${WORK_PATH}/data/lfs
[avatar]
[storage.avatars]
PATH = ${WORK_PATH}/data/avatars
[repo-avatar]
[storage.repo-avatars]
PATH = ${WORK_PATH}/data/repo-avatars
[repo-archive]
[storage.repo-archive]
PATH = ${WORK_PATH}/data/repo-archive
[packages]
[storage.packages]
PATH = ${WORK_PATH}/data/packages

89
.forgejo/upgrades/misplace-s3-app.ini Normal file
View file

@ -0,0 +1,89 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/elsewhere
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true
[attachment]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[lfs]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[repo-avatar]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[avatar]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[repo-archive]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[packages]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false

44
.forgejo/upgrades/relative-app.ini Normal file
View file

@ -0,0 +1,44 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/data
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[attachment]
PATH = relative-attachments
[lfs]
PATH = relative-lfs
[avatar]
PATH = relative-avatars
[repo-avatar]
PATH = relative-repo-avatars
[repo-archive]
PATH = relative-repo-archive
[packages]
PATH = relative-packages

47
.forgejo/upgrades/specific-app.ini Normal file
View file

@ -0,0 +1,47 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/elsewhere
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true
[attachment]
PATH = ${WORK_PATH}/data/attachments
[lfs]
PATH = ${WORK_PATH}/data/lfs
[avatar]
PATH = ${WORK_PATH}/data/avatars
[repo-avatar]
PATH = ${WORK_PATH}/data/repo-avatars
[repo-archive]
PATH = ${WORK_PATH}/data/repo-archive
[packages]
PATH = ${WORK_PATH}/data/packages

39
.forgejo/upgrades/stable-s3-app.ini Normal file
View file

@ -0,0 +1,39 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/elsewhere
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[actions]
ENABLED = true
[storage]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = 127.0.0.1:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = forgejo
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false

44
.forgejo/upgrades/storage-relative-app.ini Normal file
View file

@ -0,0 +1,44 @@
RUN_MODE = prod
WORK_PATH = ${WORK_PATH}
[server]
APP_DATA_PATH = ${WORK_PATH}/data
HTTP_PORT = 3000
SSH_LISTEN_PORT = 2222
LFS_START_SERVER = true
[database]
DB_TYPE = sqlite3
[log]
MODE = file
LEVEL = debug
ROUTER = file
[log.file]
FILE_NAME = forgejo.log
[security]
INSTALL_LOCK = true
[repository]
ENABLE_PUSH_CREATE_USER = true
DEFAULT_PUSH_CREATE_PRIVATE = false
[storage.attachments]
PATH = relative-attachments
[storage.lfs]
PATH = relative-lfs
[storage.avatars]
PATH = relative-avatars
[storage.repo-avatars]
PATH = relative-repo-avatars
[storage.repo-archive]
PATH = relative-repo-archive
[storage.packages]
PATH = relative-packages

590
.forgejo/upgrades/test-upgrade.sh Executable file
View file

@ -0,0 +1,590 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
#
# Debug loop from the source tree:
#
# ./.forgejo/upgrades/test-upgrade.sh dependencies
# ./.forgejo/upgrades/test-upgrade.sh build_all
# VERBOSE=true ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails
#
# Everything happens in /tmp/forgejo-upgrades
#
PREFIX===============
HOST_PORT=0.0.0.0:3000
STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars"
STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars"
DIR=/tmp/forgejo-upgrades
if ${VERBOSE:-false} ; then
set -ex
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
else
set -e
fi
SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
: ${FORGEJO_USER:=root}
: ${FORGEJO_REPO:=fixture}
: ${FORGEJO_PASSWORD:=admin1234}
source $SELF_DIR/fixtures.sh
function maybe_sudo() {
if test $(id -u) != 0 ; then
SUDO=sudo
fi
}
function log_info() {
echo "$PREFIX $@"
}
function dependencies() {
maybe_sudo
if ! which curl daemon jq git-lfs > /dev/null ; then
$SUDO apt-get install -y -qq curl daemon git-lfs jq
fi
if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio > /dev/null ; then
$SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
$SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-08-23T10-07-06Z -o /usr/local/bin/minio
fi
if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio > /dev/null ; then
$SUDO chmod +x /usr/local/bin/mc
$SUDO chmod +x /usr/local/bin/minio
fi
if ! test -f /usr/local/bin/garage > /dev/null ; then
$SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage
fi
if ! test -x /usr/local/bin/garage > /dev/null ; then
$SUDO chmod +x /usr/local/bin/garage
fi
}
function build() {
local version=$1
local semver=$2
if ! test -f $DIR/forgejo-$version ; then
mkdir -p $DIR
make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea
mv gitea $DIR/forgejo-$version
fi
}
function build_all() {
test -f Makefile
build 1.21.0-0 6.0.0+0-gitea-1.21.0
}
function retry() {
rm -f $DIR/wait-for.out
success=false
for delay in 1 1 5 5 15 ; do
if "$@" >> $DIR/wait-for.out 2>&1 ; then
success=true
break
fi
cat $DIR/wait-for.out
echo waiting $delay
sleep $delay
done
if test $success = false ; then
cat $DIR/wait-for.out
return 1
fi
}
function download() {
local version=$1
if ! test -f $DIR/forgejo-$version ; then
mkdir -p $DIR
wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64
chmod +x $DIR/forgejo-$version
fi
}
function cleanup_logs() {
local work_path=$DIR/forgejo-work-path
rm -f $DIR/*.log
rm -f $work_path/log/*.log
}
function clobber() {
rm -fr /tmp/forgejo-upgrades
}
function start_forgejo() {
local version=$1
download $version
local work_path=$DIR/forgejo-work-path
daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path
if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then
cat $DIR/*.log
cat $work_path/log/*.log
return 1
fi
create_user $version
}
function start_minio() {
mkdir -p $DIR/minio
daemon --chdir=$DIR --unsafe \
--env="PATH=$PATH" \
--env=MINIO_ROOT_USER=123456 \
--env=MINIO_ROOT_PASSWORD=12345678 \
--env=MINIO_VOLUMES=$DIR/minio \
--pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server
retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678
}
function start_garage() {
mkdir -p $DIR/garage/{data,meta}
cat > $DIR/garage/garage.toml <<EOF
metadata_dir = "$DIR/garage/meta"
data_dir = "$DIR/garage/data"
db_engine = "lmdb"
replication_mode = "none"
rpc_bind_addr = "127.0.0.1:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "$(openssl rand -hex 32)"
[s3_api]
s3_region = "us-east-1"
api_bind_addr = "127.0.0.1:9000"
root_domain = ".s3.garage.localhost"
[s3_web]
bind_addr = "127.0.0.1:3902"
root_domain = ".web.garage.localhost"
index = "index.html"
[k2v_api]
api_bind_addr = "127.0.0.1:3904"
[admin]
api_bind_addr = "127.0.0.1:3903"
admin_token = "$(openssl rand -base64 32)"
EOF
daemon --chdir=$DIR --unsafe \
--env="PATH=$PATH" \
--env=RUST_LOG=garage_api=debug \
--pidfile=$DIR/garage-pid --errlog=$DIR/garage-err.log --output=$DIR/garage-out.log -- /usr/local/bin/garage -c $DIR/garage/garage.toml server
retry garage -c $DIR/garage/garage.toml status
garage -c $DIR/garage/garage.toml layout assign -z dc1 -c 1 $(garage -c $DIR/garage/garage.toml status | tail -1 | grep -o '[0-9a-z]*' | head -1)
ver=$(garage -c $DIR/garage/garage.toml layout show | grep -oP '(?<=Current cluster layout version: )\d+')
garage -c $DIR/garage/garage.toml layout apply --version $((ver+1))
garage -c $DIR/garage/garage.toml key info test || garage -c $DIR/garage/garage.toml key import -n test 123456 12345678
garage -c $DIR/garage/garage.toml key allow --create-bucket test
retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678
}
function start_s3() {
local s3_backend=$1
start_$s3_backend
}
function start() {
local version=$1
local s3_backend=${2:-minio}
start_s3 $s3_backend
start_forgejo $version
}
function create_user() {
local version=$1
local work_path=$DIR/forgejo-work-path
if test -f $work_path/forgejo-token; then
return
fi
local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path"
$cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email "$FORGEJO_USER@example.com"
local scopes="--scopes all"
if echo $version | grep --quiet 1.18. ; then
scopes=""
fi
#
# forgejo-cli is to use with api/v1 enpoints
#
# tail -1 is because there are logs creating noise in the output in v1.19.4-0
#
$cli admin user generate-access-token -u $FORGEJO_USER --raw $scopes | tail -1 > $work_path/forgejo-token
( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header
( echo "#!/bin/sh" ; echo 'curl -f -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api
$work_path/forgejo-api http://${HOST_PORT}/api/v1/version
#
# forgejo-client is to use with web endpoints
#
#
# login and obtain a CSRF, all stored in the cookie file
#
( echo "#!/bin/sh" ; echo 'curl --cookie-jar '$DIR/cookies' --cookie '$DIR/cookies' -f -sS "$@"' ) > $work_path/forgejo-client-update-cookies && chmod +x $work_path/forgejo-client-update-cookies
$work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null
$work_path/forgejo-client-update-cookies --verbose -X POST --data user_name=${FORGEJO_USER} --data password=${FORGEJO_PASSWORD} http://${HOST_PORT}/user/login >& $DIR/login.html
$work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null
local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DIR/cookies)
#
# use the cookie file but do not modify it
#
( echo "#!/bin/sh" ; echo 'curl --cookie '$DIR/cookies' -H "X-Csrf-Token: '$csrf'" -f -sS "$@"' ) > $work_path/forgejo-client && chmod +x $work_path/forgejo-client
}
function stop_daemon() {
local daemon=$1
if test -f $DIR/$daemon-pid ; then
local pid=$(cat $DIR/$daemon-pid)
kill -TERM $pid
pidwait $pid || true
for delay in 1 1 2 2 5 5 ; do
if ! test -f $DIR/$daemon-pid ; then
break
fi
sleep $delay
done
! test -f $DIR/$daemon-pid
fi
}
function stop() {
stop_daemon forgejo
stop_daemon minio
stop_daemon garage
cleanup_logs
}
function reset_forgejo() {
local config=$1
local work_path=$DIR/forgejo-work-path
rm -fr $work_path
mkdir -p $work_path
WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini
}
function reset_minio() {
rm -fr $DIR/minio
}
function reset_garage() {
rm -fr $DIR/garage
}
function reset() {
local config=$1
reset_forgejo $config
reset_minio
reset_garage
}
function verify_storage() {
local work_path=$DIR/forgejo-work-path
for path in ${STORAGE_PATHS} ; do
test -d $work_path/data/$path
done
}
function cleanup_storage() {
local work_path=$DIR/forgejo-work-path
for path in ${STORAGE_PATHS} ; do
rm -fr $work_path/data/$path
done
}
function test_downgrade_1.20.2_fails() {
local work_path=$DIR/forgejo-work-path
log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails"
stop
reset default
start 1.20.3-0
stop
download 1.20.2-0
timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true
if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then
cat $work_path/log/forgejo.log
return 1
fi
}
function test_bug_storage_merged() {
local work_path=$DIR/forgejo-work-path
log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
log_info "using < 1.20.3-0 and [storage].PATH merge all storage"
for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
stop
reset merged
start $version
for path in ${STORAGE_PATHS} ; do
! test -d $work_path/data/$path
done
for path in ${STORAGE_PATHS} ; do
! test -d $work_path/merged/$path
done
test -d $work_path/merged
done
stop
log_info "upgrading from 1.20.2-0 with [storage].PATH fails"
download 1.20.3-0
timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then
cat $work_path/log/forgejo.log
return 1
fi
}
function test_bug_storage_relative_path() {
local work_path=$DIR/forgejo-work-path
log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH"
for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
stop
reset legagy-relative
start $version
test -d $work_path/relative-lfs
test -d $work_path/relative-avatars
test -d $work_path/relative-repo-avatars
done
log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH"
for version in 1.20.3-0 1.21.0-0 ; do
stop
reset legagy-relative
start $version
test -d $work_path/data/relative-lfs
test -d $work_path/data/relative-avatars
test -d $work_path/data/relative-repo-avatars
done
log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH"
for version in 1.20.3-0 1.21.0-0 ; do
stop
reset storage-relative
start $version
for path in ${STORAGE_PATHS} ; do
test -d $work_path/data/relative-$path
done
done
log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent"
for version in 1.20.2-0 ; do
stop
reset storage-relative
start $version
test -d $work_path/data/packages
test -d $work_path/relative-repo-archive
test -d $work_path/relative-attachments
test -d $work_path/relative-lfs
test -d $work_path/data/avatars
test -d $work_path/data/repo-avatars
done
log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent"
for version in 1.18.5-0 1.19.4-0 ; do
stop
reset storage-relative
start $version
test -d $work_path/relative-packages
test -d $work_path/relative-repo-archive
test -d $work_path/relative-attachments
test -d $work_path/data/lfs
test -d $work_path/data/avatars
test -d $work_path/data/repo-avatars
done
log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH"
for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
stop
reset relative
start $version
for path in ${STORAGE_PATHS} ; do
test -d $work_path/relative-$path
done
done
log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH"
for version in 1.20.3-0 1.21.0-0 ; do
stop
reset relative
start $version
for path in ${STORAGE_PATHS} ; do
test -d $work_path/data/relative-$path
done
done
stop
}
function test_bug_storage_s3_misplace() {
local work_path=$DIR/forgejo-work-path
local s3_backend=${2:-minio}
log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
for version in 1.20.2-0 1.20.3-0 ; do
log_info "Forgejo $version & $s3_backend"
stop
reset misplace-s3
start $version $s3_backend
fixture_create
for fun in ${STORAGE_FUN} ; do
fixture_${fun}_assert_s3
done
done
for version in 1.18.5-0 1.19.4-0 ; do
log_info "Forgejo $version & $s3_backend"
stop
reset misplace-s3
start $version $s3_backend
fixture_create
#
# some storage are in S3
#
fixture_attachments_assert_s3
fixture_lfs_assert_s3
#
# others are in local
#
fixture_repo_archive_assert_local elsewhere/repo-archive
fixture_avatars_assert_local elsewhere/avatars
fixture_packages_assert_local elsewhere/packages
fixture_repo_avatars_assert_local elsewhere/repo-avatars
done
}
function test_storage_stable_s3() {
local work_path=$DIR/forgejo-work-path
local s3_backend=${1:-minio}
log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 ; do
log_info "Forgejo $version & $s3_backend"
stop
reset stable-s3
start $version $s3_backend
fixture_create
for fun in ${STORAGE_FUN} ; do
fixture_${fun}_assert_s3
done
done
}
function test_bug_storage_misplace() {
local work_path=$DIR/forgejo-work-path
log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
log_info "using < 1.20 and conflicting sections misplace storage"
for version in 1.18.5-0 1.19.4-0 ; do
stop
reset misplace
start $version
#
# some storage are where they should be
#
test -d $work_path/data/packages
test -d $work_path/data/repo-archive
test -d $work_path/data/attachments
#
# others are under APP_DATA_PATH
#
test -d $work_path/elsewhere/lfs
test -d $work_path/elsewhere/avatars
test -d $work_path/elsewhere/repo-avatars
done
log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]"
for version in 1.20.2-0 ; do
stop
reset misplace
start $version
for path in ${STORAGE_PATHS} ; do
test -d $work_path/elsewhere/$path
done
done
stop
log_info "upgrading from 1.20.2-0 with conflicting sections fails"
download 1.20.3-0
timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
for path in ${STORAGE_PATHS} ; do
if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then
cat $work_path/log/forgejo.log
return 1
fi
done
}
function test_successful_upgrades() {
for config in default specific ; do
log_info "using $config app.ini"
reset $config
for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do
log_info "run $version"
cleanup_storage
start $version
verify_storage
stop
done
done
}
function run() {
local fun=$1
shift
echo Start running $fun
mkdir -p $DIR
> $DIR/$fun.out
tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" &
pid=$!
if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then
kill $pid
cat $DIR/$fun.out
echo Failure running $fun
return 1
fi
kill $pid
echo Success running $fun
}
function test_upgrades() {
run stop
run dependencies
run build_all
run test_successful_upgrades
run test_bug_storage_misplace
run test_bug_storage_merged
run test_downgrade_1.20.2_fails
run test_bug_storage_s3_misplace
run test_storage_stable_s3 minio
run test_storage_stable_s3 garage
}
"$@"

45
.forgejo/workflows/upgrade.yml Normal file
View file

@ -0,0 +1,45 @@
name: upgrade
on:
pull_request_review:
push:
branches:
- 'forgejo*'
- 'v*/forgejo*'
jobs:
upgrade:
runs-on: docker
container:
image: codeberg.org/forgejo/test_env:main
steps:
- run: apt-get install -y -qq zstd
- name: cache S3 binaries
id: S3
uses: https://code.forgejo.org/actions/cache@v3
with:
path: |
/usr/local/bin/minio
/usr/local/bin/mc
/usr/local/bin/garage
key: S3
- name: skip if S3 cache hit
if: steps.S3.outputs.cache-hit != 'true'
run: echo no hit
- uses: https://code.forgejo.org/actions/checkout@v3
- uses: https://code.forgejo.org/actions/setup-go@v4
with:
go-version: "1.21"
- run: |
git config --add safe.directory '*'
chown -R gitea:gitea . /go
- run: |
su gitea -c 'make deps-backend'
- run: |
script=$(pwd)/.forgejo/upgrades/test-upgrade.sh
$script run dependencies
$script clobber
su gitea -c "$script test_upgrades"