Template
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo synced 2024-11-25 19:26:09 +01:00
forgejo/tests/integration/quota_use_test.go

1148 lines
32 KiB
Go
Raw Normal View History

feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
// Copyright 2024 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package integration
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"code.gitea.io/gitea/models/db"
org_model "code.gitea.io/gitea/models/organization"
quota_model "code.gitea.io/gitea/models/quota"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/routers"
forgejo_context "code.gitea.io/gitea/services/context"
repo_service "code.gitea.io/gitea/services/repository"
"code.gitea.io/gitea/tests"
gouuid "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestWebQuotaEnforcementRepoMigrate(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
env.RunVisitAndPostToPageTests(t, "/repo/migrate", &Payload{
"repo_name": "migration-test",
"clone_addr": env.Users.Limited.Repo.Link() + ".git",
"service": fmt.Sprintf("%d", api.ForgejoService),
}, http.StatusOK)
})
}
func TestWebQuotaEnforcementRepoCreate(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
env.RunVisitAndPostToPageTests(t, "/repo/create", nil, http.StatusOK)
})
}
func TestWebQuotaEnforcementRepoFork(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
page := fmt.Sprintf("%s/fork", env.Users.Limited.Repo.Link())
env.RunVisitAndPostToPageTests(t, page, &Payload{
"repo_name": "fork-test",
}, http.StatusSeeOther)
})
}
func TestWebQuotaEnforcementIssueAttachment(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
// Uploading to our repo => 413
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
CreateIssueAttachment("test.txt").
ExpectStatus(http.StatusRequestEntityTooLarge)
// Uploading to the limited org repo => 413
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
CreateIssueAttachment("test.txt").
ExpectStatus(http.StatusRequestEntityTooLarge)
// Uploading to the unlimited org repo => 200
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Unlimited.Repo}).
CreateIssueAttachment("test.txt").
ExpectStatus(http.StatusOK)
})
}
func TestWebQuotaEnforcementMirrorSync(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
var mirrorRepo *repo_model.Repository
env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
mirrorRepo = ctx.CreateMirror()
}).
With(Context{
Repo: mirrorRepo,
Payload: &Payload{"action": "mirror-sync"},
}).
PostToPage(mirrorRepo.Link() + "/settings").
ExpectStatus(http.StatusOK).
ExpectFlashMessage("Quota exceeded, not pulling changes.")
})
}
func TestWebQuotaEnforcementRepoContentEditing(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
// We're only going to test the GET requests here, because the entire combo
// is covered by a route check.
// Lets create a helper!
runCheck := func(t *testing.T, path string, successStatus int) {
t.Run("#"+path, func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Uploading to a limited user's repo => 413
env.As(t, env.Users.Limited).
VisitPage(env.Users.Limited.Repo.Link() + path).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Limited org => 413
env.As(t, env.Users.Limited).
VisitPage(env.Orgs.Limited.Repo.Link() + path).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Unlimited org => 200
env.As(t, env.Users.Limited).
VisitPage(env.Orgs.Unlimited.Repo.Link() + path).
ExpectStatus(successStatus)
})
}
paths := []string{
"/_new/main",
"/_edit/main/README.md",
"/_delete/main",
"/_upload/main",
"/_diffpatch/main",
}
for _, path := range paths {
runCheck(t, path, http.StatusOK)
}
// Run another check for `_cherrypick`. It's cumbersome to dig out a valid
// commit id, so we'll use a fake, and treat 404 as a success: it's not 413,
// and that's all we care about for this test.
runCheck(t, "/_cherrypick/92cfceb39d57d914ed8b14d0e37643de0797ae56/main", http.StatusNotFound)
})
}
func TestWebQuotaEnforcementRepoBranches(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
t.Run("create", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
runTest := func(t *testing.T, path string) {
t.Run("#"+path, func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
env.As(t, env.Users.Limited).
With(Context{Payload: &Payload{"new_branch_name": "quota"}}).
PostToRepoPage("/branches/_new" + path).
ExpectStatus(http.StatusRequestEntityTooLarge)
env.As(t, env.Users.Limited).
With(Context{
Payload: &Payload{"new_branch_name": "quota"},
Repo: env.Orgs.Limited.Repo,
}).
PostToRepoPage("/branches/_new" + path).
ExpectStatus(http.StatusRequestEntityTooLarge)
env.As(t, env.Users.Limited).
With(Context{
Payload: &Payload{"new_branch_name": "quota"},
Repo: env.Orgs.Unlimited.Repo,
}).
PostToRepoPage("/branches/_new" + path).
ExpectStatus(http.StatusNotFound)
})
}
// We're testing the first two against things that don't exist, so that
// all three consistently return 404 if no quota enforcement happens.
runTest(t, "/branch/no-such-branch")
runTest(t, "/tag/no-such-tag")
runTest(t, "/commit/92cfceb39d57d914ed8b14d0e37643de0797ae56")
})
t.Run("delete & restore", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
ctx.With(Context{Payload: &Payload{"new_branch_name": "to-delete"}}).
PostToRepoPage("/branches/_new/branch/main").
ExpectStatus(http.StatusSeeOther)
})
env.As(t, env.Users.Limited).
PostToRepoPage("/branches/delete?name=to-delete").
ExpectStatus(http.StatusOK)
env.As(t, env.Users.Limited).
PostToRepoPage("/branches/restore?name=to-delete").
ExpectStatus(http.StatusOK)
})
})
}
func TestWebQuotaEnforcementRepoReleases(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
env.RunVisitAndPostToRepoPageTests(t, "/releases/new", &Payload{
"tag_name": "quota",
"tag_target": "main",
"title": "test release",
}, http.StatusSeeOther)
t.Run("attachments", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Uploading to our repo => 413
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
CreateReleaseAttachment("test.txt").
ExpectStatus(http.StatusRequestEntityTooLarge)
// Uploading to the limited org repo => 413
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
CreateReleaseAttachment("test.txt").
ExpectStatus(http.StatusRequestEntityTooLarge)
// Uploading to the unlimited org repo => 200
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Unlimited.Repo}).
CreateReleaseAttachment("test.txt").
ExpectStatus(http.StatusOK)
})
})
}
func TestWebQuotaEnforcementRepoPulls(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
// To create a pull request, we first fork the two limited repos into the
// unlimited org.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
ForkRepoInto(env.Orgs.Unlimited)
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
ForkRepoInto(env.Orgs.Unlimited)
// Then, create pull requests from the forks, back to the main repos
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
CreatePullFrom(env.Orgs.Unlimited)
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
CreatePullFrom(env.Orgs.Unlimited)
// Trying to merge the pull request will fail for both, though, due to being
// over quota.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
With(Context{Payload: &Payload{"do": "merge"}}).
PostToRepoPage("/pulls/1/merge").
ExpectStatus(http.StatusRequestEntityTooLarge)
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
With(Context{Payload: &Payload{"do": "merge"}}).
PostToRepoPage("/pulls/1/merge").
ExpectStatus(http.StatusRequestEntityTooLarge)
})
}
func TestWebQuotaEnforcementRepoTransfer(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
t.Run("direct transfer", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Trying to transfer the repository to a limited organization fails.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
With(Context{Payload: &Payload{
"action": "transfer",
"repo_name": env.Users.Limited.Repo.FullName(),
"new_owner_name": env.Orgs.Limited.Org.Name,
}}).
PostToRepoPage("/settings").
ExpectStatus(http.StatusOK).
ExpectFlashMessageContains("over quota", "The repository has not been transferred")
// Trying to transfer to a different, also limited user, also fails.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
With(Context{Payload: &Payload{
"action": "transfer",
"repo_name": env.Users.Limited.Repo.FullName(),
"new_owner_name": env.Users.Contributor.User.Name,
}}).
PostToRepoPage("/settings").
ExpectStatus(http.StatusOK).
ExpectFlashMessageContains("over quota", "The repository has not been transferred")
})
t.Run("accept & reject", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Trying to transfer to a different user, with quota lifted, starts the transfer
env.As(t, env.Users.Contributor).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
env.As(ctx.t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
With(Context{Payload: &Payload{
"action": "transfer",
"repo_name": env.Users.Limited.Repo.FullName(),
"new_owner_name": env.Users.Contributor.User.Name,
}}).
PostToRepoPage("/settings").
ExpectStatus(http.StatusSeeOther).
ExpectFlashCookieContains("This repository has been marked for transfer and awaits confirmation")
})
// Trying to accept the transfer, with quota in effect, fails
env.As(t, env.Users.Contributor).
With(Context{Repo: env.Users.Limited.Repo}).
PostToRepoPage("/action/accept_transfer").
ExpectStatus(http.StatusRequestEntityTooLarge)
// Rejecting the transfer, however, succeeds.
env.As(t, env.Users.Contributor).
With(Context{Repo: env.Users.Limited.Repo}).
PostToRepoPage("/action/reject_transfer").
ExpectStatus(http.StatusSeeOther)
})
})
}
func TestGitQuotaEnforcement(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
// Lets create a little helper that runs a task for three of our repos: the
// user's repo, the limited org repo, and the unlimited org's.
//
// We expect the last one to always work, and the expected status of the
// other two is decided by the caller.
runTestForAllRepos := func(t *testing.T, task func(t *testing.T, repo *repo_model.Repository) error, expectSuccess bool) {
t.Helper()
err := task(t, env.Users.Limited.Repo)
if expectSuccess {
require.NoError(t, err)
} else {
require.Error(t, err)
}
err = task(t, env.Orgs.Limited.Repo)
if expectSuccess {
require.NoError(t, err)
} else {
require.Error(t, err)
}
err = task(t, env.Orgs.Unlimited.Repo)
require.NoError(t, err)
}
// Run tests with quotas disabled
runTestForAllReposWithQuotaDisabled := func(t *testing.T, task func(t *testing.T, repo *repo_model.Repository) error) {
t.Helper()
t.Run("with quota disabled", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
defer test.MockVariableValue(&setting.Quota.Enabled, false)()
defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
runTestForAllRepos(t, task, true)
})
}
t.Run("push branch", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Pushing a new branch is denied if the user is over quota.
runTestForAllRepos(t, func(t *testing.T, repo *repo_model.Repository) error {
return env.As(t, env.Users.Limited).
With(Context{Repo: repo}).
LocalClone(u).
Push("HEAD:new-branch")
}, false)
// Pushing a new branch is always allowed if quota is disabled
runTestForAllReposWithQuotaDisabled(t, func(t *testing.T, repo *repo_model.Repository) error {
return env.As(t, env.Users.Limited).
With(Context{Repo: repo}).
LocalClone(u).
Push("HEAD:new-branch-wo-quota")
})
})
t.Run("push tag", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Pushing a tag is denied if the user is over quota.
runTestForAllRepos(t, func(t *testing.T, repo *repo_model.Repository) error {
return env.As(t, env.Users.Limited).
With(Context{Repo: repo}).
LocalClone(u).
Tag("new-tag").
Push("new-tag")
}, false)
// ...but succeeds if the quota feature is disabled
runTestForAllReposWithQuotaDisabled(t, func(t *testing.T, repo *repo_model.Repository) error {
return env.As(t, env.Users.Limited).
With(Context{Repo: repo}).
LocalClone(u).
Tag("new-tag-wo-quota").
Push("new-tag-wo-quota")
})
})
t.Run("Agit PR", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Opening an Agit PR is *always* accepted. At least for now.
runTestForAllRepos(t, func(t *testing.T, repo *repo_model.Repository) error {
return env.As(t, env.Users.Limited).
With(Context{Repo: repo}).
LocalClone(u).
Push("HEAD:refs/for/main/agit-pr-branch")
}, true)
})
t.Run("delete branch", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Deleting a branch is respected, and allowed.
err := env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
err := ctx.
LocalClone(u).
Push("HEAD:branch-to-delete")
require.NoError(ctx.t, err)
}).
Push(":branch-to-delete")
require.NoError(t, err)
})
t.Run("delete tag", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Deleting a tag is always allowed.
err := env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
err := ctx.
LocalClone(u).
Tag("tag-to-delete").
Push("tag-to-delete")
require.NoError(ctx.t, err)
}).
Push(":tag-to-delete")
require.NoError(t, err)
})
t.Run("mixed push", func(t *testing.T) {
t.Run("all deletes", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Pushing multiple deletes is allowed.
err := env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
err := ctx.
LocalClone(u).
Tag("mixed-push-tag").
Push("mixed-push-tag", "HEAD:mixed-push-branch")
require.NoError(ctx.t, err)
}).
Push(":mixed-push-tag", ":mixed-push-branch")
require.NoError(t, err)
})
t.Run("new & delete", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// Pushing a mix of deletions & a new branch is rejected together.
err := env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
err := ctx.
LocalClone(u).
Tag("mixed-push-tag").
Push("mixed-push-tag", "HEAD:mixed-push-branch")
require.NoError(ctx.t, err)
}).
Push(":mixed-push-tag", ":mixed-push-branch", "HEAD:mixed-push-branch-new")
require.Error(t, err)
// ...unless quota is disabled
t.Run("with quota disabled", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
defer test.MockVariableValue(&setting.Quota.Enabled, false)()
defer test.MockVariableValue(&testWebRoutes, routers.NormalRoutes())()
err := env.As(t, env.Users.Limited).
WithoutQuota(func(ctx *quotaWebEnvAsContext) {
err := ctx.
LocalClone(u).
Tag("mixed-push-tag-2").
Push("mixed-push-tag-2", "HEAD:mixed-push-branch-2")
require.NoError(ctx.t, err)
}).
Push(":mixed-push-tag-2", ":mixed-push-branch-2", "HEAD:mixed-push-branch-new-2")
require.NoError(t, err)
})
})
})
})
}
func TestQuotaConfigDefault(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
env := createQuotaWebEnv(t)
defer env.Cleanup()
t.Run("with config-based default", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
defer test.MockVariableValue(&setting.Quota.Default.Total, 0)()
env.As(t, env.Users.Ungrouped).
With(Context{
Payload: &Payload{
"uid": env.Users.Ungrouped.ID().AsString(),
"repo_name": "quota-config-default",
},
}).
PostToPage("/repo/create").
ExpectStatus(http.StatusRequestEntityTooLarge)
})
t.Run("without config-based default", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
env.As(t, env.Users.Ungrouped).
With(Context{
Payload: &Payload{
"uid": env.Users.Ungrouped.ID().AsString(),
"repo_name": "quota-config-default",
},
}).
PostToPage("/repo/create").
ExpectStatus(http.StatusSeeOther)
})
})
}
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
/**********************
* Here be dragons! *
* *
* . *
* .> )\;`a__ *
* ( _ _)/ /-." ~~ *
* `( )_ )/ *
* <_ <_ sb/dwb *
**********************/
type quotaWebEnv struct {
Users quotaWebEnvUsers
Orgs quotaWebEnvOrgs
cleaners []func()
}
type quotaWebEnvUsers struct {
Limited quotaWebEnvUser
Contributor quotaWebEnvUser
Ungrouped quotaWebEnvUser
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
}
type quotaWebEnvOrgs struct {
Limited quotaWebEnvOrg
Unlimited quotaWebEnvOrg
}
type quotaWebEnvOrg struct {
Org *org_model.Organization
Repo *repo_model.Repository
QuotaGroup *quota_model.Group
QuotaRule *quota_model.Rule
}
type quotaWebEnvUser struct {
User *user_model.User
Session *TestSession
Repo *repo_model.Repository
QuotaGroup *quota_model.Group
QuotaRule *quota_model.Rule
}
type Payload map[string]string
type quotaWebEnvAsContext struct {
t *testing.T
Doer *quotaWebEnvUser
Repo *repo_model.Repository
Payload Payload
CSRFPath *string
gitPath string
request *RequestWrapper
response *httptest.ResponseRecorder
}
type Context struct {
Repo *repo_model.Repository
Payload *Payload
CSRFPath *string
}
func (ctx *quotaWebEnvAsContext) With(opts Context) *quotaWebEnvAsContext {
if opts.Repo != nil {
ctx.Repo = opts.Repo
}
if opts.Payload != nil {
for key, value := range *opts.Payload {
ctx.Payload[key] = value
}
}
if opts.CSRFPath != nil {
ctx.CSRFPath = opts.CSRFPath
}
return ctx
}
func (ctx *quotaWebEnvAsContext) VisitPage(page string) *quotaWebEnvAsContext {
ctx.t.Helper()
ctx.request = NewRequest(ctx.t, "GET", page)
return ctx
}
func (ctx *quotaWebEnvAsContext) VisitRepoPage(page string) *quotaWebEnvAsContext {
ctx.t.Helper()
return ctx.VisitPage(ctx.Repo.Link() + page)
}
func (ctx *quotaWebEnvAsContext) ExpectStatus(status int) *quotaWebEnvAsContext {
ctx.t.Helper()
ctx.response = ctx.Doer.Session.MakeRequest(ctx.t, ctx.request, status)
return ctx
}
func (ctx *quotaWebEnvAsContext) ExpectFlashMessage(value string) {
ctx.t.Helper()
htmlDoc := NewHTMLParser(ctx.t, ctx.response.Body)
flashMessage := strings.TrimSpace(htmlDoc.Find(`.flash-message`).Text())
assert.EqualValues(ctx.t, value, flashMessage)
}
func (ctx *quotaWebEnvAsContext) ExpectFlashMessageContains(parts ...string) {
ctx.t.Helper()
htmlDoc := NewHTMLParser(ctx.t, ctx.response.Body)
flashMessage := strings.TrimSpace(htmlDoc.Find(`.flash-message`).Text())
for _, part := range parts {
assert.Contains(ctx.t, flashMessage, part)
}
}
func (ctx *quotaWebEnvAsContext) ExpectFlashCookieContains(parts ...string) {
ctx.t.Helper()
flashCookie := ctx.Doer.Session.GetCookie(forgejo_context.CookieNameFlash)
assert.NotNil(ctx.t, flashCookie)
// Need to decode the cookie twice
flashValue, err := url.QueryUnescape(flashCookie.Value)
require.NoError(ctx.t, err)
flashValue, err = url.QueryUnescape(flashValue)
require.NoError(ctx.t, err)
for _, part := range parts {
assert.Contains(ctx.t, flashValue, part)
}
}
func (ctx *quotaWebEnvAsContext) ForkRepoInto(org quotaWebEnvOrg) {
ctx.t.Helper()
ctx.
With(Context{Payload: &Payload{
"uid": org.ID().AsString(),
"repo_name": ctx.Repo.Name + "-fork",
}}).
PostToRepoPage("/fork").
ExpectStatus(http.StatusSeeOther)
}
func (ctx *quotaWebEnvAsContext) CreatePullFrom(org quotaWebEnvOrg) {
ctx.t.Helper()
url := fmt.Sprintf("/compare/main...%s:main", org.Org.Name)
ctx.
With(Context{Payload: &Payload{
"title": "PR test",
}}).
PostToRepoPage(url).
ExpectStatus(http.StatusOK)
}
func (ctx *quotaWebEnvAsContext) PostToPage(page string) *quotaWebEnvAsContext {
ctx.t.Helper()
payload := ctx.Payload
csrfPath := page
if ctx.CSRFPath != nil {
csrfPath = *ctx.CSRFPath
}
payload["_csrf"] = GetCSRF(ctx.t, ctx.Doer.Session, csrfPath)
ctx.request = NewRequestWithValues(ctx.t, "POST", page, payload)
return ctx
}
func (ctx *quotaWebEnvAsContext) PostToRepoPage(page string) *quotaWebEnvAsContext {
ctx.t.Helper()
csrfPath := ctx.Repo.Link()
return ctx.With(Context{CSRFPath: &csrfPath}).PostToPage(ctx.Repo.Link() + page)
}
func (ctx *quotaWebEnvAsContext) CreateAttachment(filename, attachmentType string) *quotaWebEnvAsContext {
ctx.t.Helper()
body := &bytes.Buffer{}
image := generateImg()
// Setup multi-part
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", filename)
require.NoError(ctx.t, err)
_, err = io.Copy(part, &image)
require.NoError(ctx.t, err)
err = writer.Close()
require.NoError(ctx.t, err)
csrf := GetCSRF(ctx.t, ctx.Doer.Session, ctx.Repo.Link())
ctx.request = NewRequestWithBody(ctx.t, "POST", fmt.Sprintf("%s/%s/attachments", ctx.Repo.Link(), attachmentType), body)
ctx.request.Header.Add("X-Csrf-Token", csrf)
ctx.request.Header.Add("Content-Type", writer.FormDataContentType())
return ctx
}
func (ctx *quotaWebEnvAsContext) CreateIssueAttachment(filename string) *quotaWebEnvAsContext {
ctx.t.Helper()
return ctx.CreateAttachment(filename, "issues")
}
func (ctx *quotaWebEnvAsContext) CreateReleaseAttachment(filename string) *quotaWebEnvAsContext {
ctx.t.Helper()
return ctx.CreateAttachment(filename, "releases")
}
func (ctx *quotaWebEnvAsContext) WithoutQuota(task func(ctx *quotaWebEnvAsContext)) *quotaWebEnvAsContext {
ctx.t.Helper()
defer ctx.Doer.SetQuota(-1)()
task(ctx)
return ctx
}
func (ctx *quotaWebEnvAsContext) CreateMirror() *repo_model.Repository {
ctx.t.Helper()
doer := ctx.Doer.User
repo, err := repo_service.CreateRepositoryDirectly(db.DefaultContext, doer, doer, repo_service.CreateRepoOptions{
Name: "test-mirror",
IsMirror: true,
Status: repo_model.RepositoryBeingMigrated,
})
require.NoError(ctx.t, err)
return repo
}
func (ctx *quotaWebEnvAsContext) LocalClone(u *url.URL) *quotaWebEnvAsContext {
ctx.t.Helper()
gitPath := ctx.t.TempDir()
doGitInitTestRepository(gitPath, git.Sha1ObjectFormat)(ctx.t)
oldPath := u.Path
oldUser := u.User
defer func() {
u.Path = oldPath
u.User = oldUser
}()
u.Path = ctx.Repo.FullName() + ".git"
u.User = url.UserPassword(ctx.Doer.User.LowerName, userPassword)
doGitAddRemote(gitPath, "origin", u)(ctx.t)
ctx.gitPath = gitPath
return ctx
}
func (ctx *quotaWebEnvAsContext) Push(params ...string) error {
ctx.t.Helper()
gitRepo, err := git.OpenRepository(git.DefaultContext, ctx.gitPath)
require.NoError(ctx.t, err)
defer gitRepo.Close()
_, _, err = git.NewCommand(git.DefaultContext, "push", "origin").
AddArguments(git.ToTrustedCmdArgs(params)...).
RunStdString(&git.RunOpts{Dir: ctx.gitPath})
return err
}
func (ctx *quotaWebEnvAsContext) Tag(tagName string) *quotaWebEnvAsContext {
ctx.t.Helper()
gitRepo, err := git.OpenRepository(git.DefaultContext, ctx.gitPath)
require.NoError(ctx.t, err)
defer gitRepo.Close()
_, _, err = git.NewCommand(git.DefaultContext, "tag").
AddArguments(git.ToTrustedCmdArgs([]string{tagName})...).
RunStdString(&git.RunOpts{Dir: ctx.gitPath})
require.NoError(ctx.t, err)
return ctx
}
func (user *quotaWebEnvUser) SetQuota(limit int64) func() {
previousLimit := user.QuotaRule.Limit
user.QuotaRule.Limit = limit
user.QuotaRule.Edit(db.DefaultContext, &limit, nil)
return func() {
user.QuotaRule.Limit = previousLimit
user.QuotaRule.Edit(db.DefaultContext, &previousLimit, nil)
}
}
func (user *quotaWebEnvUser) ID() convertAs {
return convertAs{
asString: fmt.Sprintf("%d", user.User.ID),
}
}
func (org *quotaWebEnvOrg) ID() convertAs {
return convertAs{
asString: fmt.Sprintf("%d", org.Org.ID),
}
}
type convertAs struct {
asString string
}
func (cas convertAs) AsString() string {
return cas.asString
}
func (env *quotaWebEnv) Cleanup() {
for i := len(env.cleaners) - 1; i >= 0; i-- {
env.cleaners[i]()
}
}
func (env *quotaWebEnv) As(t *testing.T, user quotaWebEnvUser) *quotaWebEnvAsContext {
t.Helper()
ctx := quotaWebEnvAsContext{
t: t,
Doer: &user,
Repo: user.Repo,
Payload: Payload{},
}
return &ctx
}
func (env *quotaWebEnv) RunVisitAndPostToRepoPageTests(t *testing.T, page string, payload *Payload, successStatus int) {
t.Helper()
// Visiting the user's repo page fails due to being over quota.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Users.Limited.Repo}).
VisitRepoPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Posting as the limited user, to the limited repo, fails due to being over
// quota.
csrfPath := env.Users.Limited.Repo.Link()
env.As(t, env.Users.Limited).
With(Context{
Payload: payload,
CSRFPath: &csrfPath,
Repo: env.Users.Limited.Repo,
}).
PostToRepoPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Visiting the limited org's repo page fails due to being over quota.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Limited.Repo}).
VisitRepoPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Posting as the limited user, to a limited org's repo, fails for the same
// reason.
csrfPath = env.Orgs.Limited.Repo.Link()
env.As(t, env.Users.Limited).
With(Context{
Payload: payload,
CSRFPath: &csrfPath,
Repo: env.Orgs.Limited.Repo,
}).
PostToRepoPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Visiting the repo page for the unlimited org succeeds.
env.As(t, env.Users.Limited).
With(Context{Repo: env.Orgs.Unlimited.Repo}).
VisitRepoPage(page).
ExpectStatus(http.StatusOK)
// Posting as the limited user, to an unlimited org's repo, succeeds.
csrfPath = env.Orgs.Unlimited.Repo.Link()
env.As(t, env.Users.Limited).
With(Context{
Payload: payload,
CSRFPath: &csrfPath,
Repo: env.Orgs.Unlimited.Repo,
}).
PostToRepoPage(page).
ExpectStatus(successStatus)
}
func (env *quotaWebEnv) RunVisitAndPostToPageTests(t *testing.T, page string, payload *Payload, successStatus int) {
t.Helper()
// Visiting the page is always fine.
env.As(t, env.Users.Limited).
VisitPage(page).
ExpectStatus(http.StatusOK)
// Posting as the Limited user fails, because it is over quota.
env.As(t, env.Users.Limited).
With(Context{Payload: payload}).
With(Context{
Payload: &Payload{
"uid": env.Users.Limited.ID().AsString(),
},
}).
PostToPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Posting to a limited org also fails, for the same reason.
env.As(t, env.Users.Limited).
With(Context{Payload: payload}).
With(Context{
Payload: &Payload{
"uid": env.Orgs.Limited.ID().AsString(),
},
}).
PostToPage(page).
ExpectStatus(http.StatusRequestEntityTooLarge)
// Posting to an unlimited repo works, however.
env.As(t, env.Users.Limited).
With(Context{Payload: payload}).
With(Context{
Payload: &Payload{
"uid": env.Orgs.Unlimited.ID().AsString(),
},
}).
PostToPage(page).
ExpectStatus(successStatus)
}
func createQuotaWebEnv(t *testing.T) *quotaWebEnv {
t.Helper()
// *** helpers ***
makeUngroupedUser := func(t *testing.T) quotaWebEnvUser {
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
t.Helper()
user := quotaWebEnvUser{}
// Create the user
userName := gouuid.NewString()
apiCreateUser(t, userName)
user.User = unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: userName})
user.Session = loginUser(t, userName)
// Create a repository for the user
repo, _, _ := tests.CreateDeclarativeRepoWithOptions(t, user.User, tests.DeclarativeRepoOptions{})
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
user.Repo = repo
return user
}
// Create a user, its quota group & rule
makeUser := func(t *testing.T, limit int64) quotaWebEnvUser {
t.Helper()
user := makeUngroupedUser(t)
userName := user.User.Name
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
// Create a quota group for them
group, err := quota_model.CreateGroup(db.DefaultContext, userName)
require.NoError(t, err)
user.QuotaGroup = group
// Create a rule
rule, err := quota_model.CreateRule(db.DefaultContext, userName, limit, quota_model.LimitSubjects{quota_model.LimitSubjectSizeAll})
require.NoError(t, err)
user.QuotaRule = rule
// Add the rule to the group
err = group.AddRuleByName(db.DefaultContext, rule.Name)
require.NoError(t, err)
// Add the user to the group
err = group.AddUserByID(db.DefaultContext, user.User.ID)
require.NoError(t, err)
return user
}
// Create a user, its quota group & rule
makeOrg := func(t *testing.T, owner *user_model.User, limit int64) quotaWebEnvOrg {
t.Helper()
org := quotaWebEnvOrg{}
// Create the org
userName := gouuid.NewString()
org.Org = &org_model.Organization{
Name: userName,
}
err := org_model.CreateOrganization(db.DefaultContext, org.Org, owner)
require.NoError(t, err)
// Create a repository for the org
orgUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: org.Org.ID})
repo, _, _ := tests.CreateDeclarativeRepoWithOptions(t, orgUser, tests.DeclarativeRepoOptions{})
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
org.Repo = repo
// Create a quota group for them
group, err := quota_model.CreateGroup(db.DefaultContext, userName)
require.NoError(t, err)
org.QuotaGroup = group
// Create a rule
rule, err := quota_model.CreateRule(db.DefaultContext, userName, limit, quota_model.LimitSubjects{quota_model.LimitSubjectSizeAll})
require.NoError(t, err)
org.QuotaRule = rule
// Add the rule to the group
err = group.AddRuleByName(db.DefaultContext, rule.Name)
require.NoError(t, err)
// Add the org to the group
err = group.AddUserByID(db.DefaultContext, org.Org.ID)
require.NoError(t, err)
return org
}
env := quotaWebEnv{}
env.cleaners = []func(){
test.MockVariableValue(&setting.Quota.Enabled, true),
test.MockVariableValue(&testWebRoutes, routers.NormalRoutes()),
}
// Create the limited user and the various orgs, and a contributor who's not
// in any of the orgs.
env.Users.Limited = makeUser(t, int64(0))
env.Users.Contributor = makeUser(t, int64(0))
env.Orgs.Limited = makeOrg(t, env.Users.Limited.User, int64(0))
env.Orgs.Unlimited = makeOrg(t, env.Users.Limited.User, int64(-1))
env.Users.Ungrouped = makeUngroupedUser(t)
feat(quota): Quota enforcement The previous commit laid out the foundation of the quota engine, this one builds on top of it, and implements the actual enforcement. Enforcement happens at the route decoration level, whenever possible. In case of the API, when over quota, a 413 error is returned, with an appropriate JSON payload. In case of web routes, a 413 HTML page is rendered with similar information. This implementation is for a **soft quota**: quota usage is checked before an operation is to be performed, and the operation is *only* denied if the user is already over quota. This makes it possible to go over quota, but has the significant advantage of being practically implementable within the current Forgejo architecture. The goal of enforcement is to deny actions that can make the user go over quota, and allow the rest. As such, deleting things should - in almost all cases - be possible. A prime exemption is deleting files via the web ui: that creates a new commit, which in turn increases repo size, thus, is denied if the user is over quota. Limitations ----------- Because we generally work at a route decorator level, and rarely look *into* the operation itself, `size:repos:public` and `size:repos:private` are not enforced at this level, the engine enforces against `size:repos:all`. This will be improved in the future. AGit does not play very well with this system, because AGit PRs count toward the repo they're opened against, while in the GitHub-style fork + pull model, it counts against the fork. This too, can be improved in the future. There's very little done on the UI side to guard against going over quota. What this patch implements, is enforcement, not prevention. The UI will still let you *try* operations that *will* result in a denial. Signed-off-by: Gergely Nagy <forgejo@gergo.csillger.hu>
2024-07-06 10:30:16 +02:00
return &env
}