Template
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo synced 2024-12-13 06:51:57 +01:00
forgejo/services/repository/files/update.go

504 lines
15 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package files
import (
"context"
"fmt"
"io"
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
asymkey_service "code.gitea.io/gitea/services/asymkey"
)
// IdentityOptions for a person's identity like an author or committer
type IdentityOptions struct {
Name string
Email string
}
// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE
type CommitDateOptions struct {
Author time.Time
Committer time.Time
}
type ChangeRepoFile struct {
Operation string
TreePath string
FromTreePath string
ContentReader io.ReadSeeker
SHA string
Options *RepoFileOptions
}
// ChangeRepoFilesOptions holds the repository files update options
type ChangeRepoFilesOptions struct {
LastCommitID string
OldBranch string
NewBranch string
Message string
Files []*ChangeRepoFile
Author *IdentityOptions
Committer *IdentityOptions
Dates *CommitDateOptions
Signoff bool
}
type RepoFileOptions struct {
treePath string
fromTreePath string
executable bool
}
// ChangeRepoFiles adds, updates or removes multiple files in the given repository
func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) {
err := repo.MustNotBeArchived()
if err != nil {
return nil, err
}
[Enhancement] Allow admin to merge pr with protected file changes (#12078) * [Enhancement] Allow admin to merge pr with protected file changes As tilte, show protected message in diff page and merge box. Signed-off-by: a1012112796 <1012112796@qq.com> * remove unused ver * Update options/locale/locale_en-US.ini Co-authored-by: Cirno the Strongest <1447794+CirnoT@users.noreply.github.com> * Add TrN * Apply suggestions from code review * fix lint * Update options/locale/locale_en-US.ini Co-authored-by: zeripath <art27@cantab.net> * Apply suggestions from code review * move pr proteced files check to TestPatch * Call TestPatch when protected branches settings changed * Apply review suggestion @CirnoT * move to service @lunny * slightly restructure routers/private/hook.go Adds a lot of comments and simplifies the logic Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * skip duplicate protected files check * fix check logic * slight refactor of TestPatch Signed-off-by: Andrew Thornton <art27@cantab.net> * When checking for protected files changes in TestPatch use the temporary repository Signed-off-by: Andrew Thornton <art27@cantab.net> * fix introduced issue with hook Signed-off-by: Andrew Thornton <art27@cantab.net> * Remove the check on PR index being greater than 0 as it unnecessary Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Cirno the Strongest <1447794+CirnoT@users.noreply.github.com> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
2020-10-13 20:50:57 +02:00
// If no branch name is set, assume default branch
if opts.OldBranch == "" {
opts.OldBranch = repo.DefaultBranch
}
if opts.NewBranch == "" {
opts.NewBranch = opts.OldBranch
}
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
if err != nil {
return nil, err
}
defer closer.Close()
// oldBranch must exist for this operation
if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty {
return nil, err
}
var treePaths []string
for _, file := range opts.Files {
// If FromTreePath is not set, set it to the opts.TreePath
if file.TreePath != "" && file.FromTreePath == "" {
file.FromTreePath = file.TreePath
}
// Check that the path given in opts.treePath is valid (not a git path)
treePath := CleanUploadFileName(file.TreePath)
if treePath == "" {
return nil, models.ErrFilenameInvalid{
Path: file.TreePath,
}
}
// If there is a fromTreePath (we are copying it), also clean it up
fromTreePath := CleanUploadFileName(file.FromTreePath)
if fromTreePath == "" && file.FromTreePath != "" {
return nil, models.ErrFilenameInvalid{
Path: file.FromTreePath,
}
}
file.Options = &RepoFileOptions{
treePath: treePath,
fromTreePath: fromTreePath,
executable: false,
}
treePaths = append(treePaths, treePath)
}
// A NewBranch can be specified for the file to be created/updated in a new branch.
// Check to make sure the branch does not already exist, otherwise we can't proceed.
// If we aren't branching to a new branch, make sure user can commit to the given branch
if opts.NewBranch != opts.OldBranch {
existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
if existingBranch != nil {
return nil, git_model.ErrBranchAlreadyExists{
BranchName: opts.NewBranch,
}
}
Improve listing performance by using go-git (#6478) * Use go-git for tree reading and commit info lookup. Signed-off-by: Filip Navara <navara@emclient.com> * Use TreeEntry.IsRegular() instead of ObjectType that was removed. Signed-off-by: Filip Navara <navara@emclient.com> * Use the treePath to optimize commit info search. Signed-off-by: Filip Navara <navara@emclient.com> * Extract the latest commit at treePath along with the other commits. Signed-off-by: Filip Navara <navara@emclient.com> * Fix listing commit info for a directory that was created in one commit and never modified after. Signed-off-by: Filip Navara <navara@emclient.com> * Avoid nearly all external 'git' invocations when doing directory listing (.editorconfig code path is still hit). Signed-off-by: Filip Navara <navara@emclient.com> * Use go-git for reading blobs. Signed-off-by: Filip Navara <navara@emclient.com> * Make SHA1 type alias for plumbing.Hash in go-git. Signed-off-by: Filip Navara <navara@emclient.com> * Make Signature type alias for object.Signature in go-git. Signed-off-by: Filip Navara <navara@emclient.com> * Fix GetCommitsInfo for repository with only one commit. Signed-off-by: Filip Navara <navara@emclient.com> * Fix PGP signature verification. Signed-off-by: Filip Navara <navara@emclient.com> * Fix issues with walking commit graph across merges. Signed-off-by: Filip Navara <navara@emclient.com> * Fix typo in condition. Signed-off-by: Filip Navara <navara@emclient.com> * Speed up loading branch list by keeping the repository reference (and thus all the loaded packfile indexes). Signed-off-by: Filip Navara <navara@emclient.com> * Fix lising submodules. Signed-off-by: Filip Navara <navara@emclient.com> * Fix build Signed-off-by: Filip Navara <navara@emclient.com> * Add back commit cache because of name-rev Signed-off-by: Filip Navara <navara@emclient.com> * Fix tests Signed-off-by: Filip Navara <navara@emclient.com> * Fix code style * Fix spelling * Address PR feedback Signed-off-by: Filip Navara <navara@emclient.com> * Update vendor module list Signed-off-by: Filip Navara <navara@emclient.com> * Fix getting trees by commit id Signed-off-by: Filip Navara <navara@emclient.com> * Fix remaining unit test failures * Fix GetTreeBySHA * Avoid running `git name-rev` if not necessary Signed-off-by: Filip Navara <navara@emclient.com> * Move Branch code to git module * Clean up GPG signature verification and fix it for tagged commits * Address PR feedback (import formatting, copyright headers) * Make blob lookup by SHA working * Update tests to use public API * Allow getting content from any type of object through the blob interface * Change test to actually expect the object content that is in the GIT repository * Change one more test to actually expect the object content that is in the GIT repository * Add comments
2019-04-19 14:17:27 +02:00
if err != nil && !git.IsErrBranchNotExist(err) {
return nil, err
}
} else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil {
return nil, err
}
message := strings.TrimSpace(opts.Message)
author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer)
t, err := NewTemporaryUploadRepository(ctx, repo)
if err != nil {
log.Error("NewTemporaryUploadRepository failed: %v", err)
}
2019-06-12 21:41:28 +02:00
defer t.Close()
hasOldBranch := true
Fix reverting a merge commit failing (#28794) Fixes #22236 --- Error occurring currently while trying to revert commit using read-tree -m approach: > 2022/12/26 16:04:43 ...rvices/pull/patch.go:240:AttemptThreeWayMerge() [E] [63a9c61a] Unable to run read-tree -m! Error: exit status 128 - fatal: this operation must be run in a work tree > - fatal: this operation must be run in a work tree We need to clone a non-bare repository for `git read-tree -m` to work. https://github.com/go-gitea/gitea/commit/bb371aee6ecf5e570cdf7b5f7f0d6f47a607a325 adds support to create a non-bare cloned temporary upload repository. After cloning a non-bare temporary upload repository, we [set default index](https://github.com/go-gitea/gitea/blob/main/services/repository/files/cherry_pick.go#L37) (`git read-tree HEAD`). This operation ends up resetting the git index file (see investigation details below), due to which, we need to call `git update-index --refresh` afterward. Here's the diff of the index file before and after we execute SetDefaultIndex: https://www.diffchecker.com/hyOP3eJy/ Notice the **ctime**, **mtime** are set to 0 after SetDefaultIndex. You can reproduce the same behavior using these steps: ```bash $ git clone https://try.gitea.io/me-heer/test.git -s -b main $ cd test $ git read-tree HEAD $ git read-tree -m 1f085d7ed8 1f085d7ed8 9933caed00 error: Entry '1' not uptodate. Cannot merge. ``` After which, we can fix like this: ``` $ git update-index --refresh $ git read-tree -m 1f085d7ed8 1f085d7ed8 9933caed00 ```
2024-01-16 16:06:51 +01:00
if err := t.Clone(opts.OldBranch, true); err != nil {
for _, file := range opts.Files {
if file.Operation == "delete" {
return nil, err
}
}
if !git.IsErrBranchNotExist(err) || !repo.IsEmpty {
return nil, err
}
if err := t.Init(repo.ObjectFormatName); err != nil {
return nil, err
}
hasOldBranch = false
opts.LastCommitID = ""
}
if hasOldBranch {
if err := t.SetDefaultIndex(); err != nil {
return nil, err
}
}
for _, file := range opts.Files {
if file.Operation == "delete" {
// Get the files in the index
filesInIndex, err := t.LsFiles(file.TreePath)
if err != nil {
return nil, fmt.Errorf("DeleteRepoFile: %w", err)
}
// Find the file we want to delete in the index
inFilelist := false
for _, indexFile := range filesInIndex {
if indexFile == file.TreePath {
inFilelist = true
break
}
}
if !inFilelist {
return nil, models.ErrRepoFileDoesNotExist{
Path: file.TreePath,
}
}
}
}
if hasOldBranch {
// Get the commit of the original branch
commit, err := t.GetBranchCommit(opts.OldBranch)
if err != nil {
return nil, err // Couldn't get a commit for the branch
}
// Assigned LastCommitID in opts if it hasn't been set
if opts.LastCommitID == "" {
opts.LastCommitID = commit.ID.String()
} else {
lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID)
if err != nil {
return nil, fmt.Errorf("ConvertToSHA1: Invalid last commit ID: %w", err)
}
opts.LastCommitID = lastCommitID.String()
}
for _, file := range opts.Files {
if err := handleCheckErrors(file, commit, opts, repo); err != nil {
return nil, err
}
}
}
contentStore := lfs.NewContentStore()
for _, file := range opts.Files {
switch file.Operation {
case "create", "update":
if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil {
return nil, err
}
case "delete":
// Remove the file from the index
if err := t.RemoveFilesFromIndex(file.TreePath); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("invalid file operation: %s %s, supported operations are create, update, delete", file.Operation, file.Options.treePath)
}
}
// Now write the tree
treeHash, err := t.WriteTree()
if err != nil {
return nil, err
}
// Now commit the tree
var commitHash string
if opts.Dates != nil {
commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer)
} else {
commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff)
}
if err != nil {
return nil, err
}
// Then push this tree to NewBranch
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
log.Error("%T %v", err, err)
return nil, err
}
commit, err := t.GetCommit(commitHash)
if err != nil {
return nil, err
}
filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths)
if err != nil {
return nil, err
}
if repo.IsEmpty {
if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty {
_ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch")
}
}
return filesResponse, nil
}
// handles the check for various issues for ChangeRepoFiles
func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions, repo *repo_model.Repository) error {
if file.Operation == "update" || file.Operation == "delete" {
fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath)
if err != nil {
return err
}
if file.SHA != "" {
// If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error
if file.SHA != fromEntry.ID.String() {
return models.ErrSHADoesNotMatch{
Path: file.Options.treePath,
GivenSHA: file.SHA,
CurrentSHA: fromEntry.ID.String(),
}
}
} else if opts.LastCommitID != "" {
// If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw
// an error, but only if we aren't creating a new branch.
if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch {
if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil {
return err
} else if changed {
return models.ErrCommitIDDoesNotMatch{
GivenCommitID: opts.LastCommitID,
CurrentCommitID: opts.LastCommitID,
}
}
// The file wasn't modified, so we are good to delete it
}
} else {
// When updating a file, a lastCommitID or SHA needs to be given to make sure other commits
// haven't been made. We throw an error if one wasn't provided.
return models.ErrSHAOrCommitIDNotProvided{}
}
file.Options.executable = fromEntry.IsExecutable()
}
if file.Operation == "create" || file.Operation == "update" {
// For the path where this file will be created/updated, we need to make
// sure no parts of the path are existing files or links except for the last
// item in the path which is the file name, and that shouldn't exist IF it is
// a new file OR is being moved to a new path.
treePathParts := strings.Split(file.Options.treePath, "/")
subTreePath := ""
for index, part := range treePathParts {
subTreePath = path.Join(subTreePath, part)
entry, err := commit.GetTreeEntryByPath(subTreePath)
if err != nil {
if git.IsErrNotExist(err) {
// Means there is no item with that name, so we're good
break
}
return err
}
if index < len(treePathParts)-1 {
if !entry.IsDir() {
return models.ErrFilePathInvalid{
Message: fmt.Sprintf("a file exists where youre trying to create a subdirectory [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeBlob,
}
}
} else if entry.IsLink() {
return models.ErrFilePathInvalid{
Message: fmt.Sprintf("a symbolic link exists where youre trying to create a subdirectory [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeSymlink,
}
} else if entry.IsDir() {
return models.ErrFilePathInvalid{
Message: fmt.Sprintf("a directory exists where youre trying to create a file [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeTree,
}
} else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" {
// The entry shouldn't exist if we are creating new file or moving to a new path
return models.ErrRepoFileAlreadyExists{
Path: file.Options.treePath,
}
}
}
}
return nil
}
// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles
func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error {
// Get the two paths (might be the same if not moving) from the index if they exist
filesInIndex, err := t.LsFiles(file.TreePath, file.FromTreePath)
if err != nil {
return fmt.Errorf("UpdateRepoFile: %w", err)
}
// If is a new file (not updating) then the given path shouldn't exist
if file.Operation == "create" {
for _, indexFile := range filesInIndex {
if indexFile == file.TreePath {
return models.ErrRepoFileAlreadyExists{
Path: file.TreePath,
}
}
}
}
// Remove the old path from the tree
if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 {
for _, indexFile := range filesInIndex {
if indexFile == file.Options.fromTreePath {
if err := t.RemoveFilesFromIndex(file.FromTreePath); err != nil {
return err
}
}
}
}
treeObjectContentReader := file.ContentReader
var lfsMetaObject *git_model.LFSMetaObject
if setting.LFS.StartServer && hasOldBranch {
// Check there is no way this can return multiple infos
filterAttribute, err := t.gitRepo.GitAttributeFirst("", file.Options.treePath, "filter")
if err != nil {
return err
}
if filterAttribute == "lfs" {
// OK so we are supposed to LFS this data!
pointer, err := lfs.GeneratePointer(treeObjectContentReader)
if err != nil {
return err
}
lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID}
treeObjectContentReader = strings.NewReader(pointer.StringContent())
}
}
// Add the object to the database
objectHash, err := t.HashObject(treeObjectContentReader)
if err != nil {
return err
}
// Add the object to the index
if file.Options.executable {
if err := t.AddObjectToIndex("100755", objectHash, file.Options.treePath); err != nil {
return err
}
} else {
if err := t.AddObjectToIndex("100644", objectHash, file.Options.treePath); err != nil {
return err
}
}
if lfsMetaObject != nil {
// We have an LFS object - create it
lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer)
if err != nil {
return err
}
Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>
2021-04-09 00:25:57 +02:00
exist, err := contentStore.Exists(lfsMetaObject.Pointer)
if err != nil {
return err
}
if !exist {
_, err := file.ContentReader.Seek(0, io.SeekStart)
if err != nil {
return err
}
if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil {
if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil {
return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err)
}
return err
}
}
}
return nil
}
// VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch
func VerifyBranchProtection(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, branchName string, treePaths []string) error {
protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, branchName)
if err != nil {
return err
}
if protectedBranch != nil {
protectedBranch.Repo = repo
globUnprotected := protectedBranch.GetUnprotectedFilePatterns()
globProtected := protectedBranch.GetProtectedFilePatterns()
canUserPush := protectedBranch.CanUserPush(ctx, doer)
for _, treePath := range treePaths {
isUnprotectedFile := false
if len(globUnprotected) != 0 {
isUnprotectedFile = protectedBranch.IsUnprotectedFile(globUnprotected, treePath)
}
if !canUserPush && !isUnprotectedFile {
return models.ErrUserCannotCommit{
UserName: doer.LowerName,
}
}
if protectedBranch.IsProtectedFile(globProtected, treePath) {
return models.ErrFilePathProtected{
Path: treePath,
}
}
}
if protectedBranch.RequireSignedCommits {
_, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), branchName)
if err != nil {
if !asymkey_service.IsErrWontSign(err) {
return err
}
return models.ErrUserCannotCommit{
UserName: doer.LowerName,
}
}
}
}
return nil
}