TudbuT/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

[GITEA] new doctor check: fix-push-mirrors-without-git-remote (#1853)

Browse source 
This adds a new `doctor` check: `fix-push-mirrors-without-git-remote`. The new check looks for push mirrors that do not have their remotes configured in git. If automatic fixing is enabled, it will remove these push mirrors from the database.

The check is not run by default, and thus, must be invoked manually. It should be usable in a half-migrated state, too, and as such, fixes #1800.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/1853
Co-authored-by: Gergely Nagy <forgejo@gergo.csillger.hu>
Co-committed-by: Gergely Nagy <forgejo@gergo.csillger.hu>
(cherry picked from commit 9038e07ef3)
(cherry picked from commit b15bafcbc7)
(cherry picked from commit 93ba05a2dd)
(cherry picked from commit e418ea8082)
(cherry picked from commit 321790a91e)
(cherry picked from commit f4e19d3323)
(cherry picked from commit 4d9923dee8)
(cherry picked from commit 049df69eda)

Conflicts:
	services/doctor/push_mirror_consistency.go
	https://codeberg.org/forgejo/forgejo/pulls/2214
This commit is contained in:
Gergely Nagy 2023-12-01 13:07:36 +00:00 committed by Earl Warren
parent 71db593057
commit c79cba8d55
No known key found for this signature in database
GPG key ID: 0579CB2928A78A00
4 changed files with 145 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
models/migrations/v1_21/v276.go
View file

@ -4,13 +4,7 @@
package v1_21 //nolint
import (
"context"
"fmt"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/git"
giturl "code.gitea.io/gitea/modules/git/url"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/setting"
"xorm.io/xorm"
@ -73,7 +67,7 @@ func migratePullMirrors(x *xorm.Engine) error {
start += len(mirrors)
for _, m := range mirrors {
remoteAddress, err := getRemoteAddress(m.RepoOwner, m.RepoName, "origin")
remoteAddress, err := repo_model.GetPushMirrorRemoteAddress(m.RepoOwner, m.RepoName, "origin")
if err != nil {
return err
}
@ -136,7 +130,7 @@ func migratePushMirrors(x *xorm.Engine) error {
start += len(mirrors)
for _, m := range mirrors {
remoteAddress, err := getRemoteAddress(m.RepoOwner, m.RepoName, m.RemoteName)
remoteAddress, err := repo_model.GetPushMirrorRemoteAddress(m.RepoOwner, m.RepoName, m.RemoteName)
if err != nil {
return err
}
@ -160,20 +154,3 @@ func migratePushMirrors(x *xorm.Engine) error {
return sess.Commit()
}
func getRemoteAddress(ownerName, repoName, remoteName string) (string, error) {
repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ownerName), strings.ToLower(repoName)+".git")
remoteURL, err := git.GetRemoteAddress(context.Background(), repoPath, remoteName)
if err != nil {
return "", fmt.Errorf("get remote %s's address of %s/%s failed: %v", remoteName, ownerName, repoName, err)
}
u, err := giturl.Parse(remoteURL)
if err != nil {
return "", err
}
u.User = nil
return u.String(), nil
}

24
models/repo/pushmirror.go
View file

@ -5,10 +5,16 @@ package repo
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/git"
giturl "code.gitea.io/gitea/modules/git/url"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -129,3 +135,21 @@ func PushMirrorsIterate(ctx context.Context, limit int, f func(idx int, bean any
}
return sess.Iterate(new(PushMirror), f)
}
// GetPushMirrorRemoteAddress returns the address of associated with a repository's given remote.
func GetPushMirrorRemoteAddress(ownerName, repoName, remoteName string) (string, error) {
repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ownerName), strings.ToLower(repoName)+".git")
remoteURL, err := git.GetRemoteAddress(context.Background(), repoPath, remoteName)
if err != nil {
return "", fmt.Errorf("get remote %s's address of %s/%s failed: %v", remoteName, ownerName, repoName, err)
}
u, err := giturl.Parse(remoteURL)
if err != nil {
return "", err
}
u.User = nil
return u.String(), nil
}

91
services/doctor/push_mirror_consistency.go Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2023 The Forgejo Authors c/o Codeberg e.V.. All rights reserved.
// SPDX-License-Identifier: MIT
package doctor
import (
"context"
"strings"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
"xorm.io/builder"
)
func FixPushMirrorsWithoutGitRemote(ctx context.Context, logger log.Logger, autofix bool) error {
var missingMirrors []*repo_model.PushMirror
err := db.Iterate(ctx, builder.Gt{"id": 0}, func(ctx context.Context, repo *repo_model.Repository) error {
pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, db.ListOptions{})
if err != nil {
return err
}
for i := 0; i < len(pushMirrors); i++ {
_, err = repo_model.GetPushMirrorRemoteAddress(repo.OwnerName, repo.Name, pushMirrors[i].RemoteName)
if err != nil {
if strings.Contains(err.Error(), "No such remote") {
missingMirrors = append(missingMirrors, pushMirrors[i])
} else if logger != nil {
logger.Warn("Unable to retrieve the remote address of a mirror: %s", err)
}
}
}
return nil
})
if err != nil {
if logger != nil {
logger.Critical("Unable to iterate across repounits to fix push mirrors without a git remote: Error %v", err)
}
return err
}
count := len(missingMirrors)
if !autofix {
if logger != nil {
if count == 0 {
logger.Info("Found no push mirrors with missing git remotes")
} else {
logger.Warn("Found %d push mirrors with missing git remotes", count)
}
}
return nil
}
for i := 0; i < len(missingMirrors); i++ {
if logger != nil {
logger.Info("Removing push mirror #%d (remote: %s), for repo: %s/%s",
missingMirrors[i].ID,
missingMirrors[i].RemoteName,
missingMirrors[i].GetRepository(ctx).OwnerName,
missingMirrors[i].GetRepository(ctx).Name)
}
err = repo_model.DeletePushMirrors(ctx, repo_model.PushMirrorOptions{
ID: missingMirrors[i].ID,
RepoID: missingMirrors[i].RepoID,
RemoteName: missingMirrors[i].RemoteName,
})
if err != nil {
if logger != nil {
logger.Critical("Error removing a push mirror (repo_id: %d, push_mirror: %d): %s", missingMirrors[i].Repo.ID, missingMirrors[i].ID, err)
}
return err
}
}
return nil
}
func init() {
Register(&Check{
Title: "Check for push mirrors without a git remote configured",
Name: "fix-push-mirrors-without-git-remote",
IsDefault: false,
Run: FixPushMirrorsWithoutGitRemote,
Priority: 7,
})
}

28
tests/integration/mirror_push_test.go
View file

@ -18,6 +18,7 @@ import (
gitea_context "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
doctor "code.gitea.io/gitea/services/doctor"
"code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
repo_service "code.gitea.io/gitea/services/repository"
@ -47,10 +48,11 @@ func testMirrorPush(t *testing.T, u *url.URL) {
ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name)
doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape("does-not-matter")), user.LowerName, userPassword)(t)
mirrors, _, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, mirrors, 1)
assert.Len(t, mirrors, 2)
ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID)
assert.True(t, ok)
@ -71,6 +73,30 @@ func testMirrorPush(t *testing.T, u *url.URL) {
assert.Equal(t, srcCommit.ID, mirrorCommit.ID)
// Test that we can "repair" push mirrors where the remote doesn't exist in git's state.
// To do that, we artificially remove the remote...
cmd := git.NewCommand(db.DefaultContext, "remote", "rm").AddDynamicArguments(mirrors[0].RemoteName)
_, _, err = cmd.RunStdString(&git.RunOpts{Dir: srcRepo.RepoPath()})
assert.NoError(t, err)
// ...then ensure that trying to get its remote address fails
_, err = repo_model.GetPushMirrorRemoteAddress(srcRepo.OwnerName, srcRepo.Name, mirrors[0].RemoteName)
assert.Error(t, err)
// ...and that we can fix it.
err = doctor.FixPushMirrorsWithoutGitRemote(db.DefaultContext, nil, true)
assert.NoError(t, err)
// ...and after fixing, we only have one remote
mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, mirrors, 1)
// ...one we can get the address of, and it's not the one we removed
remoteAddress, err := repo_model.GetPushMirrorRemoteAddress(srcRepo.OwnerName, srcRepo.Name, mirrors[0].RemoteName)
assert.NoError(t, err)
assert.Contains(t, remoteAddress, "does-not-matter")
// Cleanup
doRemovePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword, int(mirrors[0].ID))(t)
mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})