Adjustments to remove dangling repository locks

Signed-off-by: Bruno Sofiato <[email protected]>

modules/git/command.go

@@ -269,8 +269,30 @@ func CommonCmdServEnvs() []string {
 
 var ErrBrokenCommand = errors.New("git command is broken")
 
-// Run runs the command with the RunOpts
 func (c *Command) Run(opts *RunOpts) error {
+	// Check if there is some dangling locks older than the given threshold
+	if err := ForciblyUnlockRepositoryIfNeeded(c.parentContext, opts.Dir); err != nil {
+		log.Error("Error while trying to unlock repository: %v", err)
+		return err
+	}
+	// Execute the git command
+	if err := c.run(opts); err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			if exitError.ExitCode() > 128 {
+				// Errors greater than 128 means that the process was killed by an OS signal (see https://unix.stackexchange.com/questions/99112/default-exit-code-when-process-is-terminated)
+				log.Warn("It appears that the git process %s has crashed. Attempting to forcbily unlock it [repo: %s]", exitError.Pid(), opts.Dir)
+				ForciblyUnlockRepository(c.parentContext, opts.Dir)
+				return err
+			}
+			return err
+		}
+		return err
+	}
+	return nil
+}
+
+// Run runs the command with the RunOpts
+func (c *Command) run(opts *RunOpts) error {
 	if len(c.brokenArgs) != 0 {
 		log.Error("git command is broken: %s, broken args: %s", c.String(), strings.Join(c.brokenArgs, " "))
 		return ErrBrokenCommand

modules/git/repo_cleanup.go

@@ -0,0 +1,59 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+const threshold = -1 * time.Hour
+
+func ForciblyUnlockRepository(ctx context.Context, repoPath string) error {
+	return cleanLocksIfNeeded(repoPath, time.Now())
+}
+
+func ForciblyUnlockRepositoryIfNeeded(ctx context.Context, repoPath string) error {
+	lockThreshold := time.Now().Add(threshold)
+	return cleanLocksIfNeeded(repoPath, lockThreshold)
+}
+
+func cleanLocksIfNeeded(repoPath string, threshold time.Time) error {
+	if repoPath == "" {
+		return nil
+	} else {
+		log.Trace("Checking if repository %s is locked [lock threshold is %s]", repoPath, threshold)
+		return filepath.Walk(repoPath, func(filePath string, fileInfo os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if err := cleanLockIfNeeded(filePath, fileInfo, threshold); err != nil {
+				log.Error("Failed to remove lock file %s: %v", filePath, err)
+				return err
+			}
+			return nil
+		})
+	}
+}
+
+func cleanLockIfNeeded(filePath string, fileInfo os.FileInfo, threshold time.Time) error {
+	if isLock(fileInfo) && fileInfo.ModTime().Before(threshold) {
+		if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		log.Info("Lock file %s has been removed since its older than %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime())
+		return nil
+	}
+	log.Warn("Cannot exclude lock file %s because it is younger than the thredhold %s [timestamp: %s]", filePath, threshold, fileInfo.ModTime())
+	return nil
+}
+
+func isLock(lockFile os.FileInfo) bool {
+	return !lockFile.IsDir() && strings.HasSuffix(lockFile.Name(), ".lock")
+}

modules/git/repo_cleanup_test.go

@@ -0,0 +1,90 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// This test mimics a repository having dangling locks. If the locks are older than the threshold, they should be
+// removed. Otherwise, they'll remain and the command will fail.
+
+func TestMaintainExistentLock(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		// Need to use touch to change the last access time of the lock files
+		t.Skip("Skipping test on non-linux OS")
+	}
+
+	shouldRemainLocked := func(lockFiles []string, err error) {
+		assert.Error(t, err)
+		for _, lockFile := range lockFiles {
+			assert.FileExists(t, lockFile)
+		}
+	}
+
+	shouldBeUnlocked := func(lockFiles []string, err error) {
+		assert.NoError(t, err)
+		for _, lockFile := range lockFiles {
+			assert.NoFileExists(t, lockFile)
+		}
+	}
+
+	t.Run("2 days lock file", func(t *testing.T) {
+		doTestLockCleanup(t, "2 days", shouldBeUnlocked)
+	})
+
+	t.Run("1 hour lock file", func(t *testing.T) {
+		doTestLockCleanup(t, "1 hour", shouldBeUnlocked)
+	})
+
+	t.Run("1 minutes lock file", func(t *testing.T) {
+		doTestLockCleanup(t, "1 minutes", shouldRemainLocked)
+	})
+}
+
+func doTestLockCleanup(t *testing.T, lockAge string, expectedResult func(lockFiles []string, err error)) {
+	if tmpDir, err := os.MkdirTemp("", "cleanup-after-crash"); err != nil {
+		t.Fatal(err)
+	} else {
+		defer os.RemoveAll(tmpDir)
+
+		if err := os.CopyFS(tmpDir, os.DirFS("../../tests/gitea-repositories-meta/org3/repo3.git")); err != nil {
+			t.Fatal(err)
+		}
+
+		lockFiles := lockFilesFor(tmpDir)
+
+		os.MkdirAll(tmpDir+"/objects/info/commit-graphs", os.ModeSticky|os.ModePerm)
+
+		for _, lockFile := range lockFiles {
+			createLockFiles(t, lockFile, lockAge)
+		}
+
+		cmd := NewCommand(context.Background(), "fetch")
+		_, _, cmdErr := cmd.RunStdString(&RunOpts{Dir: tmpDir})
+
+		expectedResult(lockFiles, cmdErr)
+	}
+}
+
+func lockFilesFor(path string) []string {
+	return []string{
+		path + "/config.lock",
+		path + "/HEAD.lock",
+		path + "/objects/info/commit-graphs/commit-graph-chain.lock",
+	}
+}
+
+func createLockFiles(t *testing.T, file, lockAge string) {
+	cmd := exec.Command("touch", "-m", "-a", "-d", "-"+lockAge, file)
+	if err := cmd.Run(); err != nil {
+		t.Error(err)
+	}
+}