Skip to content

perf: optimize CreateFile/UpdateFile API — 3x latency, 4x QPS #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wu-yangjie wants to merge 1 commit into
base: base/v1.25.4
Choose a base branch
from
Open

perf: optimize CreateFile/UpdateFile API — 3x latency, 4x QPS #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions modules/repository/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@
package repository

import (
"context"

"code.gitea.io/gitea/modules/git"
)

// PostPushUpdates is set by services/repository at init time to handle
// post-push side effects (webhooks, activity, indexer, etc.).
// This indirection avoids an import cycle between services/repository and
// services/repository/files.
var PostPushUpdates func(ctx context.Context, opts *PushUpdateOptions) error

// PushUpdateOptions defines the push update options
type PushUpdateOptions struct {
PusherID int64
Expand Down
77 changes: 71 additions & 6 deletions services/repository/files/temp_repo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type TemporaryUploadRepository struct {
gitRepo *git.Repository
basePath string
cleanup func()
extraEnv []string // extra environment variables for git commands (e.g. GIT_INDEX_FILE)
}

// NewTemporaryUploadRepository creates a new temporary upload repository
Expand All @@ -44,6 +45,44 @@ func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUpload
return t, nil
}

// NewDirectRepoRef creates a TemporaryUploadRepository that operates directly
// on the bare repository with a temporary index file. Objects are written
// directly to the main repo — no clone, no push needed. Use UpdateRef()
// instead of Push() to finalize.
func NewDirectRepoRef(repo *repo_model.Repository) (*TemporaryUploadRepository, error) {
tmpDir, cleanup, err := repo_module.CreateTemporaryPath("direct-index")
if err != nil {
return nil, err
}
indexFile := tmpDir + "/index"
t := &TemporaryUploadRepository{
repo: repo,
basePath: repo.RepoPath(),
cleanup: cleanup,
extraEnv: []string{"GIT_INDEX_FILE=" + indexFile},
}
gitRepo, err := git.OpenRepository(context.Background(), t.basePath)
if err != nil {
cleanup()
return nil, err
}
t.gitRepo = gitRepo
return t, nil
}

// UpdateRef atomically updates a branch ref to point to commitHash.
// This replaces Push() for direct repo operations — no hooks, no subprocess.
func (t *TemporaryUploadRepository) UpdateRef(ctx context.Context, commitHash, branch, oldCommitID string) error {
cmd := gitcmd.NewCommand("update-ref").AddDynamicArguments(git.BranchPrefix+branch, commitHash)
if oldCommitID != "" {
cmd.AddDynamicArguments(oldCommitID)
}
if _, _, err := cmd.RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv}); err != nil {
return fmt.Errorf("UpdateRef: %w", err)
}
return nil
}

// Close the repository cleaning up all files
func (t *TemporaryUploadRepository) Close() {
defer t.gitRepo.Close()
Expand Down Expand Up @@ -98,15 +137,15 @@ func (t *TemporaryUploadRepository) Init(ctx context.Context, objectFormatName s

// SetDefaultIndex sets the git index to our HEAD
func (t *TemporaryUploadRepository) SetDefaultIndex(ctx context.Context) error {
if _, _, err := gitcmd.NewCommand("read-tree", "HEAD").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath}); err != nil {
if _, _, err := gitcmd.NewCommand("read-tree", "HEAD").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv}); err != nil {
return fmt.Errorf("SetDefaultIndex: %w", err)
}
return nil
}

// RefreshIndex looks at the current index and checks to see if merges or updates are needed by checking stat() information.
func (t *TemporaryUploadRepository) RefreshIndex(ctx context.Context) error {
if _, _, err := gitcmd.NewCommand("update-index", "--refresh").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath}); err != nil {
if _, _, err := gitcmd.NewCommand("update-index", "--refresh").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv}); err != nil {
return fmt.Errorf("RefreshIndex: %w", err)
}
return nil
Expand All @@ -120,6 +159,7 @@ func (t *TemporaryUploadRepository) LsFiles(ctx context.Context, filenames ...st
if err := gitcmd.NewCommand("ls-files", "-z").AddDashesAndList(filenames...).
Run(ctx, &gitcmd.RunOpts{
Dir: t.basePath,
Env: t.extraEnv,
Stdout: stdOut,
Stderr: stdErr,
}); err != nil {
Expand Down Expand Up @@ -156,6 +196,7 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(ctx context.Context, fi
if err := gitcmd.NewCommand("update-index", "--remove", "-z", "--index-info").
Run(ctx, &gitcmd.RunOpts{
Dir: t.basePath,
Env: t.extraEnv,
Stdin: stdIn,
Stdout: stdOut,
Stderr: stdErr,
Expand All @@ -173,6 +214,7 @@ func (t *TemporaryUploadRepository) HashObjectAndWrite(ctx context.Context, cont
if err := gitcmd.NewCommand("hash-object", "-w", "--stdin").
Run(ctx, &gitcmd.RunOpts{
Dir: t.basePath,
Env: t.extraEnv,
Stdin: content,
Stdout: stdOut,
Stderr: stdErr,
Expand All @@ -186,7 +228,7 @@ func (t *TemporaryUploadRepository) HashObjectAndWrite(ctx context.Context, cont

// AddObjectToIndex adds the provided object hash to the index with the provided mode and path
func (t *TemporaryUploadRepository) AddObjectToIndex(ctx context.Context, mode, objectHash, objectPath string) error {
if _, _, err := gitcmd.NewCommand("update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath}); err != nil {
if _, _, err := gitcmd.NewCommand("update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv}); err != nil {
stderr := err.Error()
if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched {
return ErrFilePathInvalid{
Expand All @@ -202,7 +244,7 @@ func (t *TemporaryUploadRepository) AddObjectToIndex(ctx context.Context, mode,

// WriteTree writes the current index as a tree to the object db and returns its hash
func (t *TemporaryUploadRepository) WriteTree(ctx context.Context) (string, error) {
stdout, _, err := gitcmd.NewCommand("write-tree").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath})
stdout, _, err := gitcmd.NewCommand("write-tree").RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv})
if err != nil {
log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %w", t.repo.FullName(), err)
Expand All @@ -220,7 +262,7 @@ func (t *TemporaryUploadRepository) GetLastCommitByRef(ctx context.Context, ref
if ref == "" {
ref = "HEAD"
}
stdout, _, err := gitcmd.NewCommand("rev-parse").AddDynamicArguments(ref).RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath})
stdout, _, err := gitcmd.NewCommand("rev-parse").AddDynamicArguments(ref).RunStdString(ctx, &gitcmd.RunOpts{Dir: t.basePath, Env: t.extraEnv})
if err != nil {
log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %w", ref, t.repo.FullName(), err)
Expand Down Expand Up @@ -339,7 +381,7 @@ func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *Commit
stderr := new(bytes.Buffer)
if err := cmdCommitTree.
Run(ctx, &gitcmd.RunOpts{
Env: env,
Env: append(env, t.extraEnv...),
Dir: t.basePath,
Stdin: messageBytes,
Stdout: stdout,
Expand Down Expand Up @@ -378,6 +420,28 @@ func (t *TemporaryUploadRepository) Push(ctx context.Context, doer *user_model.U
return nil
}

// PushInternalSkipHooks pushes with GITEA_INTERNAL_PUSH=true so that the
// gitea pre-receive/update/post-receive hook handlers exit early.
// Callers MUST handle side effects (SyncBranchesToDB, PushUpdates, etc.) themselves.
func (t *TemporaryUploadRepository) PushInternalSkipHooks(ctx context.Context, doer *user_model.User, commitHash, branch string, force bool) error {
env := repo_module.InternalPushingEnvironment(doer, t.repo)
if err := git.Push(ctx, t.basePath, git.PushOptions{
Remote: t.repo.RepoPath(),
Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch),
Env: env,
Force: force,
}); err != nil {
if git.IsErrPushOutOfDate(err) {
return err
} else if git.IsErrPushRejected(err) {
return err
}
return fmt.Errorf("unable to push (internal) from temporary repo: %s (%s) Error: %w",
t.repo.FullName(), t.basePath, err)
}
return nil
}

// DiffIndex returns a Diff of the current index to the head
func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Diff, error) {
stdoutReader, stdoutWriter, err := os.Pipe()
Expand All @@ -394,6 +458,7 @@ func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Dif
Run(ctx, &gitcmd.RunOpts{
Timeout: 30 * time.Second,
Dir: t.basePath,
Env: t.extraEnv,
Stdout: stdoutWriter,
Stderr: stderr,
PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
Expand Down
Loading
Loading