Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize: Reduce full repo scans when look tag is set #386

Merged
merged 1 commit into from
Jul 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 28 additions & 15 deletions cmd_get.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,11 @@ func doGet(c *cli.Context) error {
}

var (
firstArg string
firstArg string // Look at the first repo only, if there are more than one
argCnt int
getInfo getInfo // For fetching and looking a single repo
scr scanner
err error
)
if len(args) > 0 {
scr = &sliceScanner{slice: args}
Expand All @@ -52,36 +55,43 @@ func doGet(c *cli.Context) error {
}
scr = bufio.NewScanner(os.Stdin)
}

eg := &errgroup.Group{}
sem := make(chan struct{}, 6)
for scr.Scan() {
target := scr.Text()
if firstArg == "" {
firstArg = target
}
argCnt += 1
if parallel {
sem <- struct{}{}
eg.Go(func() error {
defer func() { <-sem }()
if err := g.get(target); err != nil {
if getInfo, err = g.get(target); err != nil {
logger.Logf("error", "failed to get %q: %s", target, err)
}
return nil
})
} else {
if err := g.get(target); err != nil {
if getInfo, err = g.get(target); err != nil {
return fmt.Errorf("failed to get %q: %w", target, err)
}
}
}
if err := scr.Err(); err != nil {
if err = scr.Err(); err != nil {
return fmt.Errorf("error occurred while reading input: %w", err)
}
if err := eg.Wait(); err != nil {
if err = eg.Wait(); err != nil {
return err
}
if andLook && firstArg != "" {
return look(firstArg, g.bare)
if andLook {
if argCnt > 1 && firstArg != "" {
return look(firstArg, g.bare)
}
if argCnt == 1 && getInfo.localRepository != nil {
return lookByLocalRepository(getInfo.localRepository)
}
}
return nil
}
Expand Down Expand Up @@ -155,14 +165,7 @@ func look(name string, bare bool) error {
case 0:
return fmt.Errorf("no repository found")
case 1:
repo := reposFound[0]
cmd := exec.Command(detectShell())
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Dir = repo.FullPath
cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
return cmdutil.RunCommand(cmd, true)
return lookByLocalRepository(reposFound[0])
default:
b := &strings.Builder{}
b.WriteString("More than one repositories are found; Try more precise name\n")
Expand All @@ -172,3 +175,13 @@ func look(name string, bare bool) error {
return errors.New(b.String())
}
}

func lookByLocalRepository(repo *LocalRepository) error {
cmd := exec.Command(detectShell())
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Dir = repo.FullPath
cmd.Env = append(os.Environ(), "GHQ_LOOK="+filepath.ToSlash(repo.RelPath))
return cmdutil.RunCommand(cmd, true)
}
50 changes: 29 additions & 21 deletions getter.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,27 @@ func getRepoLock(localRepoRoot string) bool {
return !loaded
}

type getInfo struct {
localRepository *LocalRepository
}

type getter struct {
update, shallow, silent, ssh, recursive, bare bool
vcs, branch string
}

func (g *getter) get(argURL string) error {
func (g *getter) get(argURL string) (getInfo, error) {
u, err := newURL(argURL, g.ssh, false)
if err != nil {
return fmt.Errorf("could not parse URL %q: %w", argURL, err)
return getInfo{}, fmt.Errorf("could not parse URL %q: %w", argURL, err)
}
branch := g.branch
if pos := strings.LastIndexByte(u.Path, '@'); pos >= 0 {
u.Path, branch = u.Path[:pos], u.Path[pos+1:]
}
remote, err := NewRemoteRepository(u)
if err != nil {
return err
return getInfo{}, err
}

return g.getRemoteRepository(remote, branch)
Expand All @@ -44,11 +48,14 @@ func (g *getter) get(argURL string) error {
// getRemoteRepository clones or updates a remote repository remote.
// If doUpdate is true, updates the locally cloned repository. Otherwise does nothing.
// If isShallow is true, does shallow cloning. (no effect if already cloned or the VCS is Mercurial and git-svn)
func (g *getter) getRemoteRepository(remote RemoteRepository, branch string) error {
func (g *getter) getRemoteRepository(remote RemoteRepository, branch string) (getInfo, error) {
remoteURL := remote.URL()
local, err := LocalRepositoryFromURL(remoteURL, g.bare)
if err != nil {
return err
return getInfo{}, err
}
info := getInfo{
localRepository: local,
}

var (
Expand All @@ -63,7 +70,7 @@ func (g *getter) getRemoteRepository(remote RemoteRepository, branch string) err
err = nil
}
if err != nil {
return err
return getInfo{}, err
}
}

Expand All @@ -82,7 +89,7 @@ func (g *getter) getRemoteRepository(remote RemoteRepository, branch string) err
if !ok {
vcs, repoURL, err = remote.VCS()
if err != nil {
return err
return getInfo{}, err
}
}
if l := detectLocalRepoRoot(remoteURL.Path, repoURL.Path); l != "" {
Expand All @@ -97,40 +104,41 @@ func (g *getter) getRemoteRepository(remote RemoteRepository, branch string) err
repoURL, _ = url.Parse(remoteURL.Opaque)
}
if getRepoLock(localRepoRoot) {
return vcs.Clone(&vcsGetOption{
url: repoURL,
dir: localRepoRoot,
shallow: g.shallow,
silent: g.silent,
branch: branch,
recursive: g.recursive,
bare: g.bare,
})
return info,
vcs.Clone(&vcsGetOption{
url: repoURL,
dir: localRepoRoot,
shallow: g.shallow,
silent: g.silent,
branch: branch,
recursive: g.recursive,
bare: g.bare,
})
}
return nil
return info, nil
case g.update:
logger.Log("update", fpath)
vcs, localRepoRoot := local.VCS()
if vcs == nil {
return fmt.Errorf("failed to detect VCS for %q", fpath)
return getInfo{}, fmt.Errorf("failed to detect VCS for %q", fpath)
}
repoURL := remoteURL
if remoteURL.Scheme == "codecommit" {
repoURL, _ = url.Parse(remoteURL.Opaque)
}
if getRepoLock(localRepoRoot) {
return vcs.Update(&vcsGetOption{
return info, vcs.Update(&vcsGetOption{
url: repoURL,
dir: localRepoRoot,
silent: g.silent,
recursive: g.recursive,
bare: g.bare,
})
}
return nil
return info, nil
}
logger.Log("exists", fpath)
return nil
return info, nil
}

func detectLocalRepoRoot(remotePath, repoPath string) string {
Expand Down