From e9e5442425e6137ccdf32738fb0ecee5ade8de87 Mon Sep 17 00:00:00 2001 From: Takeshi Yamashita <220619+takecy@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:23:29 +0900 Subject: [PATCH] chore: tweak performance --- README.md | 382 ++++++++++++++++++++++++++++++++++++++++------- gih/main.go | 4 +- go.mod | 4 +- go.sum | 25 +--- printer/print.go | 2 +- syncer/git.go | 2 +- syncer/syncer.go | 57 ++++--- 7 files changed, 378 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 07f2a8d..f44f3e7 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,374 @@ # git-here -> `git-here(gih)` is Run git command to all repositories in the current directory. - -
- ![unittest](https://github.com/takecy/git-here/workflows/unittest/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/takecy/git-here)](https://goreportcard.com/report/github.com/takecy/git-here) -![](https://img.shields.io/badge/golang-1.24+-blue.svg?style=flat-square) +![Go Version](https://img.shields.io/badge/golang-1.24+-blue.svg?style=flat-square) [![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/takecy/git-here) -![](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square) -[![Open in Visual Studio Code](https://open.vscode.dev/badges/open-in-vscode.svg)](https://open.vscode.dev/takecy/git-here) +**Efficiently run git commands across multiple repositories in parallel** + +**Before:** +```bash +cd project1 && git pull && cd .. +cd project2 && git pull && cd .. +cd project3 && git pull && cd .. +# ... repeat for dozens of repositories +``` + +**After:** +```bash +gih pull # Done! All repositories updated in parallel +``` + +It is just a tool to do this. It does nothing else. +I created it because I was tired of managing dozens of microservice repositories for the projects I work on. + + +## Table of Contents + +- [Features](#features) +- [Installation](#installation) +- [Usage](#usage) +- [Examples](#examples) +- [Command Reference](#command-reference) +- [Development](#development) +- [Contributing](#contributing) +- [FAQ](#faq) +- [License](#license) + + +## Features + +- 🚀 **Parallel Execution**: Execute git commands across multiple repositories simultaneously +- 🎯 **Filtering**: Use regex patterns to target specific repositories or exclude others +- 🔍 **Directory Discovery**: Automatically finds all git repositories in subdirectories +- ⚡ **High Performance**: Leverages Go's goroutines for maximum efficiency +- ⏰ **Configurable Timeouts**: Aborts immediately at the specified timeouts
+## Installation + +### via Go Install (Recommended) + +```bash +go install github.com/takecy/git-here/gih@latest +``` + +**Verify installation:** +```bash +gih version +``` + +### via Binary Download + +1. Download the latest binary from the [Release Page](https://github.com/takecy/git-here/releases) for your platform +2. Extract and place the binary in your `$PATH` +3. Make it executable: `chmod +x gih` (Unix-like systems) + +**Supported Platforms:** +- Linux (amd64, arm64) +- macOS (amd64, arm64) +- Windows (amd64, arm64) + ## Usage + +### Basic Syntax + +```bash +gih [options] [git_options] ``` -$ gih --timeout 60s pull + +### Quick Start + +```bash +# Pull all repositories in current directory +gih pull + +# Check status of all repositories +gih status + +# Fetch from all remotes +gih fetch --all ``` + +## Examples + +### Common Operations + +**Update all repositories:** +```bash +gih pull +# Equivalent to running 'git pull' in each repository directory ``` -$ gih fetch --all -p + +**Check status across all repositories:** +```bash +gih status --short +# Shows git status for each repository in a compact format ``` + +**Fetch from all remotes:** +```bash +gih fetch --all --prune +# Fetches from all remotes and prunes deleted branches ``` -$ gih --target ^cool-tool pull + +### Advanced Filtering + +**Target specific repositories with regex:** +```bash +# Only operate on repositories matching the pattern +gih --target "^(frontend|backend)" pull +# Only pulls repositories starting with 'frontend' or 'backend' + +gih --target ".*-service$" status +# Only checks status of repositories ending with '-service' ``` + +**Exclude repositories:** +```bash +# Ignore specific repositories +gih --ignore "^(test|temp)" pull +# Pulls all repositories except those starting with 'test' or 'temp' + +gih --ignore ".*-wip$" status +# Check status of all repositories except those ending with '-wip' ``` -$ gih --target ^cool-tool --ignore ^wip-command pull + +**Combine target and ignore patterns:** +```bash +gih --target "^microservice" --ignore "deprecated" pull +# Pull only microservice repositories that aren't deprecated ``` -### Default target directories -```shell -$ tree -. -├── .Hoge // ignore (start from comma) -├── repo_a // target -├── dir -│   └── repo_b // not target -└── repo_c // target +### Performance Tuning + +**Set custom timeout:** +```bash +# Increase timeout for slow operations +gih --timeout 60s pull +# Each repository operation will timeout after 60 seconds + +gih --timeout 5m clone --recursive +# For operations that might take longer ``` -## Install -### via Go -```shell -$ go install github.com/takecy/git-here/gih@latest +**Control concurrency:** +```bash +# Limit concurrent operations (useful for resource-constrained environments) +gih -c 2 pull +# Only run 2 operations in parallel + +# Maximize parallelism (default is number of CPU cores) +gih -c 20 fetch +# Run up to 20 operations in parallel +``` + +### Real-World Scenarios + +**Microservices maintenance:** +```bash +# Update all microservices before deployment +gih --target ".*-service$" pull + +# Check which microservices have uncommitted changes +gih --target ".*-service$" status --porcelain ``` -### via Binary -Download from [Release Page](https://github.com/takecy/git-here/releases) for your environment. -and copy binary to your `$PATH`. +**Open source contribution workflow:** +```bash +# Fetch latest changes from all your forks +gih fetch upstream -### Print usage +# Check status of all your projects +gih status --short ``` -$ gih -Usage: - gih [original_options] [git_options] +**Monorepo-adjacent development:** +```bash +# Update all related projects in your workspace +gih --ignore "node_modules|\.venv" pull +``` -Original Options: - --target Specific target directory with regex. - --ignore Specific ignore directory with regex. - --timeout Specific timeout of performed commnad during on one directory. - 5s, 10m... +### Directory Structure -Commands: - version Print version. Whether check new version exists, and ask you to upgrade to latest version. - Same as git command. (fetch, pull, status...) +git-here discovers repositories based on this structure: -Options: - Same as git. +``` +workspace/ +├── .hidden-repo/ # ❌ Ignored (hidden directories starting with .) +├── project-a/ # ✅ Target (contains .git) +│ └── .git/ +├── project-b/ # ✅ Target (contains .git) +│ └── .git/ +├── non-git-dir/ # ❌ Ignored (no .git directory) +│ └── some-file.txt +└── nested/ + └── deep-repo/ # ❌ Not target (only scans direct subdirectories) + └── .git/ ``` -
+**Key Rules:** +- Only direct subdirectories are scanned (not deeply nested) +- Directories must contain a `.git` folder to be considered repositories +- Hidden directories (starting with `.`) are automatically ignored + +## Command Reference + +### Options + +| Option | Description | Default | Example | +|--------|-------------|---------|----------| +| `--target` | Regex pattern to target specific directories | `""` (all) | `--target "^api"` | +| `--ignore` | Regex pattern to ignore directories | `""` (none) | `--ignore "test$"` | +| `--timeout` | Timeout per repository operation | `20s` | `--timeout 60s` | +| `-c` | Concurrency level (max parallel operations) | CPU cores | `-c 4` | + +### Commands + +**Built-in:** +- `gih version` - Show version information and check for updates +- `gih` - Show help message + +**Git Commands:** +Any valid git command can be used: +- `gih pull` - Pull latest changes +- `gih fetch --all` - Fetch from all remotes +- `gih status --short` - Show compact status +- `gih push origin main` - Push to origin/main +- `gih checkout -b feature/new` - Create and checkout new branch +- `gih commit -m "message"` - Commit changes +- And many more... + +### Exit Codes + +- `0` - Success (all operations completed) +- `1` - General error (invalid arguments, setup issues) +- `2` - Some repositories failed (partial success) ## Development -* Go 1.24+ +### Prerequisites -#### Why this repository have vendor? -It is to simplify development. You can start right away just by cloning. +- Go 1.24 or later -### Prepare +### Setup + +```bash +# Clone the repository +git clone https://github.com/takecy/git-here.git +cd git-here + +# Build for development +make build + +# Run tests +make test + +# Run linter +make lint ``` -$ git clone git@github.com:takecy/git-here.git -$ cd git-here -$ DEBUG=* go run gih/main.go version + +### Available Make Commands + +```bash +make build # Build development binary (gih_dev) +make install # Install production binary +make test # Run all tests with race detection +make lint # Run golangci-lint +make tidy # Run go mod tidy +make update # Update all dependencies ``` -### Testing +### Development Workflow + +```bash +# Build and test your changes +make build +make test +make lint + +# Test manually with debug output +DEBUG=* go run gih/main.go status + +# Test with the development binary +./gih_dev status ``` -$ make test + +### Architecture + +The project is structured into several key packages: + +- **`gih/main.go`** - CLI entry point and argument parsing +- **`syncer/`** - Core orchestration logic + - `syncer.go` - Main execution controller + - `dir.go` - Repository discovery + - `git.go` - Git command execution +- **`printer/`** - Output formatting and coloring + +## Contributing + +We welcome contributions! Here's how to get started: + +### Reporting Issues + +- Use the [GitHub Issues](https://github.com/takecy/git-here/issues) page +- Include your OS, Go version, and git-here version +- Provide steps to reproduce the issue +- Include relevant command output + +### Pull Requests + +1. **Fork** the repository +2. **Create** a feature branch: `git checkout -b feature/amazing-feature` +3. **Make** your changes with tests +4. **Run** the full test suite: `make test lint` +5. **Commit** your changes: `git commit -m 'Add amazing feature'` +6. **Push** to your branch: `git push origin feature/amazing-feature` +7. **Open** a Pull Request + +### Code Style + +- Follow standard Go conventions +- Run `make lint` before submitting +- Add tests for new functionality +- Update documentation as needed + +### Testing + +```bash +# Run all tests +make test + +# Run specific package tests +go test -v ./syncer/ + +# Run with coverage +go test -cover ./... ``` -
+## FAQ + +### Q: Why doesn't git-here work with deeply nested repositories? +**A:** By design, git-here only scans direct subdirectories for performance and clarity. If you need to work with deeply nested repos, consider running git-here from different directory levels. + +### Q: Can I use git-here with non-git commands? +**A:** No, git-here is specifically designed for git commands. It expects git repositories and uses git-specific logic. + +### Q: How do I handle repositories that require different authentication? +**A:** git-here uses your existing git configuration and SSH keys. Ensure your git setup works normally first. + +### Q: What happens if one repository fails? +**A:** Individual repository failures don't stop the entire operation. Failed repositories are reported, but git-here continues with the remaining repositories. + +### Q: Can I use environment variables for configuration? +**A:** Currently, git-here uses command-line flags. Environment variable support may be added in future versions. + +### Q: How do I update git-here? +**A:** Run `go install github.com/takecy/git-here/gih@latest` or download the latest binary from the releases page. ## License -[MIT](./LICENSE) + +[MIT](./LICENSE) © [takecy](https://github.com/takecy) \ No newline at end of file diff --git a/gih/main.go b/gih/main.go index 08cba2d..9c34776 100644 --- a/gih/main.go +++ b/gih/main.go @@ -12,8 +12,8 @@ import ( // set by build var ( - version = "0.14.2" - goversion = "1.24.2" + version = "0.14.4" + goversion = "1.24.5" ) const usage = `Run git command to all repositories in the current directory. diff --git a/go.mod b/go.mod index d1bc8a2..8148128 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( github.com/fatih/color v1.18.0 github.com/matryer/is v1.4.0 github.com/pkg/errors v0.9.1 - golang.org/x/sync v0.13.0 + golang.org/x/sync v0.16.0 ) require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - golang.org/x/sys v0.32.0 // indirect + golang.org/x/sys v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index 80f8119..91405c6 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,15 @@ -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= -golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/printer/print.go b/printer/print.go index 2488163..a7d492f 100644 --- a/printer/print.go +++ b/printer/print.go @@ -29,7 +29,7 @@ const errTmpl = `{{.Repo }} {{.Msg | red}} ` -const cmdTmpl = `Git commad is +const cmdTmpl = `Git command is {{.Cmd | green}} {{.Ops | green}} ` diff --git a/syncer/git.go b/syncer/git.go index ba91ad3..9a52b10 100644 --- a/syncer/git.go +++ b/syncer/git.go @@ -8,7 +8,7 @@ import ( "os/exec" ) -// Gitter is strcut +// Gitter is struct type Gitter struct { writer io.Writer errWriter io.Writer diff --git a/syncer/syncer.go b/syncer/syncer.go index 9b49225..5a2dfd6 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -15,7 +15,7 @@ import ( // Sync is struct type Sync struct { - // TimeOut is timeout of performed command on one direcotory. + // TimeOut is timeout of performed command on one directory. TimeOut string // TargetDir is target directory regex pattern. @@ -24,19 +24,19 @@ type Sync struct { // IgnoreDir is ignore sync target directory regex pattern. IgnoreDir string - // Command is it command. + // Command is the git subcommand to execute. Command string // Options is git command options. Options []string - // Writer is instance + // Writer is the printer instance for output formatting. Writer *printer.Printer // ConNum is concurrency level ConNum int - // Gitter is instance + // Gitter is the git command executor instance. Gitter *Gitter } @@ -59,21 +59,36 @@ func (s *Sync) Run() (err error) { s.Writer.PrintCmd(s.Command, s.Options) + // + // compile regex patterns for performance + // + var ignoreRegex, targetRegex *regexp.Regexp + + if s.IgnoreDir != "" { + ignoreRegex, err = regexp.Compile(s.IgnoreDir) + if err != nil { + return errors.Wrapf(err, "invalid ignore directory regex pattern: %s", s.IgnoreDir) + } + } + + if s.TargetDir != "" { + targetRegex, err = regexp.Compile(s.TargetDir) + if err != nil { + return errors.Wrapf(err, "invalid target directory regex pattern: %s", s.TargetDir) + } + } + // // retrieve target repos // repos := make([]string, 0, len(dirs)) for _, d := range dirs { - if s.IgnoreDir != "" { - if isMatch, _ := regexp.MatchString(s.IgnoreDir, d); isMatch { - continue - } + if ignoreRegex != nil && ignoreRegex.MatchString(d) { + continue } - if s.TargetDir != "" { - if isMatch, _ := regexp.MatchString(s.TargetDir, d); !isMatch { - continue - } + if targetRegex != nil && !targetRegex.MatchString(d) { + continue } repos = append(repos, d) @@ -125,16 +140,22 @@ func (s *Sync) Run() (err error) { }) } + // Handle timeout in a separate goroutine + done := make(chan struct{}) go func() { - for { - <-ctx.Done() - s.Writer.PrintMsgErr(fmt.Sprintf("---- Timeouted (%v) [%v]----", time.Since(start).String(), ctx.Err())) - os.Exit(1) + defer close(done) + if err := eg.Wait(); err != nil { + s.Writer.PrintMsgErr(fmt.Sprintf("Error.exists: %v", err)) } }() - if err := eg.Wait(); err != nil { - s.Writer.PrintMsgErr(fmt.Sprintf("Error.exists: %v", err)) + select { + case <-done: + // All goroutines completed successfully + case <-ctx.Done(): + s.Writer.PrintMsgErr(fmt.Sprintf("---- Timeouted (%v) ----", time.Since(start).String())) + // returns no error + return } s.Writer.PrintMsg(fmt.Sprintf("All done. (%v)", time.Since(start).Round(time.Millisecond)))