From 19d87bcaff40ff4adbc1fbc439c2fe3e88c703d2 Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <me@zygoon.pl>
Date: Tue, 14 Jan 2025 13:54:55 +0100
Subject: [PATCH] Add support for ordering jobs explicitly

When debugging failures caused by interactions between jobs, being able
to order jobs explicitly is critical. The new -explicit-order option
strongly orders jobs following the order in which they were matched
to command line arguments.

Signed-off-by: Zygmunt Krynicki <me@zygoon.pl>
---
 README.md          |  4 ++++
 cmd/spread/main.go |  2 ++
 spread/project.go  | 29 +++++++++++++++++++++++------
 spread/runner.go   | 20 ++++++++++++++++----
 4 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index fd5f21f8..503a9c68 100644
--- a/README.md
+++ b/README.md
@@ -669,6 +669,10 @@ suffix. Matching multiple components at once is also possible separating them
 with a colon; they don't have to be consecutive as long as the ordering is
 correct.
 
+With the `-explicit-order` option, the tasks are selected for execution in
+order corresponding to the command line arguments that matched them. Note that
+this automatically limits spread to use only one worker per system.
+
 For example, assuming the two jobs above, these parameters would all match
 at least one of them:
 
diff --git a/cmd/spread/main.go b/cmd/spread/main.go
index 57ac5e2f..b8bc339f 100644
--- a/cmd/spread/main.go
+++ b/cmd/spread/main.go
@@ -34,6 +34,7 @@ var (
 	seed           = flag.Int64("seed", 0, "Seed for job order permutation")
 	repeat         = flag.Int("repeat", 0, "Number of times to repeat each task")
 	garbageCollect = flag.Bool("gc", false, "Garbage collect backend resources when possible")
+	explicitOrder  = flag.Bool("explicit-order", false, "Order jobs explicitly")
 )
 
 func main() {
@@ -96,6 +97,7 @@ func run() error {
 		Seed:           *seed,
 		Repeat:         *repeat,
 		GarbageCollect: *garbageCollect,
+		ExplicitOrder:  *explicitOrder,
 	}
 
 	project, err := spread.Load(".")
diff --git a/spread/project.go b/spread/project.go
index e70470f0..0183e8cf 100644
--- a/spread/project.go
+++ b/spread/project.go
@@ -742,6 +742,7 @@ func checkSystems(context fmt.Stringer, systems []string) error {
 
 type Filter interface {
 	Pass(job *Job) bool
+	PassOrder(*Job) (int, bool)
 }
 
 type filterExp struct {
@@ -755,10 +756,15 @@ type filter struct {
 }
 
 func (f *filter) Pass(job *Job) bool {
+	_, ok := f.PassOrder(job)
+	return ok
+}
+
+func (f *filter) PassOrder(job *Job) (int, bool) {
 	if len(f.exps) == 0 {
-		return true
+		return 0, true
 	}
-	for _, exp := range f.exps {
+	for i, exp := range f.exps {
 		if exp.firstSample > 0 {
 			if job.Sample < exp.firstSample {
 				continue
@@ -768,10 +774,10 @@ func (f *filter) Pass(job *Job) bool {
 			}
 		}
 		if exp.regexp.MatchString(job.Name) {
-			return true
+			return i, true
 		}
 	}
-	return false
+	return 0, false
 }
 
 func NewFilter(args []string) (Filter, error) {
@@ -846,6 +852,8 @@ func (p *Project) backendNames() []string {
 func (p *Project) Jobs(options *Options) ([]*Job, error) {
 	var jobs []*Job
 
+	jobOrder := make(map[string]int)
+
 	hasFilter := options.Filter != nil
 	manualBackends := hasFilter
 	manualSystems := hasFilter
@@ -958,8 +966,12 @@ func (p *Project) Jobs(options *Options) ([]*Job, error) {
 							}
 							job.Environment = env.Variant(variant)
 
-							if options.Filter != nil && !options.Filter.Pass(job) {
-								continue
+							if options.Filter != nil {
+								order, ok := options.Filter.PassOrder(job)
+								if !ok {
+									continue
+								}
+								jobOrder[job.Name] = order
 							}
 
 							jobs = append(jobs, job)
@@ -1059,6 +1071,11 @@ func (p *Project) Jobs(options *Options) ([]*Job, error) {
 	}
 
 	sort.Sort(jobsByName(jobs))
+	if options.ExplicitOrder {
+		sort.SliceStable(jobs, func(i, j int) bool {
+			return jobOrder[jobs[i].Name] < jobOrder[jobs[j].Name]
+		})
+	}
 
 	return jobs, nil
 }
diff --git a/spread/runner.go b/spread/runner.go
index c84ff63d..e2711870 100644
--- a/spread/runner.go
+++ b/spread/runner.go
@@ -36,6 +36,7 @@ type Options struct {
 	Seed           int64
 	Repeat         int
 	GarbageCollect bool
+	ExplicitOrder  bool
 }
 
 type Runner struct {
@@ -197,6 +198,9 @@ func (r *Runner) loop() (err error) {
 			for _, job := range r.pending {
 				if job.Backend == backend && job.System == system {
 					if system.Workers > workers[system] {
+						if workers[system] > 0 && r.options.ExplicitOrder {
+							break
+						}
 						workers[system]++
 						r.alive++
 					} else {
@@ -225,10 +229,18 @@ func (r *Runner) loop() (err error) {
 		for _, system := range backend.Systems {
 			n := workers[system]
 			for i := 0; i < n; i++ {
-				// Use a different seed per worker, so that the work-stealing
-				// logic will have a better chance of producing the same
-				// ordering on each of the workers.
-				order := rand.New(rand.NewSource(seed + int64(i))).Perm(len(r.pending))
+				var order []int
+				if r.options.ExplicitOrder {
+					order = make([]int, len(r.pending))
+					for i := range order {
+						order[i] = i
+					}
+				} else {
+					// Use a different seed per worker, so that the work-stealing
+					// logic will have a better chance of producing the same
+					// ordering on each of the workers.
+					order = rand.New(rand.NewSource(seed + int64(i))).Perm(len(r.pending))
+				}
 				go r.worker(backend, system, order)
 			}
 		}