From f45cb01c928e79b00211f97d2052f8378c7be610 Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Wed, 18 Jan 2023 20:21:37 +0530 Subject: [PATCH 1/6] changed module path --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 224eab98..6343a6bc 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/gocraft/work +module github.com/shahbaz275817/work go 1.14 From b441d80f71f60f295dc36b48691e4896583d9223 Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Wed, 18 Jan 2023 20:27:55 +0530 Subject: [PATCH 2/6] changed import path --- benches/bench_work/main.go | 2 +- cmd/workenqueue/main.go | 2 +- cmd/workfakedata/main.go | 2 +- webui/webui.go | 4 ++-- webui/webui_test.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benches/bench_work/main.go b/benches/bench_work/main.go index aaf1de69..57a2a123 100644 --- a/benches/bench_work/main.go +++ b/benches/bench_work/main.go @@ -7,8 +7,8 @@ import ( "time" "github.com/gocraft/health" - "github.com/gocraft/work" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work" ) var namespace = "bench_test" diff --git a/cmd/workenqueue/main.go b/cmd/workenqueue/main.go index 924f6ec8..f91ea509 100644 --- a/cmd/workenqueue/main.go +++ b/cmd/workenqueue/main.go @@ -7,8 +7,8 @@ import ( "os" "time" - "github.com/gocraft/work" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work" ) var redisHostPort = flag.String("redis", ":6379", "redis hostport") diff --git a/cmd/workfakedata/main.go b/cmd/workfakedata/main.go index 72e77b04..33aa8065 100644 --- a/cmd/workfakedata/main.go +++ b/cmd/workfakedata/main.go @@ -6,8 +6,8 @@ import ( "math/rand" "time" - "github.com/gocraft/work" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work" ) var redisHostPort = flag.String("redis", ":6379", "redis hostport") diff --git a/webui/webui.go b/webui/webui.go index 51e250d5..ab418c6f 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -9,9 +9,9 @@ import ( "github.com/braintree/manners" "github.com/gocraft/web" - "github.com/gocraft/work" - "github.com/gocraft/work/webui/internal/assets" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work" + "github.com/shahbaz275817/work/webui/internal/assets" ) // Server implements an HTTP server which exposes a JSON API to view and manage gocraft/work items. diff --git a/webui/webui_test.go b/webui/webui_test.go index 6bffc745..eca197cd 100644 --- a/webui/webui_test.go +++ b/webui/webui_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/gocraft/work" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work" "github.com/stretchr/testify/assert" ) From e9b95e3ab1f6d9f72ba2749e3a65a22d51d7b9b8 Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Wed, 7 Feb 2024 15:05:32 +0530 Subject: [PATCH 3/6] add route for retrying jobs of a specific queue --- client.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ webui/webui.go | 6 ++++++ 2 files changed, 58 insertions(+) diff --git a/client.go b/client.go index 939f33ca..e738ed68 100644 --- a/client.go +++ b/client.go @@ -363,6 +363,58 @@ func (c *Client) DeleteDeadJob(diedAt int64, jobID string) error { return nil } +// Retry retries failed jobs of a specific type from the dead queue. +func (c *Client) RetryDeadOfType(jobType string) error { + // Get queues for job names + queues, err := c.Queues() + if err != nil { + logError("client.retry_all_dead_jobs.queues", err) + return err + } + + fmt.Println("Queues: %+V", queues) + // Extract job names + var jobNames []string + for _, q := range queues { + if q.JobName == jobType { + jobNames = append(jobNames, q.JobName) + } + } + fmt.Println("JobNames: %+V", jobNames) + + script := redis.NewScript(len(jobNames)+1, redisLuaRequeueAllDeadCmd) + + args := make([]interface{}, 0, len(jobNames)+1+3) + args = append(args, redisKeyDead(c.namespace)) // KEY[1] + for _, jobName := range jobNames { + args = append(args, redisKeyJobs(c.namespace, jobName)) // KEY[2, 3, ...] + } + args = append(args, redisKeyJobsPrefix(c.namespace)) // ARGV[1] + args = append(args, nowEpochSeconds()) + args = append(args, 1000) + + conn := c.pool.Get() + defer conn.Close() + + fmt.Println("args: %+V", args) + // Cap iterations for safety (which could reprocess 1k*1k jobs). + // This is conceptually an infinite loop but let's be careful. + for i := 0; i < 1000; i++ { + res, err := redis.Int64(script.Do(conn, args...)) + if err != nil { + logError("client.retry_all_dead_jobs.do", err) + return err + } + + if res == 0 { + break + } + } + + return nil + +} + // RetryDeadJob retries a dead job. The job will be re-queued on the normal work queue for eventual processing by a worker. func (c *Client) RetryDeadJob(diedAt int64, jobID string) error { // Get queues for job names diff --git a/webui/webui.go b/webui/webui.go index ab418c6f..70718aac 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -57,6 +57,7 @@ func NewServer(namespace string, pool *redis.Pool, hostPort string) *Server { router.Get("/dead_jobs", (*context).deadJobs) router.Post("/delete_dead_job/:died_at:\\d.*/:job_id", (*context).deleteDeadJob) router.Post("/retry_dead_job/:died_at:\\d.*/:job_id", (*context).retryDeadJob) + router.Post("/retry_dead_job_type/:job_type", (*context).retryDeadJobOfType) router.Post("/delete_all_dead_jobs", (*context).deleteAllDeadJobs) router.Post("/retry_all_dead_jobs", (*context).retryAllDeadJobs) @@ -204,6 +205,11 @@ func (c *context) retryDeadJob(rw web.ResponseWriter, r *web.Request) { render(rw, map[string]string{"status": "ok"}, err) } +func (c *context) retryDeadJobOfType(rw web.ResponseWriter, r *web.Request) { + err := c.client.RetryDeadOfType(r.PathParams["job_type"]) + + render(rw, map[string]string{"status": "ok"}, err) +} func (c *context) deleteAllDeadJobs(rw web.ResponseWriter, r *web.Request) { err := c.client.DeleteAllDeadJobs() From 47303d24fa1ed352a57abb426dbca7470c39be74 Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Wed, 7 Feb 2024 15:39:06 +0530 Subject: [PATCH 4/6] fix import --- cmd/workwebui/main.go | 2 +- go.mod | 2 -- go.sum | 4 ---- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/workwebui/main.go b/cmd/workwebui/main.go index 6d3e32fc..b1efdf27 100644 --- a/cmd/workwebui/main.go +++ b/cmd/workwebui/main.go @@ -8,8 +8,8 @@ import ( "strconv" "time" - "github.com/gocraft/work/webui" "github.com/gomodule/redigo/redis" + "github.com/shahbaz275817/work/webui" ) var ( diff --git a/go.mod b/go.mod index 6343a6bc..b9e58f38 100644 --- a/go.mod +++ b/go.mod @@ -15,13 +15,11 @@ require ( github.com/garyburd/redigo v1.6.0 // indirect github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b - github.com/gocraft/work v0.5.1 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/gomodule/redigo v2.0.0+incompatible github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb github.com/kr/pretty v0.2.0 // indirect github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 // indirect - github.com/robfig/cron v1.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.5.1 github.com/youtube/vitess v2.1.1+incompatible // indirect diff --git a/go.sum b/go.sum index 63aa5f0a..89fa3ef1 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 h1:pKjeDsx7HGGbjr7V github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0/go.mod h1:rWibcVfwbUxi/QXW84U7vNTcIcZFd6miwbt8ritxh/Y= github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b h1:g2Qcs0B+vOQE1L3a7WQ/JUUSzJnHbTz14qkJSqEWcF4= github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b/go.mod h1:Ag7UMbZNGrnHwaXPJOUKJIVgx4QOWMOWZngrvsN6qak= -github.com/gocraft/work v0.5.1 h1:3bRjMiOo6N4zcRgZWV3Y7uX7R22SF+A9bPTk4xRXr34= -github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= @@ -41,8 +39,6 @@ github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 h1:yOXfzNV7q github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701/go.mod h1:VtBIF1XX0c1nKkeAPk8i4aXkYopqQgfDqolHUIHPwNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 06d7adf44232567dbb5504171a840a6c5def1b2a Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Thu, 8 Feb 2024 12:32:42 +0530 Subject: [PATCH 5/6] fixes and removed irrelevant code --- benches/bench_work/main.go | 2 +- cmd/workenqueue/main.go | 2 +- cmd/workfakedata/main.go | 2 +- cmd/workwebui/main.go | 2 +- go.mod | 4 +++- go.sum | 4 ++++ webui/webui.go | 4 ++-- webui/webui_test.go | 2 +- 8 files changed, 14 insertions(+), 8 deletions(-) diff --git a/benches/bench_work/main.go b/benches/bench_work/main.go index 57a2a123..aaf1de69 100644 --- a/benches/bench_work/main.go +++ b/benches/bench_work/main.go @@ -7,8 +7,8 @@ import ( "time" "github.com/gocraft/health" + "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work" ) var namespace = "bench_test" diff --git a/cmd/workenqueue/main.go b/cmd/workenqueue/main.go index f91ea509..924f6ec8 100644 --- a/cmd/workenqueue/main.go +++ b/cmd/workenqueue/main.go @@ -7,8 +7,8 @@ import ( "os" "time" + "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work" ) var redisHostPort = flag.String("redis", ":6379", "redis hostport") diff --git a/cmd/workfakedata/main.go b/cmd/workfakedata/main.go index 33aa8065..72e77b04 100644 --- a/cmd/workfakedata/main.go +++ b/cmd/workfakedata/main.go @@ -6,8 +6,8 @@ import ( "math/rand" "time" + "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work" ) var redisHostPort = flag.String("redis", ":6379", "redis hostport") diff --git a/cmd/workwebui/main.go b/cmd/workwebui/main.go index b1efdf27..6d3e32fc 100644 --- a/cmd/workwebui/main.go +++ b/cmd/workwebui/main.go @@ -8,8 +8,8 @@ import ( "strconv" "time" + "github.com/gocraft/work/webui" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work/webui" ) var ( diff --git a/go.mod b/go.mod index b9e58f38..224eab98 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/shahbaz275817/work +module github.com/gocraft/work go 1.14 @@ -15,11 +15,13 @@ require ( github.com/garyburd/redigo v1.6.0 // indirect github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b + github.com/gocraft/work v0.5.1 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/gomodule/redigo v2.0.0+incompatible github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb github.com/kr/pretty v0.2.0 // indirect github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 // indirect + github.com/robfig/cron v1.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.5.1 github.com/youtube/vitess v2.1.1+incompatible // indirect diff --git a/go.sum b/go.sum index 89fa3ef1..63aa5f0a 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 h1:pKjeDsx7HGGbjr7V github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0/go.mod h1:rWibcVfwbUxi/QXW84U7vNTcIcZFd6miwbt8ritxh/Y= github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b h1:g2Qcs0B+vOQE1L3a7WQ/JUUSzJnHbTz14qkJSqEWcF4= github.com/gocraft/web v0.0.0-20190207150652-9707327fb69b/go.mod h1:Ag7UMbZNGrnHwaXPJOUKJIVgx4QOWMOWZngrvsN6qak= +github.com/gocraft/work v0.5.1 h1:3bRjMiOo6N4zcRgZWV3Y7uX7R22SF+A9bPTk4xRXr34= +github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= @@ -39,6 +41,8 @@ github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 h1:yOXfzNV7q github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701/go.mod h1:VtBIF1XX0c1nKkeAPk8i4aXkYopqQgfDqolHUIHPwNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/webui/webui.go b/webui/webui.go index 70718aac..85f1bdca 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -9,9 +9,9 @@ import ( "github.com/braintree/manners" "github.com/gocraft/web" + "github.com/gocraft/work" + "github.com/gocraft/work/webui/internal/assets" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work" - "github.com/shahbaz275817/work/webui/internal/assets" ) // Server implements an HTTP server which exposes a JSON API to view and manage gocraft/work items. diff --git a/webui/webui_test.go b/webui/webui_test.go index eca197cd..6bffc745 100644 --- a/webui/webui_test.go +++ b/webui/webui_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" + "github.com/gocraft/work" "github.com/gomodule/redigo/redis" - "github.com/shahbaz275817/work" "github.com/stretchr/testify/assert" ) From f1aed88871e7f9004a0ff3ec6483e5db43aaa348 Mon Sep 17 00:00:00 2001 From: Shahbaz Khan Date: Thu, 8 Feb 2024 12:33:39 +0530 Subject: [PATCH 6/6] typo --- webui/webui.go | 1 + 1 file changed, 1 insertion(+) diff --git a/webui/webui.go b/webui/webui.go index 85f1bdca..17914f9c 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -205,6 +205,7 @@ func (c *context) retryDeadJob(rw web.ResponseWriter, r *web.Request) { render(rw, map[string]string{"status": "ok"}, err) } + func (c *context) retryDeadJobOfType(rw web.ResponseWriter, r *web.Request) { err := c.client.RetryDeadOfType(r.PathParams["job_type"])