From e51572874b8f7a221f6e762ce6e1e6c656b02f36 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 18:17:53 -0600 Subject: [PATCH 1/9] use the kube goapi instead of the rest api. --- examples/slack8s-deployment.yaml | 4 - main.go | 129 ++++++------------------------- 2 files changed, 25 insertions(+), 108 deletions(-) diff --git a/examples/slack8s-deployment.yaml b/examples/slack8s-deployment.yaml index fecdf5e..7d771b6 100644 --- a/examples/slack8s-deployment.yaml +++ b/examples/slack8s-deployment.yaml @@ -33,8 +33,4 @@ spec: configMapKeyRef: name: slack8s key: slack-channel - - name: kube-proxy - image: gcr.io/google_containers/kubectl:v0.18.0-120-gaeb4ac55ad12b1-dirty - imagePullPolicy: Always - args: ["proxy", "-p", "8001"] diff --git a/main.go b/main.go index 553680c..9016eaf 100644 --- a/main.go +++ b/main.go @@ -1,57 +1,24 @@ package main import ( - "encoding/json" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" - "strconv" "strings" - "time" "github.com/nlopes/slack" -) - -// The GET request to the Kubernetes event watch API returns a JSON object -// which unmarshals into this Response type. -type Response struct { - Type string `json:"type"` - Object Event `json:"object"` -} - -// The Event type and its child-types, contain only the values of the response -// that our alerts currently care about. -type Event struct { - Source EventSource `json:"source"` - InvolvedObject EventInvolvedObject `json:"involvedObject"` - Metadata EventMetadata `json:"metadata"` - Reason string `json:"reason"` - Message string `json:"message"` - FirstTimestamp time.Time `json:"firstTimestamp"` - LastTimestamp time.Time `json:"lastTimestamp"` - Count int `json:"count"` -} - -type EventMetadata struct { - Name string `json:"name"` - Namespace string `json:"namespace"` -} -type EventSource struct { - Component string `json:"component"` -} - -type EventInvolvedObject struct { - Kind string `json:"kind"` -} + "k8s.io/kubernetes/pkg/api" + client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) // Sends a message to the Slack channel about the Event. -func send_message(e Event, color string) error { +func sendMessage(e *api.Event, color string) error { api := slack.New(os.Getenv("SLACK_TOKEN")) params := slack.PostMessageParameters{} + metadata := e.GetObjectMeta() attachment := slack.Attachment{ // The fallback message shows in clients such as IRC or OS X notifications. Fallback: e.Message, @@ -67,7 +34,7 @@ func send_message(e Event, color string) error { }, slack.AttachmentField{ Title: "Name", - Value: e.Metadata.Name, + Value: metadata.GetName(), Short: true, }, slack.AttachmentField{ @@ -104,83 +71,37 @@ func send_message(e Event, color string) error { } func main() { - url := fmt.Sprintf("http://localhost:8001/api/v1/events?watch=true") - req, err := http.NewRequest("GET", url, nil) + + kubeClient, err := client.NewInCluster() if err != nil { - log.Fatal("NewRequest: ", err) + log.Fatalf("Failed to create client: %v", err) } - client := &http.Client{} - resp, err := client.Do(req) + + // Setup a watcher for pods + podClient := kubeClient.Pods(api.NamespaceAll) + options := api.ListOptions{LabelSelector: labels.Everything()} + w, err := podClient.Watch(options) if err != nil { - log.Fatal("Do: ", err) - } - defer resp.Body.Close() - dec := json.NewDecoder(resp.Body) - if resp.StatusCode != 200 { - log.Printf(string(resp.Status) + ": " + string(resp.StatusCode)) - log.Fatal("Non 200 status code returned from Kubernetes API.") + log.Fatalf("Failed to set up watch: %v", err) } - for { - var r Response - if err := dec.Decode(&r); err == io.EOF { - log.Printf("EOF detected.") - break - } else if err != nil { - // Debug output to help when we've failed to decode. - htmlData, er := ioutil.ReadAll(resp.Body) - if er != nil { - log.Printf("Already failed to decode, but also failed to read response for log output.") - } - log.Printf(string(htmlData)) - log.Fatal("Decode: ", err) - } - e := r.Object - - // Log all events for now. - log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp) - + select { + case watchEvent, _ := <-w.ResultChan(): send := false color := "" - - // @todo refactor the configuration of which things to post. - if e.Reason == "SuccessfulCreate" { - send = true - color = "good" - } else if e.Reason == "NodeReady" { + if watchEvent.Type == watch.Added { send = true color = "good" - } else if e.Reason == "NodeNotReady" { + } else if watchEvent.Type == watch.Deleted { send = true color = "warning" - } else if e.Reason == "NodeOutOfDisk" { - send = true - color = "danger" - } - - // For now, dont alert multiple times, except if it's a backoff - if e.Count > 1 { - send = false - } - if e.Reason == "BackOff" && e.Count == 3 { - send = true - color = "danger" - } - - // Do not send any events that are more than 1 minute old. - // This assumes events are processed quickly (very likely) - // in exchange for not re-notifying of events after a crash - // or fresh start. - diff := time.Now().Sub(e.LastTimestamp) - diffMinutes := int(diff.Minutes()) - if diffMinutes > 1 { - log.Printf("Supressed %s minute old message: %s", strconv.Itoa(diffMinutes), e.Message) - send = false } + fmt.Printf("%+v\n", watchEvent) if send { - err = send_message(e, color) + event, _ := watchEvent.Object.(*api.Event) + err = sendMessage(event, color) if err != nil { - log.Fatal("send_message: ", err) + log.Fatal("sendMessage: ", err) } } } From 68550cc469a88b6578f1821ddc1395ddac4caa44 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 18:20:44 -0600 Subject: [PATCH 2/9] watching all events, not just pods. --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9016eaf..ae89dc8 100644 --- a/main.go +++ b/main.go @@ -78,9 +78,9 @@ func main() { } // Setup a watcher for pods - podClient := kubeClient.Pods(api.NamespaceAll) + eventClient := kubeClient.Events(api.NamespaceAll) options := api.ListOptions{LabelSelector: labels.Everything()} - w, err := podClient.Watch(options) + w, err := eventClient.Watch(options) if err != nil { log.Fatalf("Failed to set up watch: %v", err) } From 0c93e4bca9045a7cc2ef41d9de4dd092857776da Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 18:43:55 -0600 Subject: [PATCH 3/9] log all events. --- main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index ae89dc8..b2bc006 100644 --- a/main.go +++ b/main.go @@ -77,7 +77,7 @@ func main() { log.Fatalf("Failed to create client: %v", err) } - // Setup a watcher for pods + // Setup a watcher for events. eventClient := kubeClient.Events(api.NamespaceAll) options := api.ListOptions{LabelSelector: labels.Everything()} w, err := eventClient.Watch(options) @@ -86,6 +86,10 @@ func main() { } select { case watchEvent, _ := <-w.ResultChan(): + + // Log all events for now. + log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp) + send := false color := "" if watchEvent.Type == watch.Added { @@ -95,13 +99,11 @@ func main() { send = true color = "warning" } - fmt.Printf("%+v\n", watchEvent) - if send { event, _ := watchEvent.Object.(*api.Event) err = sendMessage(event, color) if err != nil { - log.Fatal("sendMessage: ", err) + log.Fatalf("sendMessage: %v", err) } } } From e4cdaee5f7327fc51402334fd519f280a78ec5bc Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 18:50:04 -0600 Subject: [PATCH 4/9] bring back more alerts. --- main.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index b2bc006..0dd7ff1 100644 --- a/main.go +++ b/main.go @@ -87,6 +87,7 @@ func main() { select { case watchEvent, _ := <-w.ResultChan(): + e, _ := watchEvent.Object.(*api.Event) // Log all events for now. log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp) @@ -98,10 +99,31 @@ func main() { } else if watchEvent.Type == watch.Deleted { send = true color = "warning" + } else if e.Reason == "SuccessfulCreate" { + send = true + color = "good" + } else if e.Reason == "NodeReady" { + send = true + color = "good" + } else if e.Reason == "NodeNotReady" { + send = true + color = "warning" + } else if e.Reason == "NodeOutOfDisk" { + send = true + color = "danger" + } + + // For now, dont alert multiple times, except if it's a backoff + if e.Count > 1 { + send = false } + if e.Reason == "BackOff" && e.Count == 3 { + send = true + color = "danger" + } + if send { - event, _ := watchEvent.Object.(*api.Event) - err = sendMessage(event, color) + err = sendMessage(e, color) if err != nil { log.Fatalf("sendMessage: %v", err) } From 1825f7e0b53a9390f42ec305ceb818c04eeb1481 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 19:06:57 -0600 Subject: [PATCH 5/9] fix import and bad type. --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 0dd7ff1..cfc0c70 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "strconv" "strings" "github.com/nlopes/slack" @@ -89,7 +90,7 @@ func main() { e, _ := watchEvent.Object.(*api.Event) // Log all events for now. - log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp) + log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp) send := false color := "" From f22b8b5405395628678e97ba131d4f593ba80e77 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 19:19:49 -0600 Subject: [PATCH 6/9] loop --- main.go | 78 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/main.go b/main.go index cfc0c70..3c79876 100644 --- a/main.go +++ b/main.go @@ -85,48 +85,50 @@ func main() { if err != nil { log.Fatalf("Failed to set up watch: %v", err) } - select { - case watchEvent, _ := <-w.ResultChan(): + for { + select { + case watchEvent, _ := <-w.ResultChan(): - e, _ := watchEvent.Object.(*api.Event) - // Log all events for now. - log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp) + e, _ := watchEvent.Object.(*api.Event) + // Log all events for now. + log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp) - send := false - color := "" - if watchEvent.Type == watch.Added { - send = true - color = "good" - } else if watchEvent.Type == watch.Deleted { - send = true - color = "warning" - } else if e.Reason == "SuccessfulCreate" { - send = true - color = "good" - } else if e.Reason == "NodeReady" { - send = true - color = "good" - } else if e.Reason == "NodeNotReady" { - send = true - color = "warning" - } else if e.Reason == "NodeOutOfDisk" { - send = true - color = "danger" - } + send := false + color := "" + if watchEvent.Type == watch.Added { + send = true + color = "good" + } else if watchEvent.Type == watch.Deleted { + send = true + color = "warning" + } else if e.Reason == "SuccessfulCreate" { + send = true + color = "good" + } else if e.Reason == "NodeReady" { + send = true + color = "good" + } else if e.Reason == "NodeNotReady" { + send = true + color = "warning" + } else if e.Reason == "NodeOutOfDisk" { + send = true + color = "danger" + } - // For now, dont alert multiple times, except if it's a backoff - if e.Count > 1 { - send = false - } - if e.Reason == "BackOff" && e.Count == 3 { - send = true - color = "danger" - } + // For now, dont alert multiple times, except if it's a backoff + if e.Count > 1 { + send = false + } + if e.Reason == "BackOff" && e.Count == 3 { + send = true + color = "danger" + } - if send { - err = sendMessage(e, color) - if err != nil { - log.Fatalf("sendMessage: %v", err) + if send { + err = sendMessage(e, color) + if err != nil { + log.Fatalf("sendMessage: %v", err) + } } } } From 185c94ff3f5300b1890cf4ae1829174489a2ab54 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 22 Jun 2016 19:47:05 -0600 Subject: [PATCH 7/9] quiet down. --- main.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 3c79876..f2fe5d9 100644 --- a/main.go +++ b/main.go @@ -90,6 +90,7 @@ func main() { case watchEvent, _ := <-w.ResultChan(): e, _ := watchEvent.Object.(*api.Event) + // Log all events for now. log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp) @@ -115,6 +116,11 @@ func main() { color = "danger" } + // kubelet is loud. + if e.Source.Component == "kubelet" { + send = false + } + // For now, dont alert multiple times, except if it's a backoff if e.Count > 1 { send = false @@ -125,10 +131,7 @@ func main() { } if send { - err = sendMessage(e, color) - if err != nil { - log.Fatalf("sendMessage: %v", err) - } + sendMessage(e, color) } } } From eb61cfaab6d2ebd81111c22ecf8e7952806d0210 Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 6 Jul 2016 16:29:49 -0600 Subject: [PATCH 8/9] controllermanager is also loud. --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index f2fe5d9..a019809 100644 --- a/main.go +++ b/main.go @@ -116,9 +116,11 @@ func main() { color = "danger" } - // kubelet is loud. + // kubelet and controllermanager are loud. if e.Source.Component == "kubelet" { send = false + } else if e.Source.Component == "controllermanager" { + send = false } // For now, dont alert multiple times, except if it's a backoff From 1adfad3a391a46ec9f64a4e3723b75983561e0ed Mon Sep 17 00:00:00 2001 From: Matt Tucker Date: Wed, 6 Jul 2016 16:40:42 -0600 Subject: [PATCH 9/9] default-scheduler is also loud --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index a019809..0883cf3 100644 --- a/main.go +++ b/main.go @@ -121,6 +121,8 @@ func main() { send = false } else if e.Source.Component == "controllermanager" { send = false + } else if e.Source.Component == "default-scheduler" { + send = false } // For now, dont alert multiple times, except if it's a backoff