diff --git a/examples/slack8s-deployment.yaml b/examples/slack8s-deployment.yaml index fecdf5e..7d771b6 100644 --- a/examples/slack8s-deployment.yaml +++ b/examples/slack8s-deployment.yaml @@ -33,8 +33,4 @@ spec: configMapKeyRef: name: slack8s key: slack-channel - - name: kube-proxy - image: gcr.io/google_containers/kubectl:v0.18.0-120-gaeb4ac55ad12b1-dirty - imagePullPolicy: Always - args: ["proxy", "-p", "8001"] diff --git a/main.go b/main.go index 553680c..0883cf3 100644 --- a/main.go +++ b/main.go @@ -1,57 +1,25 @@ package main import ( - "encoding/json" "fmt" - "io" - "io/ioutil" "log" - "net/http" "os" "strconv" "strings" - "time" "github.com/nlopes/slack" -) - -// The GET request to the Kubernetes event watch API returns a JSON object -// which unmarshals into this Response type. -type Response struct { - Type string `json:"type"` - Object Event `json:"object"` -} -// The Event type and its child-types, contain only the values of the response -// that our alerts currently care about. -type Event struct { - Source EventSource `json:"source"` - InvolvedObject EventInvolvedObject `json:"involvedObject"` - Metadata EventMetadata `json:"metadata"` - Reason string `json:"reason"` - Message string `json:"message"` - FirstTimestamp time.Time `json:"firstTimestamp"` - LastTimestamp time.Time `json:"lastTimestamp"` - Count int `json:"count"` -} - -type EventMetadata struct { - Name string `json:"name"` - Namespace string `json:"namespace"` -} - -type EventSource struct { - Component string `json:"component"` -} - -type EventInvolvedObject struct { - Kind string `json:"kind"` -} + "k8s.io/kubernetes/pkg/api" + client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/watch" +) // Sends a message to the Slack channel about the Event. -func send_message(e Event, color string) error { +func sendMessage(e *api.Event, color string) error { api := slack.New(os.Getenv("SLACK_TOKEN")) params := slack.PostMessageParameters{} + metadata := e.GetObjectMeta() attachment := slack.Attachment{ // The fallback message shows in clients such as IRC or OS X notifications. Fallback: e.Message, @@ -67,7 +35,7 @@ func send_message(e Event, color string) error { }, slack.AttachmentField{ Title: "Name", - Value: e.Metadata.Name, + Value: metadata.GetName(), Short: true, }, slack.AttachmentField{ @@ -104,83 +72,70 @@ func send_message(e Event, color string) error { } func main() { - url := fmt.Sprintf("http://localhost:8001/api/v1/events?watch=true") - req, err := http.NewRequest("GET", url, nil) + + kubeClient, err := client.NewInCluster() if err != nil { - log.Fatal("NewRequest: ", err) + log.Fatalf("Failed to create client: %v", err) } - client := &http.Client{} - resp, err := client.Do(req) + + // Setup a watcher for events. + eventClient := kubeClient.Events(api.NamespaceAll) + options := api.ListOptions{LabelSelector: labels.Everything()} + w, err := eventClient.Watch(options) if err != nil { - log.Fatal("Do: ", err) - } - defer resp.Body.Close() - dec := json.NewDecoder(resp.Body) - if resp.StatusCode != 200 { - log.Printf(string(resp.Status) + ": " + string(resp.StatusCode)) - log.Fatal("Non 200 status code returned from Kubernetes API.") + log.Fatalf("Failed to set up watch: %v", err) } for { - var r Response - if err := dec.Decode(&r); err == io.EOF { - log.Printf("EOF detected.") - break - } else if err != nil { - // Debug output to help when we've failed to decode. - htmlData, er := ioutil.ReadAll(resp.Body) - if er != nil { - log.Printf("Already failed to decode, but also failed to read response for log output.") + select { + case watchEvent, _ := <-w.ResultChan(): + + e, _ := watchEvent.Object.(*api.Event) + + // Log all events for now. + log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp) + + send := false + color := "" + if watchEvent.Type == watch.Added { + send = true + color = "good" + } else if watchEvent.Type == watch.Deleted { + send = true + color = "warning" + } else if e.Reason == "SuccessfulCreate" { + send = true + color = "good" + } else if e.Reason == "NodeReady" { + send = true + color = "good" + } else if e.Reason == "NodeNotReady" { + send = true + color = "warning" + } else if e.Reason == "NodeOutOfDisk" { + send = true + color = "danger" } - log.Printf(string(htmlData)) - log.Fatal("Decode: ", err) - } - e := r.Object - - // Log all events for now. - log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp) - - send := false - color := "" - - // @todo refactor the configuration of which things to post. - if e.Reason == "SuccessfulCreate" { - send = true - color = "good" - } else if e.Reason == "NodeReady" { - send = true - color = "good" - } else if e.Reason == "NodeNotReady" { - send = true - color = "warning" - } else if e.Reason == "NodeOutOfDisk" { - send = true - color = "danger" - } - // For now, dont alert multiple times, except if it's a backoff - if e.Count > 1 { - send = false - } - if e.Reason == "BackOff" && e.Count == 3 { - send = true - color = "danger" - } + // kubelet and controllermanager are loud. + if e.Source.Component == "kubelet" { + send = false + } else if e.Source.Component == "controllermanager" { + send = false + } else if e.Source.Component == "default-scheduler" { + send = false + } - // Do not send any events that are more than 1 minute old. - // This assumes events are processed quickly (very likely) - // in exchange for not re-notifying of events after a crash - // or fresh start. - diff := time.Now().Sub(e.LastTimestamp) - diffMinutes := int(diff.Minutes()) - if diffMinutes > 1 { - log.Printf("Supressed %s minute old message: %s", strconv.Itoa(diffMinutes), e.Message) - send = false - } + // For now, dont alert multiple times, except if it's a backoff + if e.Count > 1 { + send = false + } + if e.Reason == "BackOff" && e.Count == 3 { + send = true + color = "danger" + } - if send { - err = send_message(e, color) - if err != nil { - log.Fatal("send_message: ", err) + if send { + sendMessage(e, color) } } }