Skip to content

Use the Go API instead of REST #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ultimateboy wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Use the Go API instead of REST #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e515728
use the kube goapi instead of the rest api.
Jun 23, 2016
68550cc
watching all events, not just pods.
Jun 23, 2016
0c93e4b
log all events.
Jun 23, 2016
e4cdaee
bring back more alerts.
Jun 23, 2016
1825f7e
fix import and bad type.
Jun 23, 2016
f22b8b5
loop
Jun 23, 2016
185c94f
quiet down.
Jun 23, 2016
eb61cfa
controllermanager is also loud.
Jul 6, 2016
1adfad3
default-scheduler is also loud
Jul 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions examples/slack8s-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,4 @@ spec:
configMapKeyRef:
name: slack8s
key: slack-channel
- name: kube-proxy
image: gcr.io/google_containers/kubectl:v0.18.0-120-gaeb4ac55ad12b1-dirty
imagePullPolicy: Always
args: ["proxy", "-p", "8001"]

171 changes: 63 additions & 108 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,57 +1,25 @@
package main

import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"

"github.com/nlopes/slack"
)

// The GET request to the Kubernetes event watch API returns a JSON object
// which unmarshals into this Response type.
type Response struct {
Type string `json:"type"`
Object Event `json:"object"`
}

// The Event type and its child-types, contain only the values of the response
// that our alerts currently care about.
type Event struct {
Source EventSource `json:"source"`
InvolvedObject EventInvolvedObject `json:"involvedObject"`
Metadata EventMetadata `json:"metadata"`
Reason string `json:"reason"`
Message string `json:"message"`
FirstTimestamp time.Time `json:"firstTimestamp"`
LastTimestamp time.Time `json:"lastTimestamp"`
Count int `json:"count"`
}

type EventMetadata struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
}

type EventSource struct {
Component string `json:"component"`
}

type EventInvolvedObject struct {
Kind string `json:"kind"`
}
"k8s.io/kubernetes/pkg/api"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/labels"
"k8s.io/kubernetes/pkg/watch"
)

// Sends a message to the Slack channel about the Event.
func send_message(e Event, color string) error {
func sendMessage(e *api.Event, color string) error {
api := slack.New(os.Getenv("SLACK_TOKEN"))
params := slack.PostMessageParameters{}
metadata := e.GetObjectMeta()
attachment := slack.Attachment{
// The fallback message shows in clients such as IRC or OS X notifications.
Fallback: e.Message,
Expand All @@ -67,7 +35,7 @@ func send_message(e Event, color string) error {
},
slack.AttachmentField{
Title: "Name",
Value: e.Metadata.Name,
Value: metadata.GetName(),
Short: true,
},
slack.AttachmentField{
Expand Down Expand Up @@ -104,83 +72,70 @@ func send_message(e Event, color string) error {
}

func main() {
url := fmt.Sprintf("http://localhost:8001/api/v1/events?watch=true")
req, err := http.NewRequest("GET", url, nil)

kubeClient, err := client.NewInCluster()
if err != nil {
log.Fatal("NewRequest: ", err)
log.Fatalf("Failed to create client: %v", err)
}
client := &http.Client{}
resp, err := client.Do(req)

// Setup a watcher for events.
eventClient := kubeClient.Events(api.NamespaceAll)
options := api.ListOptions{LabelSelector: labels.Everything()}
w, err := eventClient.Watch(options)
if err != nil {
log.Fatal("Do: ", err)
}
defer resp.Body.Close()
dec := json.NewDecoder(resp.Body)
if resp.StatusCode != 200 {
log.Printf(string(resp.Status) + ": " + string(resp.StatusCode))
log.Fatal("Non 200 status code returned from Kubernetes API.")
log.Fatalf("Failed to set up watch: %v", err)
}
for {
var r Response
if err := dec.Decode(&r); err == io.EOF {
log.Printf("EOF detected.")
break
} else if err != nil {
// Debug output to help when we've failed to decode.
htmlData, er := ioutil.ReadAll(resp.Body)
if er != nil {
log.Printf("Already failed to decode, but also failed to read response for log output.")
select {
case watchEvent, _ := <-w.ResultChan():

e, _ := watchEvent.Object.(*api.Event)

// Log all events for now.
log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(int(e.Count)), e.FirstTimestamp, e.LastTimestamp)

send := false
color := ""
if watchEvent.Type == watch.Added {
send = true
color = "good"
} else if watchEvent.Type == watch.Deleted {
send = true
color = "warning"
} else if e.Reason == "SuccessfulCreate" {
send = true
color = "good"
} else if e.Reason == "NodeReady" {
send = true
color = "good"
} else if e.Reason == "NodeNotReady" {
send = true
color = "warning"
} else if e.Reason == "NodeOutOfDisk" {
send = true
color = "danger"
}
log.Printf(string(htmlData))
log.Fatal("Decode: ", err)
}
e := r.Object

// Log all events for now.
log.Printf("Reason: %s\nMessage: %s\nCount: %s\nFirstTimestamp: %s\nLastTimestamp: %s\n\n", e.Reason, e.Message, strconv.Itoa(e.Count), e.FirstTimestamp, e.LastTimestamp)

send := false
color := ""

// @todo refactor the configuration of which things to post.
if e.Reason == "SuccessfulCreate" {
send = true
color = "good"
} else if e.Reason == "NodeReady" {
send = true
color = "good"
} else if e.Reason == "NodeNotReady" {
send = true
color = "warning"
} else if e.Reason == "NodeOutOfDisk" {
send = true
color = "danger"
}

// For now, dont alert multiple times, except if it's a backoff
if e.Count > 1 {
send = false
}
if e.Reason == "BackOff" && e.Count == 3 {
send = true
color = "danger"
}
// kubelet and controllermanager are loud.
if e.Source.Component == "kubelet" {
send = false
} else if e.Source.Component == "controllermanager" {
send = false
} else if e.Source.Component == "default-scheduler" {
send = false
}

// Do not send any events that are more than 1 minute old.
// This assumes events are processed quickly (very likely)
// in exchange for not re-notifying of events after a crash
// or fresh start.
diff := time.Now().Sub(e.LastTimestamp)
diffMinutes := int(diff.Minutes())
if diffMinutes > 1 {
log.Printf("Supressed %s minute old message: %s", strconv.Itoa(diffMinutes), e.Message)
send = false
}
// For now, dont alert multiple times, except if it's a backoff
if e.Count > 1 {
send = false
}
if e.Reason == "BackOff" && e.Count == 3 {
send = true
color = "danger"
}

if send {
err = send_message(e, color)
if err != nil {
log.Fatal("send_message: ", err)
if send {
sendMessage(e, color)
}
}
}
Expand Down