Skip to content

Commit 70b3e76

Browse files
rossmcdonalddanielnelson
rossmcdonald
authored and
danielnelson
committed
Add input for receiving papertrail webhooks (influxdata#2038)
1 parent 58ee962 commit 70b3e76

File tree

8 files changed

+351
-4
lines changed

8 files changed

+351
-4
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ be deprecated eventually.
7171
- [#1100](https://github.com/influxdata/telegraf/issues/1100): Add collectd parser
7272
- [#1820](https://github.com/influxdata/telegraf/issues/1820): easier plugin testing without outputs
7373
- [#2493](https://github.com/influxdata/telegraf/pull/2493): Check signature in the GitHub webhook plugin
74+
- [#2038](https://github.com/influxdata/telegraf/issues/2038): Add papertrail support to webhooks
7475

7576
### Bugfixes
7677

‎plugins/inputs/webhooks/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ $ sudo service telegraf start
1919
- [Github](github/)
2020
- [Mandrill](mandrill/)
2121
- [Rollbar](rollbar/)
22+
- [Papertrail](papertrail/)
2223

2324
## Adding new webhooks plugin
2425

‎plugins/inputs/webhooks/papertrail/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# papertrail webhooks
2+
3+
Enables Telegraf to act as a [Papertrail Webhook](http://help.papertrailapp.com/kb/how-it-works/web-hooks/).
4+
5+
## Events
6+
7+
[Full documentation](http://help.papertrailapp.com/kb/how-it-works/web-hooks/#callback).
8+
9+
Events from Papertrail come in two forms:
10+
11+
* The [event-based callback](http://help.papertrailapp.com/kb/how-it-works/web-hooks/#callback):
12+
13+
* A point is created per event, with the timestamp as `received_at`
14+
* Each point has a field counter (`count`), which is set to `1` (signifying the event occurred)
15+
* Each event "hostname" object is converted to a `host` tag
16+
* The "saved_search" name in the payload is added as an `event` tag
17+
18+
* The [count-based callback](http://help.papertrailapp.com/kb/how-it-works/web-hooks/#count-only-webhooks)
19+
20+
* A point is created per timeseries object per count, with the timestamp as the "timeseries" key (the unix epoch of the event)
21+
* Each point has a field counter (`count`), which is set to the value of each "timeseries" object
22+
* Each count "source_name" object is converted to a `host` tag
23+
* The "saved_search" name in the payload is added as an `event` tag
24+
25+
The current functionality is very basic, however this allows you to
26+
track the number of events by host and saved search.
27+
28+
When an event is received, any point will look similar to:
29+
30+
```
31+
papertrail,host=myserver.example.com,event=saved_search_name count=3i 1453248892000000000
32+
```

‎plugins/inputs/webhooks/papertrail/papertrail_test.go‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package papertrail
2+
3+
import (
4+
"net/http"
5+
"net/http/httptest"
6+
"net/url"
7+
"strings"
8+
"testing"
9+
10+
"github.com/influxdata/telegraf/testutil"
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
const (
15+
contentType = "application/x-www-form-urlencoded"
16+
)
17+
18+
func post(pt *PapertrailWebhook, contentType string, body string) *httptest.ResponseRecorder {
19+
req, _ := http.NewRequest("POST", "/", strings.NewReader(body))
20+
req.Header.Set("Content-Type", contentType)
21+
w := httptest.NewRecorder()
22+
pt.eventHandler(w, req)
23+
return w
24+
}
25+
26+
func TestWrongContentType(t *testing.T) {
27+
var acc testutil.Accumulator
28+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
29+
form := url.Values{}
30+
form.Set("payload", sampleEventPayload)
31+
data := form.Encode()
32+
33+
resp := post(pt, "", data)
34+
require.Equal(t, http.StatusUnsupportedMediaType, resp.Code)
35+
}
36+
37+
func TestMissingPayload(t *testing.T) {
38+
var acc testutil.Accumulator
39+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
40+
41+
resp := post(pt, contentType, "")
42+
require.Equal(t, http.StatusBadRequest, resp.Code)
43+
}
44+
45+
func TestPayloadNotJSON(t *testing.T) {
46+
var acc testutil.Accumulator
47+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
48+
49+
resp := post(pt, contentType, "payload={asdf]")
50+
require.Equal(t, http.StatusBadRequest, resp.Code)
51+
}
52+
53+
func TestPayloadInvalidJSON(t *testing.T) {
54+
var acc testutil.Accumulator
55+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
56+
57+
resp := post(pt, contentType, `payload={"value": 42}`)
58+
require.Equal(t, http.StatusBadRequest, resp.Code)
59+
}
60+
61+
func TestEventPayload(t *testing.T) {
62+
var acc testutil.Accumulator
63+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
64+
65+
form := url.Values{}
66+
form.Set("payload", sampleEventPayload)
67+
resp := post(pt, contentType, form.Encode())
68+
require.Equal(t, http.StatusOK, resp.Code)
69+
70+
fields := map[string]interface{}{
71+
"count": uint64(1),
72+
}
73+
74+
tags1 := map[string]string{
75+
"event": "Important stuff",
76+
"host": "abc",
77+
}
78+
tags2 := map[string]string{
79+
"event": "Important stuff",
80+
"host": "def",
81+
}
82+
83+
acc.AssertContainsTaggedFields(t, "papertrail", fields, tags1)
84+
acc.AssertContainsTaggedFields(t, "papertrail", fields, tags2)
85+
}
86+
87+
func TestCountPayload(t *testing.T) {
88+
var acc testutil.Accumulator
89+
pt := &PapertrailWebhook{Path: "/papertrail", acc: &acc}
90+
form := url.Values{}
91+
form.Set("payload", sampleCountPayload)
92+
resp := post(pt, contentType, form.Encode())
93+
require.Equal(t, http.StatusOK, resp.Code)
94+
95+
fields1 := map[string]interface{}{
96+
"count": uint64(5),
97+
}
98+
fields2 := map[string]interface{}{
99+
"count": uint64(3),
100+
}
101+
102+
tags1 := map[string]string{
103+
"event": "Important stuff",
104+
"host": "arthur",
105+
}
106+
tags2 := map[string]string{
107+
"event": "Important stuff",
108+
"host": "ford",
109+
}
110+
111+
acc.AssertContainsTaggedFields(t, "papertrail", fields1, tags1)
112+
acc.AssertContainsTaggedFields(t, "papertrail", fields2, tags2)
113+
}
114+
115+
const sampleEventPayload = `{
116+
"events": [
117+
{
118+
"id": 7711561783320576,
119+
"received_at": "2011-05-18T20:30:02-07:00",
120+
"display_received_at": "May 18 20:30:02",
121+
"source_ip": "208.75.57.121",
122+
"source_name": "abc",
123+
"source_id": 2,
124+
"hostname": "abc",
125+
"program": "CROND",
126+
"severity": "Info",
127+
"facility": "Cron",
128+
"message": "message body"
129+
},
130+
{
131+
"id": 7711562567655424,
132+
"received_at": "2011-05-18T20:30:02-07:00",
133+
"display_received_at": "May 18 20:30:02",
134+
"source_ip": "208.75.57.120",
135+
"source_name": "server1",
136+
"source_id": 19,
137+
"hostname": "def",
138+
"program": "CROND",
139+
"severity": "Info",
140+
"facility": "Cron",
141+
"message": "A short event"
142+
}
143+
],
144+
"saved_search": {
145+
"id": 42,
146+
"name": "Important stuff",
147+
"query": "cron OR server1",
148+
"html_edit_url": "https://papertrailapp.com/searches/42/edit",
149+
"html_search_url": "https://papertrailapp.com/searches/42"
150+
},
151+
"max_id": "7711582041804800",
152+
"min_id": "7711561783320576"
153+
}`
154+
155+
const sampleCountPayload = `{
156+
"counts": [
157+
{
158+
"source_name": "arthur",
159+
"source_id": 4,
160+
"timeseries": {
161+
"1453248895": 5
162+
}
163+
},
164+
{
165+
"source_name": "ford",
166+
"source_id": 3,
167+
"timeseries": {
168+
"1453248927": 3
169+
}
170+
}
171+
],
172+
"saved_search": {
173+
"id": 42,
174+
"name": "Important stuff",
175+
"query": "cron OR server1",
176+
"html_edit_url": "https://papertrailapp.com/searches/42/edit",
177+
"html_search_url": "https://papertrailapp.com/searches/42"
178+
},
179+
"max_id": "7711582041804800",
180+
"min_id": "7711561783320576"
181+
}`

‎plugins/inputs/webhooks/papertrail/papertrail_webhooks.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package papertrail
2+
3+
import (
4+
"encoding/json"
5+
"log"
6+
"net/http"
7+
"time"
8+
9+
"github.com/gorilla/mux"
10+
"github.com/influxdata/telegraf"
11+
)
12+
13+
type PapertrailWebhook struct {
14+
Path string
15+
acc telegraf.Accumulator
16+
}
17+
18+
func (pt *PapertrailWebhook) Register(router *mux.Router, acc telegraf.Accumulator) {
19+
router.HandleFunc(pt.Path, pt.eventHandler).Methods("POST")
20+
log.Printf("I! Started the papertrail_webhook on %s", pt.Path)
21+
pt.acc = acc
22+
}
23+
24+
func (pt *PapertrailWebhook) eventHandler(w http.ResponseWriter, r *http.Request) {
25+
if r.Header.Get("Content-Type") != "application/x-www-form-urlencoded" {
26+
http.Error(w, "Unsupported Media Type", http.StatusUnsupportedMediaType)
27+
return
28+
}
29+
30+
data := r.PostFormValue("payload")
31+
if data == "" {
32+
http.Error(w, "Bad Request", http.StatusBadRequest)
33+
return
34+
}
35+
36+
var payload Payload
37+
err := json.Unmarshal([]byte(data), &payload)
38+
if err != nil {
39+
http.Error(w, "Bad Request", http.StatusBadRequest)
40+
return
41+
}
42+
43+
if payload.Events != nil {
44+
45+
// Handle event-based payload
46+
for _, e := range payload.Events {
47+
// Warning: Duplicate event timestamps will overwrite each other
48+
tags := map[string]string{
49+
"host": e.Hostname,
50+
"event": payload.SavedSearch.Name,
51+
}
52+
fields := map[string]interface{}{
53+
"count": uint64(1),
54+
}
55+
pt.acc.AddFields("papertrail", fields, tags, e.ReceivedAt)
56+
}
57+
58+
} else if payload.Counts != nil {
59+
60+
// Handle count-based payload
61+
for _, c := range payload.Counts {
62+
for ts, count := range *c.TimeSeries {
63+
tags := map[string]string{
64+
"host": c.SourceName,
65+
"event": payload.SavedSearch.Name,
66+
}
67+
fields := map[string]interface{}{
68+
"count": count,
69+
}
70+
pt.acc.AddFields("papertrail", fields, tags, time.Unix(ts, 0))
71+
}
72+
}
73+
} else {
74+
http.Error(w, "Bad Request", http.StatusBadRequest)
75+
return
76+
}
77+
78+
w.WriteHeader(http.StatusOK)
79+
}

‎plugins/inputs/webhooks/papertrail/papertrail_webhooks_models.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package papertrail
2+
3+
import (
4+
"time"
5+
)
6+
7+
type Event struct {
8+
ID int64 `json:"id"`
9+
ReceivedAt time.Time `json:"received_at"`
10+
DisplayReceivedAt string `json:"display_received_at"`
11+
SourceIP string `json:"source_ip"`
12+
SourceName string `json:"source_name"`
13+
SourceID int `json:"source_id"`
14+
Hostname string `json:"hostname"`
15+
Program string `json:"program"`
16+
Severity string `json:"severity"`
17+
Facility string `json:"facility"`
18+
Message string `json:"message"`
19+
}
20+
21+
type Count struct {
22+
SourceName string `json:"source_name"`
23+
SourceID int64 `json:"source_id"`
24+
TimeSeries *map[int64]uint64 `json:"timeseries"`
25+
}
26+
27+
type SavedSearch struct {
28+
ID int64 `json:"id"`
29+
Name string `json:"name"`
30+
Query string `json:"query"`
31+
EditURL string `json:"html_edit_url"`
32+
SearchURL string `json:"html_search_url"`
33+
}
34+
35+
type Payload struct {
36+
Events []*Event `json:"events"`
37+
Counts []*Count `json:"counts"`
38+
SavedSearch *SavedSearch `json:"saved_search"`
39+
MaxID string `json:"max_id"`
40+
MinID string `json:"min_id"`
41+
}

‎plugins/inputs/webhooks/webhooks.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/influxdata/telegraf/plugins/inputs/webhooks/filestack"
1414
"github.com/influxdata/telegraf/plugins/inputs/webhooks/github"
1515
"github.com/influxdata/telegraf/plugins/inputs/webhooks/mandrill"
16+
"github.com/influxdata/telegraf/plugins/inputs/webhooks/papertrail"
1617
"github.com/influxdata/telegraf/plugins/inputs/webhooks/rollbar"
1718
)
1819

@@ -27,10 +28,11 @@ func init() {
2728
type Webhooks struct {
2829
ServiceAddress string
2930

30-
Github *github.GithubWebhook
31-
Filestack *filestack.FilestackWebhook
32-
Mandrill *mandrill.MandrillWebhook
33-
Rollbar *rollbar.RollbarWebhook
31+
Github *github.GithubWebhook
32+
Filestack *filestack.FilestackWebhook
33+
Mandrill *mandrill.MandrillWebhook
34+
Rollbar *rollbar.RollbarWebhook
35+
Papertrail *papertrail.PapertrailWebhook
3436
}
3537

3638
func NewWebhooks() *Webhooks {
@@ -54,6 +56,9 @@ func (wb *Webhooks) SampleConfig() string {
5456
5557
[inputs.webhooks.rollbar]
5658
path = "/rollbar"
59+
60+
[inputs.webhooks.papertrail]
61+
path = "/papertrail"
5762
`
5863
}
5964

0 commit comments

Comments
 (0)