本文通过修改alermanager的webhook方式调用飞书，发送飞书卡片
发送机制
webhook的实现位于alertmanager/notify/webhook.go
总的来说就是包装msg并发送http请求，重点在于需要修改的是msg对象。
// Notify implements the Notifier interface.
func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
groupKey, err := notify.ExtractGroupKey(ctx)
if err != nil {
level.Error(n.logger).Log("err", err)
}
msg := &Message{
Version: "4",
Data: data,
GroupKey: groupKey.String(),
TruncatedAlerts: numTruncated,
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
req, err := http.NewRequest("POST", n.conf.URL.String(), &buf)
if err != nil {
return true, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", userAgentHeader)
resp, err := n.client.Do(req.WithContext(ctx))
if err != nil {
return true, err
}
notify.Drain(resp)
return n.retrier.Check(resp.StatusCode, nil)
}
Data对象
type Data struct {
Receiver string `json:"receiver"`
Status string `json:"status"`
Alerts Alerts `json:"alerts"`
GroupLabels KV `json:"groupLabels"`
CommonLabels KV `json:"commonLabels"`
CommonAnnotations KV `json:"commonAnnotations"`
ExternalURL string `json:"externalURL"`
}
对应序列化之后的json(取自官方文档)
{
"version": "4",
"groupKey": <string>, // key identifying the group of alerts (e.g. to deduplicate)
"truncatedAlerts": <int>, // how many alerts have been truncated due to "max_alerts"
"status": "<resolved|firing>",
"receiver": <string>,
"groupLabels": <object>,
"commonLabels": <object>,
"commonAnnotations": <object>,
"externalURL": <string>, // backlink to the Alertmanager.
"alerts": [
{
"status": "<resolved|firing>",
"labels": <object>,
"annotations": <object>,
"startsAt": "<rfc3339>",
"endsAt": "<rfc3339>",
"generatorURL": <string> // identifies the entity that caused the alert
},
]
}
飞书所需格式
飞书格式.png
开搞
把msg转换一下就完事了
这是转换类,需要注意的一点是我转换的Annotations里面的行对应的是rule.yml里配置的annotation，因为我配置的三行信息是项目名，环境，ip所以将三行信息写了进去。
package webhook
import (
"strings"
"time"
)
type FSMessagev2 struct {
MsgType string `json:"msg_type"`
Card Cards `json:"card"`
}
type Te struct {
Content string `json:"content,omitempty"`
Tag string `json:"tag,omitempty"`
}
type Element struct {
Tag string `json:"tag,omitempty"`
Text Te `json:"text,omitempty"`
Elements []Te `json:"elements,omitempty"`
}
type Cards struct {
Config Conf `json:"config"`
Elements []Element `json:"elements"`
}
type Conf struct {
WideScreenMode bool `json:"wide_screen_mode"`
}
func transformMsgToFs(message *Message) *FSMessagev2 {
var NEXT_LINE = "\n"
list := make([]Element, 0)
for index, msg := range message.Data.Alerts {
var text = ""
if index == 0 {
text = "**" + msg.Labels.Values()[0] + "**" + NEXT_LINE
}
app := strings.Trim(msg.Annotations.Values()[0],"_")
env := strings.Trim(msg.Annotations.Values()[1],"_")
ip := strings.Trim(msg.Annotations.Values()[2],"_")
text += "["+app+"("+env+")]("+"http://gitlab.hio.cn/hio/"+app+"/tree/"+env+") " + ip + NEXT_LINE
if msg.Status == "firing" {
msg.Status = "🔥"
text += "状态:" + msg.Status
list = append(list, Element{Tag: "div", Text: Te{Tag: "lark_md", Content: text}})
list = append(list, Element{Tag: "note", Elements: []Te{{Tag: "plain_text", Content: msg.StartsAt.In(time.Local).Format("2006-01-02 15:04:05")}}})
} else {
msg.Status = "✅"
text += "状态:" + msg.Status
list = append(list, Element{Tag: "div", Text: Te{Tag: "lark_md", Content: text}})
list = append(list, Element{Tag: "note", Elements: []Te{{Tag: "plain_text", Content: msg.StartsAt.In(time.Local).Format("2006-01-02 15:04:05") + " ~ " + msg.EndsAt.In(time.Local).Format("2006-01-02 15:04:05")}}})
}
if index < len(message.Data.Alerts)-1{
list = append(list, Element{Tag: "hr"})
}
}
feishu := &FSMessagev2{
MsgType: "interactive",
Card: Cards{
Config: Conf{
WideScreenMode: false,
},
Elements: list,
},
}
return feishu
}
将转换类在webhook中调用
func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
groupKey, err := notify.ExtractGroupKey(ctx)
if err != nil {
level.Error(n.logger).Log("err", err)
}
msg := &Message{
Version: "4",
Data: data,
GroupKey: groupKey.String(),
TruncatedAlerts: numTruncated,
}
//变化在这里
fs := transformMsgToFs(msg)
//b,_ := json.Marshal(fs)
//fmt.Println(string(b))
buf := new(bytes.Buffer)
//变化在这里
json.NewEncoder(buf).Encode(fs)
var tr *http.Transport
tr = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
res, err := client.Post(n.conf.URL.String(), "application/json", buf)
if err != nil {
return true, err
}
defer res.Body.Close()
notify.Drain(res)
return n.retrier.Check(res.StatusCode, nil)
}
我的告警格式
image.png
对应的label设置
# 从eureka拉取微服务信息，通过relable替换为采集path，需配合eureka.instantce.metaMap的配置
- job_name: production
eureka_sd_configs:
- server: 'http://XXXX/eureka'
- server: 'http://XXXXX/eureka'
relabel_configs:
- source_labels: [ __meta_eureka_app_instance_metadata_prometheus_path ]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [ __meta_eureka_app_instance_port ]
action: keep
regex: (8086|8081)
- source_labels: [ __meta_eureka_app_instance_ip_addr ]
action: replace
target_label: instance
- source_labels: [ job ]
target_label: env
- action: labeldrop
regex: (job)
# - source_labels: [ __meta_eureka_app_instance_metadata_prometheus_env ]
# target_label: env
- source_labels: [ __meta_eureka_app_instance_vip_address ]
target_label: application
飞书结果
image.png
