import "github.com/sirupsen/logrus"
func handleRequest() {
if err := doSomething(); err != nil {
logrus.WithError(err).WithField("service", "payment").Error("处理支付失败")
}
}
var errorCounts = sync.Map{} // key: serviceName, value: int
func recordError(service string) {
count, _ := errorCounts.LoadOrStore(service, 0)
newCount := count.(int) + 1
errorCounts.Store(service, newCount)
if newCount > 10 { // 阈值设为10
triggerAlert(service, newCount)
}
import "net/http"
import "bytes"
import "encoding/json"
func sendDingTalkAlert(service string, count int) {
url := "https://www.php.cn/link/93b5129e24b9c92e5b8e7115056b46bd"
payload := map[string]interface{}{
"msgtype": "text",
"text": map[string]string{
"content": fmt.Sprintf("[预警] 服务 %s 在过去1分钟内发生 %d 次错误", service, count),
},
}
data, _ := json.Marshal(payload)
http.Post(url, "application/json", bytes.NewBuffer(data))
}
import "github.com/prometheus/client_golang/prometheus"
var errorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "service_errors_total",
Help: "Total number of service errors",
},
[]string{"service"},
)
func init() {
prometheus.MustRegister(errorCounter)
}
func recordErrorPrometheus(service string) {
errorCounter.WithLabelValues(service).Inc()
}
- alert: HighErrorRate
expr: rate(service_errors_total[5m]) > 2 // 每分钟超过2次
for: 2m
labels:
severity: warning
annotations:
summary: "高错误率: {{ $labels.service }}"
