mox/metrics/panic.go
Mechiel Lukkien 3620d6f05e
initialize metric mox_panic_total with 0, so the alerting rule also catches the first panic for a label
increase() and rate() don't seem to assume a previous value of 0 when a vector
gets a first value for a label. you would think that an increase() on a
first-value mox_panic_total{"..."}=1 would return 1, and similar for rate(), but
that doesn't appear to be the behaviour. so we just explicitly initialize the
count to 0 for each possible label value. mox has more vector metrics, but
panics feels like the most important, and it's too much code to initialize them
all, for all combinations of label values. there is probably a better way that
fixes this for all cases...
2023-09-15 16:47:17 +02:00

73 lines
1.6 KiB
Go

package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var metricPanic = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "mox_panic_total",
Help: "Number of unhandled panics, by package.",
},
[]string{
"pkg",
},
)
type Panic string
const (
Ctl Panic = "ctl"
Import Panic = "import"
Serve Panic = "serve"
Imapserver Panic = "imapserver"
Mtastsdb Panic = "mtastsdb"
Queue Panic = "queue"
Smtpclient Panic = "smtpclient"
Smtpserver Panic = "smtpserver"
Dkimverify Panic = "dkimverify"
Spfverify Panic = "spfverify"
Upgradethreads Panic = "upgradethreads"
Importmanage Panic = "importmanage"
Importmessages Panic = "importmessages"
Webadmin Panic = "webadmin"
Webmailsendevent Panic = "webmailsendevent"
Webmail Panic = "webmail"
Webmailrequest Panic = "webmailrequest"
Webmailquery Panic = "webmailquery"
Webmailhandle Panic = "webmailhandle"
)
func init() {
// Ensure the panic counts are initialized to 0, so the query for change also picks
// up the first panic.
names := []Panic{
Ctl,
Import,
Serve,
Imapserver,
Mtastsdb,
Queue,
Smtpclient,
Smtpserver,
Dkimverify,
Spfverify,
Upgradethreads,
Importmanage,
Importmessages,
Webadmin,
Webmailsendevent,
Webmail,
Webmailrequest,
Webmailquery,
Webmailhandle,
}
for _, name := range names {
metricPanic.WithLabelValues(string(name)).Add(0)
}
}
func PanicInc(name Panic) {
metricPanic.WithLabelValues(string(name)).Inc()
}