Move some code around
Move the metrics code into its own package. Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira
parent
a3ddb5e63a
commit
8f56f51598
6 changed files with 179 additions and 67 deletions
|
|
@ -20,10 +20,12 @@ import (
|
|||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"garm/apiserver/params"
|
||||
"garm/auth"
|
||||
gErrors "garm/errors"
|
||||
"garm/metrics"
|
||||
runnerParams "garm/params"
|
||||
"garm/runner"
|
||||
"garm/util"
|
||||
|
|
@ -31,28 +33,32 @@ import (
|
|||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
func NewAPIController(r *runner.Runner, auth *auth.Authenticator, hub *wsWriter.Hub, controllerInfo runnerParams.ControllerInfo) (*APIController, error) {
|
||||
func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *wsWriter.Hub) (*APIController, error) {
|
||||
controllerInfo, err := r.GetControllerInfo(auth.GetAdminContext())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting controller info")
|
||||
}
|
||||
return &APIController{
|
||||
r: r,
|
||||
auth: auth,
|
||||
auth: authenticator,
|
||||
hub: hub,
|
||||
upgrader: websocket.Upgrader{
|
||||
ReadBufferSize: 1024,
|
||||
WriteBufferSize: 16384,
|
||||
},
|
||||
controllerInfo: controllerInfo,
|
||||
cachedControllerInfo: controllerInfo,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type APIController struct {
|
||||
r *runner.Runner
|
||||
auth *auth.Authenticator
|
||||
hub *wsWriter.Hub
|
||||
upgrader websocket.Upgrader
|
||||
controllerInfo runnerParams.ControllerInfo
|
||||
r *runner.Runner
|
||||
auth *auth.Authenticator
|
||||
hub *wsWriter.Hub
|
||||
upgrader websocket.Upgrader
|
||||
cachedControllerInfo runnerParams.ControllerInfo
|
||||
mux sync.Mutex
|
||||
}
|
||||
|
||||
func handleError(w http.ResponseWriter, err error) {
|
||||
|
|
@ -89,18 +95,30 @@ func handleError(w http.ResponseWriter, err error) {
|
|||
}
|
||||
}
|
||||
|
||||
// metric to count total webhooks received
|
||||
// at this point the webhook is not yet authenticated and
|
||||
// we don't know if it's meant for us or not
|
||||
var webhooksReceived = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "garm_webhooks_received",
|
||||
Help: "The total number of webhooks received",
|
||||
}, []string{"valid", "reason", "hostname", "controller_id"})
|
||||
|
||||
func init() {
|
||||
err := prometheus.Register(webhooksReceived)
|
||||
// controllerInfo calls into runner.GetControllerInfo(), but instead of erroring out, will
|
||||
// fall back on a cached version of that info. If successful, the cached version is updated.
|
||||
func (a *APIController) controllerInfo() runnerParams.ControllerInfo {
|
||||
// Atempt to fetch controller info. We do this on every call, in case the hostname
|
||||
// changes while garm is running. The ControllerID will never change, once initialized.
|
||||
info, err := a.r.GetControllerInfo(auth.GetAdminContext())
|
||||
if err != nil {
|
||||
log.Printf("error registering prometheus metric: %q", err)
|
||||
// The call may fail, but we shouldn't loose metrics just because something went
|
||||
// terribly wrong while fetching the hostname.
|
||||
log.Printf("failed to get new controller info; falling back on cached version: %s", err)
|
||||
return a.cachedControllerInfo
|
||||
}
|
||||
// Set new controller info and return it.
|
||||
a.mux.Lock()
|
||||
defer a.mux.Unlock()
|
||||
a.cachedControllerInfo = info
|
||||
return a.cachedControllerInfo
|
||||
}
|
||||
|
||||
func (a *APIController) webhookMetricLabelValues(valid, reason string) []string {
|
||||
controllerInfo := a.controllerInfo()
|
||||
return []string{
|
||||
valid, reason,
|
||||
controllerInfo.Hostname, controllerInfo.ControllerID.String(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -115,23 +133,31 @@ func (a *APIController) handleWorkflowJobEvent(w http.ResponseWriter, r *http.Re
|
|||
signature := r.Header.Get("X-Hub-Signature-256")
|
||||
hookType := r.Header.Get("X-Github-Hook-Installation-Target-Type")
|
||||
|
||||
controllerInfo := a.r.GetControllerInfo(r.Context())
|
||||
var labelValues []string
|
||||
defer func() {
|
||||
if len(labelValues) == 0 {
|
||||
return
|
||||
}
|
||||
if err := metrics.RecordWebhookWithLabels(labelValues...); err != nil {
|
||||
log.Printf("failed to record metric: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := a.r.DispatchWorkflowJob(hookType, signature, body); err != nil {
|
||||
if errors.Is(err, gErrors.ErrNotFound) {
|
||||
webhooksReceived.WithLabelValues("false", "owner_unknown", controllerInfo.Hostname, controllerInfo.ControllerID.String()).Inc()
|
||||
labelValues = a.webhookMetricLabelValues("false", "owner_unknown")
|
||||
log.Printf("got not found error from DispatchWorkflowJob. webhook not meant for us?: %q", err)
|
||||
return
|
||||
} else if strings.Contains(err.Error(), "signature") { // TODO: check error type
|
||||
webhooksReceived.WithLabelValues("false", "signature_invalid", controllerInfo.Hostname, controllerInfo.ControllerID.String()).Inc()
|
||||
labelValues = a.webhookMetricLabelValues("false", "signature_invalid")
|
||||
} else {
|
||||
webhooksReceived.WithLabelValues("false", "unknown", controllerInfo.Hostname, controllerInfo.ControllerID.String()).Inc()
|
||||
labelValues = a.webhookMetricLabelValues("false", "unknown")
|
||||
}
|
||||
|
||||
handleError(w, err)
|
||||
return
|
||||
}
|
||||
webhooksReceived.WithLabelValues("true", "", controllerInfo.Hostname, controllerInfo.ControllerID.String()).Inc()
|
||||
labelValues = a.webhookMetricLabelValues("true", "")
|
||||
}
|
||||
|
||||
func (a *APIController) CatchAll(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
|
|||
|
|
@ -1,126 +0,0 @@
|
|||
package controllers
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"garm/auth"
|
||||
"garm/runner"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type GarmCollector struct {
|
||||
healthMetric *prometheus.Desc
|
||||
instanceMetric *prometheus.Desc
|
||||
runner *runner.Runner
|
||||
}
|
||||
|
||||
func NewGarmCollector(r *runner.Runner) *GarmCollector {
|
||||
return &GarmCollector{
|
||||
runner: r,
|
||||
instanceMetric: prometheus.NewDesc(
|
||||
"garm_runner_status",
|
||||
"Status of the runner",
|
||||
[]string{"name", "status", "runner_status", "pool_owner", "pool_type", "pool_id", "hostname", "controller_id"}, nil,
|
||||
),
|
||||
healthMetric: prometheus.NewDesc(
|
||||
"garm_health",
|
||||
"Health of the runner",
|
||||
[]string{"hostname", "controller_id"}, nil,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *GarmCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- c.instanceMetric
|
||||
ch <- c.healthMetric
|
||||
}
|
||||
|
||||
func (c *GarmCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
controllerInfo := c.runner.GetControllerInfo(auth.GetAdminContext())
|
||||
|
||||
c.CollectInstanceMetric(ch, controllerInfo.Hostname, controllerInfo.ControllerID.String())
|
||||
c.CollectHealthMetric(ch, controllerInfo.Hostname, controllerInfo.ControllerID.String())
|
||||
}
|
||||
|
||||
func (c *GarmCollector) CollectHealthMetric(ch chan<- prometheus.Metric, hostname string, controllerID string) {
|
||||
m, err := prometheus.NewConstMetric(
|
||||
c.healthMetric,
|
||||
prometheus.GaugeValue,
|
||||
1,
|
||||
hostname,
|
||||
controllerID,
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("error on creating health metric: %s", err)
|
||||
return
|
||||
}
|
||||
ch <- m
|
||||
}
|
||||
|
||||
// CollectInstanceMetric collects the metrics for the runner instances
|
||||
// reflecting the statuses and the pool they belong to.
|
||||
func (c *GarmCollector) CollectInstanceMetric(ch chan<- prometheus.Metric, hostname string, controllerID string) {
|
||||
|
||||
ctx := auth.GetAdminContext()
|
||||
|
||||
instances, err := c.runner.ListAllInstances(ctx)
|
||||
if err != nil {
|
||||
log.Printf("cannot collect metrics, listing instances: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
pools, err := c.runner.ListAllPools(ctx)
|
||||
if err != nil {
|
||||
log.Printf("listing pools: %s", err)
|
||||
// continue anyway
|
||||
}
|
||||
|
||||
type poolInfo struct {
|
||||
Name string
|
||||
Type string
|
||||
}
|
||||
|
||||
poolNames := make(map[string]poolInfo)
|
||||
for _, pool := range pools {
|
||||
if pool.EnterpriseName != "" {
|
||||
poolNames[pool.ID] = poolInfo{
|
||||
Name: pool.EnterpriseName,
|
||||
Type: string(pool.PoolType()),
|
||||
}
|
||||
} else if pool.OrgName != "" {
|
||||
poolNames[pool.ID] = poolInfo{
|
||||
Name: pool.OrgName,
|
||||
Type: string(pool.PoolType()),
|
||||
}
|
||||
} else {
|
||||
poolNames[pool.ID] = poolInfo{
|
||||
Name: pool.RepoName,
|
||||
Type: string(pool.PoolType()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, instance := range instances {
|
||||
|
||||
m, err := prometheus.NewConstMetric(
|
||||
c.instanceMetric,
|
||||
prometheus.GaugeValue,
|
||||
1,
|
||||
instance.Name,
|
||||
string(instance.Status),
|
||||
string(instance.RunnerStatus),
|
||||
poolNames[instance.PoolID].Name,
|
||||
poolNames[instance.PoolID].Type,
|
||||
instance.PoolID,
|
||||
hostname,
|
||||
controllerID,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("cannot collect metrics, creating metric: %s", err)
|
||||
continue
|
||||
}
|
||||
ch <- m
|
||||
}
|
||||
}
|
||||
|
|
@ -23,24 +23,28 @@ import (
|
|||
|
||||
"garm/apiserver/controllers"
|
||||
"garm/auth"
|
||||
"garm/config"
|
||||
"garm/util"
|
||||
)
|
||||
|
||||
func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, cfg *config.Config, authMiddleware, initMiddleware, instanceMiddleware, metricsMiddlerware auth.Middleware) *mux.Router {
|
||||
func WithMetricsRouter(parentRouter *mux.Router, disableAuth bool, metricsMiddlerware auth.Middleware) *mux.Router {
|
||||
if parentRouter == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
metricsRouter := parentRouter.PathPrefix("/metrics").Subrouter()
|
||||
if !disableAuth {
|
||||
metricsRouter.Use(metricsMiddlerware.Middleware)
|
||||
}
|
||||
metricsRouter.Handle("/", promhttp.Handler()).Methods("GET", "OPTIONS")
|
||||
metricsRouter.Handle("", promhttp.Handler()).Methods("GET", "OPTIONS")
|
||||
return parentRouter
|
||||
}
|
||||
|
||||
func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddleware, initMiddleware, instanceMiddleware auth.Middleware) *mux.Router {
|
||||
router := mux.NewRouter()
|
||||
logMiddleware := util.NewLoggingMiddleware(logWriter)
|
||||
router.Use(logMiddleware)
|
||||
|
||||
if cfg.Metrics.Enable {
|
||||
metricsRouter := router.PathPrefix("/metrics").Subrouter()
|
||||
if !cfg.Metrics.DisableAuth {
|
||||
metricsRouter.Use(metricsMiddlerware.Middleware)
|
||||
}
|
||||
metricsRouter.Handle("/", promhttp.Handler()).Methods("GET", "OPTIONS")
|
||||
metricsRouter.Handle("", promhttp.Handler()).Methods("GET", "OPTIONS")
|
||||
}
|
||||
|
||||
// Handles github webhooks
|
||||
webhookRouter := router.PathPrefix("/webhooks").Subrouter()
|
||||
webhookRouter.PathPrefix("/").Handler(http.HandlerFunc(han.CatchAll))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue