DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

fix: stop metrics collector ticker on ctx.Done

Browse source 
Signed-off-by: Mario Constanti <mario.constanti@mercedes-benz.com>
This commit is contained in:
Mario Constanti 2024-02-20 14:57:32 +01:00
parent 17d74dfbf0
commit 0a53b8f6d8
2 changed files with 62 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/garm/main.go
View file

@ -226,7 +226,7 @@ func main() {
}
slog.InfoContext(ctx, "start metrics collection")
runnerMetrics.CollectObjectMetric(ctx, runner, time.NewTicker(cfg.Metrics.Duration()))
runnerMetrics.CollectObjectMetric(ctx, runner, cfg.Metrics.Duration())
}
if cfg.Default.DebugServer {

106
runner/metrics/metrics.go
View file

@ -6,69 +6,85 @@ import (
"time"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner"
)
func CollectObjectMetric(ctx context.Context, r *runner.Runner, ticker *time.Ticker) {
func CollectObjectMetric(ctx context.Context, r *runner.Runner, duration time.Duration) {
ctx = auth.GetAdminContext(ctx)
// get controller info for health metrics
controllerInfo, err := r.GetControllerInfo(ctx)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot get controller info")
}
// we do not want to wait until the first ticker happens
// for that we start an initial collection immediately
slog.InfoContext(ctx, "collecting metrics")
if err := collectMetrics(ctx, r, controllerInfo); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect metrics")
}
go func() {
// we wan't to initiate the collection immediately
for ; true; <-ticker.C {
ticker := time.NewTicker(duration)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
default:
case <-ticker.C:
slog.InfoContext(ctx, "collecting metrics")
var err error
slog.DebugContext(ctx, "collecting organization metrics")
err = CollectOrganizationMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect organization metrics")
}
slog.DebugContext(ctx, "collecting enterprise metrics")
err = CollectEnterpriseMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect enterprise metrics")
}
slog.DebugContext(ctx, "collecting repository metrics")
err = CollectRepositoryMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect repository metrics")
}
slog.DebugContext(ctx, "collecting provider metrics")
err = CollectProviderMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect provider metrics")
}
slog.DebugContext(ctx, "collecting pool metrics")
err = CollectPoolMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect pool metrics")
}
slog.DebugContext(ctx, "collecting instance metrics")
err = CollectInstanceMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect instance metrics")
}
slog.DebugContext(ctx, "collecting health metrics")
err = CollectHealthMetric(ctx, r, controllerInfo)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect health metrics")
if err := collectMetrics(ctx, r, controllerInfo); err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect metrics")
}
}
}
}()
}
func collectMetrics(ctx context.Context, r *runner.Runner, controllerInfo params.ControllerInfo) error {
slog.DebugContext(ctx, "collecting organization metrics")
err := CollectOrganizationMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting enterprise metrics")
err = CollectEnterpriseMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting repository metrics")
err = CollectRepositoryMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting provider metrics")
err = CollectProviderMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting pool metrics")
err = CollectPoolMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting instance metrics")
err = CollectInstanceMetric(ctx, r)
if err != nil {
return err
}
slog.DebugContext(ctx, "collecting health metrics")
err = CollectHealthMetric(ctx, r, controllerInfo)
if err != nil {
return err
}
return nil
}