diff --git a/cmd/garm/main.go b/cmd/garm/main.go index dfac21a9..df7a82e8 100644 --- a/cmd/garm/main.go +++ b/cmd/garm/main.go @@ -226,7 +226,7 @@ func main() { } slog.InfoContext(ctx, "start metrics collection") - runnerMetrics.CollectObjectMetric(ctx, runner, time.NewTicker(cfg.Metrics.Duration())) + runnerMetrics.CollectObjectMetric(ctx, runner, cfg.Metrics.Duration()) } if cfg.Default.DebugServer { diff --git a/runner/metrics/metrics.go b/runner/metrics/metrics.go index 911226dc..59fc6bbb 100644 --- a/runner/metrics/metrics.go +++ b/runner/metrics/metrics.go @@ -6,69 +6,85 @@ import ( "time" "github.com/cloudbase/garm/auth" + "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner" ) -func CollectObjectMetric(ctx context.Context, r *runner.Runner, ticker *time.Ticker) { +func CollectObjectMetric(ctx context.Context, r *runner.Runner, duration time.Duration) { ctx = auth.GetAdminContext(ctx) + // get controller info for health metrics controllerInfo, err := r.GetControllerInfo(ctx) if err != nil { slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot get controller info") } + // we do not want to wait until the first ticker happens + // for that we start an initial collection immediately + slog.InfoContext(ctx, "collecting metrics") + if err := collectMetrics(ctx, r, controllerInfo); err != nil { + slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect metrics") + } + go func() { - // we wan't to initiate the collection immediately - for ; true; <-ticker.C { + ticker := time.NewTicker(duration) + defer ticker.Stop() + for { select { case <-ctx.Done(): return - default: + case <-ticker.C: slog.InfoContext(ctx, "collecting metrics") - var err error - slog.DebugContext(ctx, "collecting organization metrics") - err = CollectOrganizationMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect organization metrics") - } - - slog.DebugContext(ctx, "collecting enterprise metrics") - err = CollectEnterpriseMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect enterprise metrics") - } - - slog.DebugContext(ctx, "collecting repository metrics") - err = CollectRepositoryMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect repository metrics") - } - - slog.DebugContext(ctx, "collecting provider metrics") - err = CollectProviderMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect provider metrics") - } - - slog.DebugContext(ctx, "collecting pool metrics") - err = CollectPoolMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect pool metrics") - } - - slog.DebugContext(ctx, "collecting instance metrics") - err = CollectInstanceMetric(ctx, r) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect instance metrics") - } - - slog.DebugContext(ctx, "collecting health metrics") - err = CollectHealthMetric(ctx, r, controllerInfo) - if err != nil { - slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect health metrics") + if err := collectMetrics(ctx, r, controllerInfo); err != nil { + slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect metrics") } } } }() } + +func collectMetrics(ctx context.Context, r *runner.Runner, controllerInfo params.ControllerInfo) error { + slog.DebugContext(ctx, "collecting organization metrics") + err := CollectOrganizationMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting enterprise metrics") + err = CollectEnterpriseMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting repository metrics") + err = CollectRepositoryMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting provider metrics") + err = CollectProviderMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting pool metrics") + err = CollectPoolMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting instance metrics") + err = CollectInstanceMetric(ctx, r) + if err != nil { + return err + } + + slog.DebugContext(ctx, "collecting health metrics") + err = CollectHealthMetric(ctx, r, controllerInfo) + if err != nil { + return err + } + return nil +}