DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

fix: improve metrics collection loop

Browse source 
by adding the context from main and make auth.GetAdminContext accepting a
context we are now able to stop the metrics collection loop once the
context is canceled

Signed-off-by: Mario Constanti <mario.constanti@mercedes-benz.com>
This commit is contained in:
Mario Constanti 2024-02-20 06:33:21 +01:00
parent 1d8d9459eb
commit 97f172eb51
8 changed files with 56 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

2
apiserver/controllers/controllers.go
View file

@ -37,7 +37,7 @@ import (
)
func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *wsWriter.Hub) (*APIController, error) {
controllerInfo, err := r.GetControllerInfo(auth.GetAdminContext())
controllerInfo, err := r.GetControllerInfo(auth.GetAdminContext(nil))
if err != nil {
return nil, errors.Wrap(err, "failed to get controller info")
}

6
auth/context.go
View file

@ -238,8 +238,10 @@ func UserID(ctx context.Context) string {
// GetAdminContext will return an admin context. This can be used internally
// when fetching users.
func GetAdminContext() context.Context {
ctx := context.Background()
func GetAdminContext(ctx context.Context) context.Context {
if ctx == nil {
ctx = context.Background()
}
ctx = SetUserID(ctx, "")
ctx = SetAdmin(ctx, true)
ctx = SetIsEnabled(ctx, true)

2
cmd/garm/main.go
View file

@ -220,7 +220,7 @@ func main() {
router = routers.WithMetricsRouter(router, cfg.Metrics.DisableAuth, metricsMiddleware)
slog.InfoContext(ctx, "start metrics collection")
runnerMetrics.CollectObjectMetric(runner, time.NewTicker(cfg.Metrics.Period))
runnerMetrics.CollectObjectMetric(ctx, runner, time.NewTicker(cfg.Metrics.Period))
}
if cfg.Default.DebugServer {

2
runner/enterprises_test.go
View file

@ -60,7 +60,7 @@ type EnterpriseTestSuite struct {
}
func (s *EnterpriseTestSuite) SetupTest() {
adminCtx := auth.GetAdminContext()
adminCtx := auth.GetAdminContext(nil)
// create testing sqlite database
dbCfg := garmTesting.GetTestSqliteDBConfig(s.T())

86
runner/metrics/metrics.go
View file

@ -1,6 +1,7 @@
package metrics
import (
"context"
"log/slog"
"time"
@ -8,9 +9,8 @@ import (
"github.com/cloudbase/garm/runner"
)
func CollectObjectMetric(r *runner.Runner, timer *time.Ticker) {
ctx := auth.GetAdminContext()
func CollectObjectMetric(ctx context.Context, r *runner.Runner, ticker *time.Ticker) {
ctx = auth.GetAdminContext(ctx)
controllerInfo, err := r.GetControllerInfo(ctx)
if err != nil {
@ -19,51 +19,55 @@ func CollectObjectMetric(r *runner.Runner, timer *time.Ticker) {
go func() {
// we wan't to initiate the collection immediately
for ; true; <-timer.C {
for ; true; <-ticker.C {
select {
case <-ctx.Done():
return
default:
slog.InfoContext(ctx, "collecting metrics")
slog.InfoContext(ctx, "collecting metrics")
var err error
slog.DebugContext(ctx, "collecting organization metrics")
err = CollectOrganizationMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect organization metrics")
}
var err error
slog.DebugContext(ctx, "collecting organization metrics")
err = CollectOrganizationMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect organization metrics")
}
slog.DebugContext(ctx, "collecting enterprise metrics")
err = CollectEnterpriseMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect enterprise metrics")
}
slog.DebugContext(ctx, "collecting enterprise metrics")
err = CollectEnterpriseMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect enterprise metrics")
}
slog.DebugContext(ctx, "collecting repository metrics")
err = CollectRepositoryMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect repository metrics")
}
slog.DebugContext(ctx, "collecting repository metrics")
err = CollectRepositoryMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect repository metrics")
}
slog.DebugContext(ctx, "collecting provider metrics")
err = CollectProviderMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect provider metrics")
}
slog.DebugContext(ctx, "collecting provider metrics")
err = CollectProviderMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect provider metrics")
}
slog.DebugContext(ctx, "collecting pool metrics")
err = CollectPoolMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect pool metrics")
}
slog.DebugContext(ctx, "collecting pool metrics")
err = CollectPoolMetric(ctx, r)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect pool metrics")
}
slog.DebugContext(ctx, "collecting health metrics")
err = CollectHealthMetric(ctx, r, controllerInfo)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect health metrics")
}
slog.DebugContext(ctx, "collecting health metrics")
err = CollectHealthMetric(ctx, r, controllerInfo)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect health metrics")
}
slog.DebugContext(ctx, "collecting instance metrics")
err = CollectInstanceMetric(ctx, r, controllerInfo)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect instance metrics")
slog.DebugContext(ctx, "collecting instance metrics")
err = CollectInstanceMetric(ctx, r, controllerInfo)
if err != nil {
slog.With(slog.Any("error", err)).ErrorContext(ctx, "cannot collect instance metrics")
}
}
}
}()

2
runner/organizations_test.go
View file

@ -60,7 +60,7 @@ type OrgTestSuite struct {
}
func (s *OrgTestSuite) SetupTest() {
adminCtx := auth.GetAdminContext()
adminCtx := auth.GetAdminContext(nil)
// create testing sqlite database
dbCfg := garmTesting.GetTestSqliteDBConfig(s.T())

2
runner/pools_test.go
View file

@ -47,7 +47,7 @@ type PoolTestSuite struct {
}
func (s *PoolTestSuite) SetupTest() {
adminCtx := auth.GetAdminContext()
adminCtx := auth.GetAdminContext(nil)
// create testing sqlite database
dbCfg := garmTesting.GetTestSqliteDBConfig(s.T())

4
runner/repositories_test.go
View file

@ -59,7 +59,7 @@ type RepoTestSuite struct {
}
func (s *RepoTestSuite) SetupTest() {
adminCtx := auth.GetAdminContext()
adminCtx := auth.GetAdminContext(nil)
// create testing sqlite database
dbCfg := garmTesting.GetTestSqliteDBConfig(s.T())
@ -90,7 +90,7 @@ func (s *RepoTestSuite) SetupTest() {
var minIdleRunners uint = 20
providerMock := runnerCommonMocks.NewProvider(s.T())
fixtures := &RepoTestFixtures{
AdminContext: auth.GetAdminContext(),
AdminContext: auth.GetAdminContext(nil),
Store: db,
StoreRepos: repos,
Providers: map[string]common.Provider{