DevFW-CICD/garm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add rate limits metrics and credentials details page

Browse source 
This change adds metrics for rate limits. Rate limits are now recorded
via a rate limit check loop (as before), but in addition, we are now
taking the rate limit info that gets returned in all github responses
and we're recording that as it happens as opposed to every 30 seconds.

The loop remains to update rate limits even for credentials that are
used rarely.

This change also adds a credentials details page in the webUI.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2026-02-09 19:09:29 +02:00 • committed by Gabriel
parent 0255db1760
commit b600a21980
141 changed files with 292 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

88
util/github/client.go
View file

@ -66,6 +66,9 @@ func (g *githubClient) ListEntityHooks(ctx context.Context, opts *github.ListOpt
default:
return nil, nil, fmt.Errorf("invalid entity type: %s", g.entity.EntityType)
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, response, err
}
@ -82,14 +85,19 @@ func (g *githubClient) GetEntityHook(ctx context.Context, id int64) (ret *github
).Inc()
}
}()
var response *github.Response
switch g.entity.EntityType {
case params.ForgeEntityTypeRepository:
ret, _, err = g.repo.GetHook(ctx, g.entity.Owner, g.entity.Name, id)
ret, response, err = g.repo.GetHook(ctx, g.entity.Owner, g.entity.Name, id)
case params.ForgeEntityTypeOrganization:
ret, _, err = g.org.GetHook(ctx, g.entity.Owner, id)
ret, response, err = g.org.GetHook(ctx, g.entity.Owner, id)
default:
return nil, errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, err
}
@ -106,14 +114,18 @@ func (g *githubClient) createGithubEntityHook(ctx context.Context, hook *github.
).Inc()
}
}()
var response *github.Response
switch g.entity.EntityType {
case params.ForgeEntityTypeRepository:
ret, _, err = g.repo.CreateHook(ctx, g.entity.Owner, g.entity.Name, hook)
ret, response, err = g.repo.CreateHook(ctx, g.entity.Owner, g.entity.Name, hook)
case params.ForgeEntityTypeOrganization:
ret, _, err = g.org.CreateHook(ctx, g.entity.Owner, hook)
ret, response, err = g.org.CreateHook(ctx, g.entity.Owner, hook)
default:
return nil, errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, err
}
@ -149,6 +161,9 @@ func (g *githubClient) DeleteEntityHook(ctx context.Context, id int64) (ret *git
default:
return nil, errors.New("invalid entity type")
}
if err == nil && ret != nil {
g.recordLimits(ret.Rate)
}
return ret, err
}
@ -173,6 +188,10 @@ func (g *githubClient) PingEntityHook(ctx context.Context, id int64) (ret *githu
default:
return nil, errors.New("invalid entity type")
}
if err == nil && ret != nil {
g.recordLimits(ret.Rate)
}
return ret, err
}
@ -204,7 +223,9 @@ func (g *githubClient) ListEntityRunners(ctx context.Context, opts *github.ListR
default:
return nil, nil, errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, response, err
}
@ -236,7 +257,9 @@ func (g *githubClient) ListEntityRunnerApplicationDownloads(ctx context.Context)
default:
return nil, nil, errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, response, err
}
@ -308,6 +331,9 @@ func (g *githubClient) RemoveEntityRunner(ctx context.Context, runnerID int64) e
default:
return errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
if err := parseError(response, err); err != nil {
return fmt.Errorf("error removing runner %d: %w", runnerID, err)
@ -344,6 +370,9 @@ func (g *githubClient) CreateEntityRegistrationToken(ctx context.Context) (*gith
default:
return nil, nil, errors.New("invalid entity type")
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
return ret, response, err
}
@ -371,6 +400,10 @@ func (g *githubClient) getOrganizationRunnerGroupIDByName(ctx context.Context, e
}
return 0, fmt.Errorf("error fetching runners: %w", err)
}
if err == nil && ghResp != nil {
g.recordLimits(ghResp.Rate)
}
for _, runnerGroup := range runnerGroups.RunnerGroups {
if runnerGroup.Name != nil && *runnerGroup.Name == rgName {
return *runnerGroup.ID, nil
@ -407,6 +440,9 @@ func (g *githubClient) getEnterpriseRunnerGroupIDByName(ctx context.Context, ent
}
return 0, fmt.Errorf("error fetching runners: %w", err)
}
if err == nil && ghResp != nil {
g.recordLimits(ghResp.Rate)
}
for _, runnerGroup := range runnerGroups.RunnerGroups {
if runnerGroup.Name != nil && *runnerGroup.Name == rgName {
return *runnerGroup.ID, nil
@ -483,6 +519,9 @@ func (g *githubClient) GetEntityJITConfig(ctx context.Context, instance string,
case params.ForgeEntityTypeEnterprise:
ret, response, err = g.enterprise.GenerateEnterpriseJITConfig(ctx, g.entity.Owner, &req)
}
if err == nil && response != nil {
g.recordLimits(response.Rate)
}
if err != nil {
metrics.GithubOperationFailedCount.WithLabelValues(
"GetEntityJITConfig", // label: operation
@ -538,6 +577,35 @@ func (g *githubClient) GithubBaseURL() *url.URL {
return g.cli.BaseURL
}
func (g *githubClient) recordLimits(core github.Rate) {
limit := params.GithubRateLimit{
Limit: core.Limit,
Used: core.Used,
Remaining: core.Remaining,
Reset: core.Reset.Unix(),
}
cache.SetCredentialsRateLimit(g.entity.Credentials.ID, limit)
// Record Prometheus metrics
credID := fmt.Sprintf("%d", g.entity.Credentials.ID)
credName := g.entity.Credentials.Name
endpoint := g.entity.Credentials.Endpoint.Name
if endpoint == "" {
endpoint = g.entity.Credentials.BaseURL
}
labels := map[string]string{
"credential_name": credName,
"credential_id": credID,
"endpoint": endpoint,
}
metrics.GithubRateLimitLimit.With(labels).Set(float64(core.Limit))
metrics.GithubRateLimitRemaining.With(labels).Set(float64(core.Remaining))
metrics.GithubRateLimitUsed.With(labels).Set(float64(core.Used))
metrics.GithubRateLimitResetTimestamp.With(labels).Set(float64(core.Reset.Unix()))
}
func NewRateLimitClient(ctx context.Context, credentials params.ForgeCredentials) (common.RateLimitClient, error) {
httpClient, err := credentials.GetHTTPClient(ctx)
if err != nil {
@ -623,5 +691,13 @@ func Client(ctx context.Context, entity params.ForgeEntity) (common.GithubClient
entity: entity,
}
limits, err := cli.RateLimit(ctx)
if err == nil && limits != nil {
core := limits.GetCore()
if core != nil {
cli.recordLimits(*core)
}
}
return cli, nil
}