DevFW-CICD/garm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add runner rotate ability to CLI

Browse source 
This change adds a new "generation" field to pools, scalesets and
runners. The generation field is inherited by runners from scale sets
or pools at the time of creation.

The generation field on scalesets and pools is incremented when the
pool or scale set is updated in a way that might influence how runners
are created (flavor, image, specs, runner groups, etc).

Using this new field, we can determine if existing runners have diverged
from the settings of the pool/scale set that spawned them.

In the CLI we now have a new set of commands available for both
pools and scalesets that lists runners, with an optional --outdated
flag and a new "rotate" flag that removes all idle runners. Optionally
the --outdated flag can be passed to the rotate command to only remove
outdated runners.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2026-02-08 23:48:57 +02:00 • committed by Gabriel
parent 61b4b4cadd
commit 80e042ee88
27 changed files with 648 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

11
database/sql/instances.go
View file

@ -90,6 +90,7 @@ func (s *sqlDatabase) CreateInstance(ctx context.Context, poolID string, param p
JitConfiguration: secret,
AditionalLabels: labels,
AgentID: param.AgentID,
Generation: param.Generation,
}
q = tx.Create(&newInstance)
if q.Error != nil {
@ -511,14 +512,18 @@ func (s *sqlDatabase) listInstancesBatched(queryModifier func(*gorm.DB) *gorm.DB
return ret, err
}
func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string) ([]params.Instance, error) {
func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string, outdatedOnly bool) ([]params.Instance, error) {
u, err := uuid.Parse(poolID)
if err != nil {
return nil, fmt.Errorf("error parsing id: %w", runnerErrors.ErrBadRequest)
}
ret, err := s.listInstancesBatched(func(query *gorm.DB) *gorm.DB {
return query.Where("pool_id = ?", u)
q := query.Where("pool_id = ?", u)
if outdatedOnly {
q = q.Where("instances.generation < (SELECT pools.generation FROM pools WHERE pools.id = instances.pool_id)")
}
return q
})
if err != nil {
return nil, fmt.Errorf("failed to list pool instances: %w", err)
@ -527,7 +532,7 @@ func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string) ([]par
}
func (s *sqlDatabase) ListAllInstances(_ context.Context) ([]params.Instance, error) {
ret, err := s.listInstancesBatched(nil) // No query modifier for all instances
ret, err := s.listInstancesBatched(nil)
if err != nil {
return nil, fmt.Errorf("failed to list all instances: %w", err)
}

4
database/sql/instances_test.go
View file

@ -668,14 +668,14 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateAddressErr() {
}
func (s *InstancesTestSuite) TestListPoolInstances() {
instances, err := s.Store.ListPoolInstances(s.adminCtx, s.Fixtures.Pool.ID)
instances, err := s.Store.ListPoolInstances(s.adminCtx, s.Fixtures.Pool.ID, false)
s.Require().Nil(err)
s.equalInstancesByName(s.Fixtures.Instances, instances)
}
func (s *InstancesTestSuite) TestListPoolInstancesInvalidPoolID() {
_, err := s.Store.ListPoolInstances(s.adminCtx, "dummy-pool-id")
_, err := s.Store.ListPoolInstances(s.adminCtx, "dummy-pool-id", false)
s.Require().Equal("error parsing id: invalid request", err.Error())
}

21
database/sql/models.go
View file

@ -121,6 +121,14 @@ type Pool struct {
GitHubRunnerGroup string
EnableShell bool
// Generation holds the numeric generation of the pool. This number
// will be incremented, every time certain settings of the pool, which
// may influence how runners are created (flavor, specs, image) are changed.
// When a runner is created, this generation will be copied to the runners as
// well. That way if some settings diverge, we can target those runners
// to be recreated.
Generation uint64
RepoID *uuid.UUID `gorm:"index"`
Repository Repository `gorm:"foreignKey:RepoID;"`
@ -178,6 +186,13 @@ type ScaleSet struct {
ExtraSpecs datatypes.JSON
EnableShell bool
// Generation is the scaleset generation at the time of creating this instance.
// This field is to track a divergence between when the instance was created
// and the settings currently set on a scaleset. We can then use this field to know
// if the instance is out of date with the scaleset, allowing us to remove it if we
// need to.
Generation uint64
RepoID *uuid.UUID `gorm:"index"`
Repository Repository `gorm:"foreignKey:RepoID;"`
@ -336,6 +351,12 @@ type Instance struct {
GitHubRunnerGroup string
AditionalLabels datatypes.JSON
Capabilities datatypes.JSON
// Generation is the pool generation at the time of creating this instance.
// This field is to track a divergence between when the instance was created
// and the settings currently set on a pool. We can then use this field to know
// if the instance is out of date with the pool, allowing us to remove it if we
// need to.
Generation uint64
PoolID *uuid.UUID
Pool Pool `gorm:"foreignKey:PoolID"`

2
database/sql/pools_test.go
View file

@ -150,7 +150,7 @@ func (s *PoolsTestSuite) TestListAllPools() {
func (s *PoolsTestSuite) TestListAllPoolsDBFetchErr() {
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`enable_shell`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id`,`pools`.`template_id`,`pools`.`priority` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`enable_shell`,`pools`.`generation`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id`,`pools`.`template_id`,`pools`.`priority` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
WillReturnError(fmt.Errorf("mocked fetching all pools error"))
_, err := s.StoreSQLMocked.ListAllPools(s.adminCtx)

8
database/sql/scaleset_instances.go
View file

@ -65,9 +65,13 @@ func (s *sqlDatabase) CreateScaleSetInstance(_ context.Context, scaleSetID uint,
return s.sqlToParamsInstance(newInstance)
}
func (s *sqlDatabase) ListScaleSetInstances(_ context.Context, scalesetID uint) ([]params.Instance, error) {
func (s *sqlDatabase) ListScaleSetInstances(_ context.Context, scalesetID uint, outdatedOnly bool) ([]params.Instance, error) {
ret, err := s.listInstancesBatched(func(query *gorm.DB) *gorm.DB {
return query.Where("scale_set_fk_id = ?", scalesetID)
q := query.Where("scale_set_fk_id = ?", scalesetID)
if outdatedOnly {
q = q.Where("instances.generation < (SELECT scale_sets.generation FROM scale_sets WHERE scale_sets.id = instances.scale_set_fk_id)")
}
return q
})
if err != nil {
return nil, fmt.Errorf("failed to list scaleset instances: %w", err)

12
database/sql/scalesets.go
View file

@ -284,6 +284,7 @@ func (s *sqlDatabase) getEntityScaleSet(tx *gorm.DB, entityType params.ForgeEnti
}
func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param params.UpdateScaleSetParams) (params.ScaleSet, error) {
incrementGeneration := false
if param.Enabled != nil && scaleSet.Enabled != *param.Enabled {
scaleSet.Enabled = *param.Enabled
}
@ -306,6 +307,7 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param
if param.EnableShell != nil {
scaleSet.EnableShell = *param.EnableShell
incrementGeneration = true
}
if param.Name != "" {
@ -314,14 +316,17 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param
if param.GitHubRunnerGroup != nil && *param.GitHubRunnerGroup != "" {
scaleSet.GitHubRunnerGroup = *param.GitHubRunnerGroup
incrementGeneration = true
}
if param.Flavor != "" {
scaleSet.Flavor = param.Flavor
incrementGeneration = true
}
if param.Image != "" {
scaleSet.Image = param.Image
incrementGeneration = true
}
if param.Prefix != "" {
@ -338,22 +343,25 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param
if param.OSArch != "" {
scaleSet.OSArch = param.OSArch
incrementGeneration = true
}
if param.OSType != "" {
scaleSet.OSType = param.OSType
incrementGeneration = true
}
if param.ExtraSpecs != nil {
scaleSet.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
incrementGeneration = true
}
if param.RunnerBootstrapTimeout != nil && *param.RunnerBootstrapTimeout > 0 {
scaleSet.RunnerBootstrapTimeout = *param.RunnerBootstrapTimeout
}
if param.GitHubRunnerGroup != nil {
scaleSet.GitHubRunnerGroup = *param.GitHubRunnerGroup
if incrementGeneration {
scaleSet.Generation++
}
if q := tx.Save(&scaleSet); q.Error != nil {

2
database/sql/scalesets_test.go
View file

@ -356,7 +356,7 @@ func (s *ScaleSetsTestSuite) TestScaleSetOperations() {
})
s.T().Run("List repo scale set instances", func(_ *testing.T) {
instances, err := s.Store.ListScaleSetInstances(s.adminCtx, repoScaleSet.ID)
instances, err := s.Store.ListScaleSetInstances(s.adminCtx, repoScaleSet.ID, false)
s.Require().NoError(err)
s.Require().NotEmpty(instances)
s.Require().Len(instances, 1)

15
database/sql/util.go
View file

@ -75,6 +75,7 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) (params.Instance, e
GitHubRunnerGroup: instance.GitHubRunnerGroup,
AditionalLabels: labels,
Heartbeat: instance.Heartbeat,
Generation: instance.Generation,
}
if len(instance.Capabilities) > 0 {
@ -299,6 +300,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) (params.Pool, error) {
CreatedAt: pool.CreatedAt,
UpdatedAt: pool.UpdatedAt,
EnableShell: pool.EnableShell,
Generation: pool.Generation,
}
if pool.TemplateID != nil && *pool.TemplateID != 0 {
@ -376,6 +378,7 @@ func (s *sqlDatabase) sqlToCommonScaleSet(scaleSet ScaleSet) (params.ScaleSet, e
LastMessageID: scaleSet.LastMessageID,
DesiredRunnerCount: scaleSet.DesiredRunnerCount,
EnableShell: scaleSet.EnableShell,
Generation: scaleSet.Generation,
}
if scaleSet.TemplateID != nil && *scaleSet.TemplateID != 0 {
@ -539,20 +542,24 @@ func (s *sqlDatabase) getOrCreateTag(tx *gorm.DB, tagName string) (Tag, error) {
}
func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePoolParams) (params.Pool, error) {
incrementGeneration := false
if param.Enabled != nil && pool.Enabled != *param.Enabled {
pool.Enabled = *param.Enabled
}
if param.Flavor != "" {
pool.Flavor = param.Flavor
incrementGeneration = true
}
if param.EnableShell != nil {
pool.EnableShell = *param.EnableShell
incrementGeneration = true
}
if param.Image != "" {
pool.Image = param.Image
incrementGeneration = true
}
if param.Prefix != "" {
@ -573,14 +580,17 @@ func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePool
if param.OSArch != "" {
pool.OSArch = param.OSArch
incrementGeneration = true
}
if param.OSType != "" {
pool.OSType = param.OSType
incrementGeneration = true
}
if param.ExtraSpecs != nil {
pool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
incrementGeneration = true
}
if param.RunnerBootstrapTimeout != nil && *param.RunnerBootstrapTimeout > 0 {
@ -589,12 +599,17 @@ func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePool
if param.GitHubRunnerGroup != nil {
pool.GitHubRunnerGroup = *param.GitHubRunnerGroup
incrementGeneration = true
}
if param.Priority != nil {
pool.Priority = *param.Priority
}
if incrementGeneration {
pool.Generation++
}
if q := tx.Save(&pool); q.Error != nil {
return params.Pool{}, fmt.Errorf("error saving database entry: %w", q.Error)
}