DevFW-CICD/garm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add pool balancing strategy

Browse source 
This change adds the ability to specify the pool balancing strategy to
use when processing queued jobs. Before this change, GARM would round-robin
through all pools that matched the set of tags requested by queued jobs.

When round-robin (default) is used for an entity (repo, org or enterprise)
and you have 2 pools defined for that entity with a common set of tags that
match 10 jobs (for example), then those jobs would trigger the creation of
a new runner in each of the two pools in turn. Job 1 would go to pool 1,
job 2 would go to pool 2, job 3 to pool 1, job 4 to pool 2 and so on.

When "stack" is used, those same 10 jobs would trigger the creation of a
new runner in the pool with the highest priority, every time.

In both cases, if a pool is full, the next one would be tried automatically.

For the stack case, this would mean that if pool 2 had a priority of 10 and
pool 1 would have a priority of 5, pool 2 would be saturated first, then
pool 1.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2024-03-14 20:04:34 +00:00
parent 569d512819
commit ce3c917ae5
34 changed files with 362 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

14
database/sql/enterprise.go
View file

@ -13,7 +13,7 @@ import (
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) CreateEnterprise(_ context.Context, name, credentialsName, webhookSecret string) (params.Enterprise, error) {
func (s *sqlDatabase) CreateEnterprise(_ context.Context, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Enterprise, error) {
if webhookSecret == "" {
return params.Enterprise{}, errors.New("creating enterprise: missing secret")
}
@ -22,9 +22,10 @@ func (s *sqlDatabase) CreateEnterprise(_ context.Context, name, credentialsName,
return params.Enterprise{}, errors.Wrap(err, "encoding secret")
}
newEnterprise := Enterprise{
Name: name,
WebhookSecret: secret,
CredentialsName: credentialsName,
Name: name,
WebhookSecret: secret,
CredentialsName: credentialsName,
PoolBalancerType: poolBalancerType,
}
q := s.conn.Create(&newEnterprise)
@ -117,6 +118,10 @@ func (s *sqlDatabase) UpdateEnterprise(ctx context.Context, enterpriseID string,
enterprise.WebhookSecret = secret
}
if param.PoolBalancerType != "" {
enterprise.PoolBalancerType = param.PoolBalancerType
}
q := s.conn.Save(&enterprise)
if q.Error != nil {
return params.Enterprise{}, errors.Wrap(q.Error, "saving enterprise")
@ -152,6 +157,7 @@ func (s *sqlDatabase) CreateEnterprisePool(ctx context.Context, enterpriseID str
Enabled: param.Enabled,
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
GitHubRunnerGroup: param.GitHubRunnerGroup,
Priority: param.Priority,
}
if len(param.ExtraSpecs) > 0 {

10
database/sql/enterprise_test.go
View file

@ -85,6 +85,7 @@ func (s *EnterpriseTestSuite) SetupTest() {
fmt.Sprintf("test-enterprise-%d", i),
fmt.Sprintf("test-creds-%d", i),
fmt.Sprintf("test-webhook-secret-%d", i),
params.PoolBalancerTypeRoundRobin,
)
if err != nil {
s.FailNow(fmt.Sprintf("failed to create database object (test-enterprise-%d)", i))
@ -162,7 +163,8 @@ func (s *EnterpriseTestSuite) TestCreateEnterprise() {
context.Background(),
s.Fixtures.CreateEnterpriseParams.Name,
s.Fixtures.CreateEnterpriseParams.CredentialsName,
s.Fixtures.CreateEnterpriseParams.WebhookSecret)
s.Fixtures.CreateEnterpriseParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
// assertions
s.Require().Nil(err)
@ -192,7 +194,8 @@ func (s *EnterpriseTestSuite) TestCreateEnterpriseInvalidDBPassphrase() {
context.Background(),
s.Fixtures.CreateEnterpriseParams.Name,
s.Fixtures.CreateEnterpriseParams.CredentialsName,
s.Fixtures.CreateEnterpriseParams.WebhookSecret)
s.Fixtures.CreateEnterpriseParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
s.Require().NotNil(err)
s.Require().Equal("encoding secret: invalid passphrase length (expected length 32 characters)", err.Error())
@ -209,7 +212,8 @@ func (s *EnterpriseTestSuite) TestCreateEnterpriseDBCreateErr() {
context.Background(),
s.Fixtures.CreateEnterpriseParams.Name,
s.Fixtures.CreateEnterpriseParams.CredentialsName,
s.Fixtures.CreateEnterpriseParams.WebhookSecret)
s.Fixtures.CreateEnterpriseParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
s.assertSQLMockExpectations()
s.Require().NotNil(err)

2
database/sql/instances_test.go
View file

@ -77,7 +77,7 @@ func (s *InstancesTestSuite) SetupTest() {
s.Store = db
// create an organization for testing purposes
org, err := s.Store.CreateOrganization(context.Background(), "test-org", "test-creds", "test-webhookSecret")
org, err := s.Store.CreateOrganization(context.Background(), "test-org", "test-creds", "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
if err != nil {
s.FailNow(fmt.Sprintf("failed to create org: %s", err))
}

36
database/sql/models.go
View file

@ -83,37 +83,41 @@ type Pool struct {
Enterprise Enterprise `gorm:"foreignKey:EnterpriseID"`
Instances []Instance `gorm:"foreignKey:PoolID"`
Priority uint `gorm:"index:idx_pool_priority"`
}
type Repository struct {
Base
CredentialsName string
Owner string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
Name string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:RepoID"`
Jobs []WorkflowJob `gorm:"foreignKey:RepoID;constraint:OnDelete:SET NULL"`
CredentialsName string
Owner string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
Name string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:RepoID"`
Jobs []WorkflowJob `gorm:"foreignKey:RepoID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
}
type Organization struct {
Base
CredentialsName string
Name string `gorm:"index:idx_org_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:OrgID"`
Jobs []WorkflowJob `gorm:"foreignKey:OrgID;constraint:OnDelete:SET NULL"`
CredentialsName string
Name string `gorm:"index:idx_org_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:OrgID"`
Jobs []WorkflowJob `gorm:"foreignKey:OrgID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
}
type Enterprise struct {
Base
CredentialsName string
Name string `gorm:"index:idx_ent_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:EnterpriseID"`
Jobs []WorkflowJob `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:SET NULL"`
CredentialsName string
Name string `gorm:"index:idx_ent_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:EnterpriseID"`
Jobs []WorkflowJob `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
}
type Address struct {

14
database/sql/organizations.go
View file

@ -28,7 +28,7 @@ import (
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) CreateOrganization(_ context.Context, name, credentialsName, webhookSecret string) (params.Organization, error) {
func (s *sqlDatabase) CreateOrganization(_ context.Context, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Organization, error) {
if webhookSecret == "" {
return params.Organization{}, errors.New("creating org: missing secret")
}
@ -37,9 +37,10 @@ func (s *sqlDatabase) CreateOrganization(_ context.Context, name, credentialsNam
return params.Organization{}, fmt.Errorf("failed to encrypt string")
}
newOrg := Organization{
Name: name,
WebhookSecret: secret,
CredentialsName: credentialsName,
Name: name,
WebhookSecret: secret,
CredentialsName: credentialsName,
PoolBalancerType: poolBalancerType,
}
q := s.conn.Create(&newOrg)
@ -121,6 +122,10 @@ func (s *sqlDatabase) UpdateOrganization(ctx context.Context, orgID string, para
org.WebhookSecret = secret
}
if param.PoolBalancerType != "" {
org.PoolBalancerType = param.PoolBalancerType
}
q := s.conn.Save(&org)
if q.Error != nil {
return params.Organization{}, errors.Wrap(q.Error, "saving org")
@ -169,6 +174,7 @@ func (s *sqlDatabase) CreateOrganizationPool(ctx context.Context, orgID string,
Enabled: param.Enabled,
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
GitHubRunnerGroup: param.GitHubRunnerGroup,
Priority: param.Priority,
}
if len(param.ExtraSpecs) > 0 {

10
database/sql/organizations_test.go
View file

@ -85,6 +85,7 @@ func (s *OrgTestSuite) SetupTest() {
fmt.Sprintf("test-org-%d", i),
fmt.Sprintf("test-creds-%d", i),
fmt.Sprintf("test-webhook-secret-%d", i),
params.PoolBalancerTypeRoundRobin,
)
if err != nil {
s.FailNow(fmt.Sprintf("failed to create database object (test-org-%d)", i))
@ -162,7 +163,8 @@ func (s *OrgTestSuite) TestCreateOrganization() {
context.Background(),
s.Fixtures.CreateOrgParams.Name,
s.Fixtures.CreateOrgParams.CredentialsName,
s.Fixtures.CreateOrgParams.WebhookSecret)
s.Fixtures.CreateOrgParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
// assertions
s.Require().Nil(err)
@ -192,7 +194,8 @@ func (s *OrgTestSuite) TestCreateOrganizationInvalidDBPassphrase() {
context.Background(),
s.Fixtures.CreateOrgParams.Name,
s.Fixtures.CreateOrgParams.CredentialsName,
s.Fixtures.CreateOrgParams.WebhookSecret)
s.Fixtures.CreateOrgParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
s.Require().NotNil(err)
s.Require().Equal("failed to encrypt string", err.Error())
@ -209,7 +212,8 @@ func (s *OrgTestSuite) TestCreateOrganizationDBCreateErr() {
context.Background(),
s.Fixtures.CreateOrgParams.Name,
s.Fixtures.CreateOrgParams.CredentialsName,
s.Fixtures.CreateOrgParams.WebhookSecret)
s.Fixtures.CreateOrgParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin)
s.assertSQLMockExpectations()
s.Require().NotNil(err)

4
database/sql/pools.go
View file

@ -190,7 +190,9 @@ func (s *sqlDatabase) findPoolByTags(id string, poolType params.PoolType, tags [
Group("pools.id").
Preload("Tags").
Having("count(1) = ?", len(tags)).
Where(where, tags, u).Find(&pools)
Where(where, tags, u).
Order("priority desc").
Find(&pools)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {

4
database/sql/pools_test.go
View file

@ -61,7 +61,7 @@ func (s *PoolsTestSuite) SetupTest() {
s.Store = db
// create an organization for testing purposes
org, err := s.Store.CreateOrganization(context.Background(), "test-org", "test-creds", "test-webhookSecret")
org, err := s.Store.CreateOrganization(context.Background(), "test-org", "test-creds", "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
if err != nil {
s.FailNow(fmt.Sprintf("failed to create org: %s", err))
}
@ -128,7 +128,7 @@ func (s *PoolsTestSuite) TestListAllPools() {
func (s *PoolsTestSuite) TestListAllPoolsDBFetchErr() {
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id`,`pools`.`priority` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
WillReturnError(fmt.Errorf("mocked fetching all pools error"))
_, err := s.StoreSQLMocked.ListAllPools(context.Background())

16
database/sql/repositories.go
View file

@ -28,7 +28,7 @@ import (
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) CreateRepository(_ context.Context, owner, name, credentialsName, webhookSecret string) (params.Repository, error) {
func (s *sqlDatabase) CreateRepository(_ context.Context, owner, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Repository, error) {
if webhookSecret == "" {
return params.Repository{}, errors.New("creating repo: missing secret")
}
@ -37,10 +37,11 @@ func (s *sqlDatabase) CreateRepository(_ context.Context, owner, name, credentia
return params.Repository{}, fmt.Errorf("failed to encrypt string")
}
newRepo := Repository{
Name: name,
Owner: owner,
WebhookSecret: secret,
CredentialsName: credentialsName,
Name: name,
Owner: owner,
WebhookSecret: secret,
CredentialsName: credentialsName,
PoolBalancerType: poolBalancerType,
}
q := s.conn.Create(&newRepo)
@ -121,6 +122,10 @@ func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param
repo.WebhookSecret = secret
}
if param.PoolBalancerType != "" {
repo.PoolBalancerType = param.PoolBalancerType
}
q := s.conn.Save(&repo)
if q.Error != nil {
return params.Repository{}, errors.Wrap(q.Error, "saving repo")
@ -169,6 +174,7 @@ func (s *sqlDatabase) CreateRepositoryPool(ctx context.Context, repoID string, p
Enabled: param.Enabled,
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
GitHubRunnerGroup: param.GitHubRunnerGroup,
Priority: param.Priority,
}
if len(param.ExtraSpecs) > 0 {

4
database/sql/repositories_test.go
View file

@ -96,6 +96,7 @@ func (s *RepoTestSuite) SetupTest() {
fmt.Sprintf("test-repo-%d", i),
fmt.Sprintf("test-creds-%d", i),
fmt.Sprintf("test-webhook-secret-%d", i),
params.PoolBalancerTypeRoundRobin,
)
if err != nil {
s.FailNow(fmt.Sprintf("failed to create database object (test-repo-%d): %v", i, err))
@ -176,6 +177,7 @@ func (s *RepoTestSuite) TestCreateRepository() {
s.Fixtures.CreateRepoParams.Name,
s.Fixtures.CreateRepoParams.CredentialsName,
s.Fixtures.CreateRepoParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin,
)
// assertions
@ -209,6 +211,7 @@ func (s *RepoTestSuite) TestCreateRepositoryInvalidDBPassphrase() {
s.Fixtures.CreateRepoParams.Name,
s.Fixtures.CreateRepoParams.CredentialsName,
s.Fixtures.CreateRepoParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin,
)
s.Require().NotNil(err)
@ -228,6 +231,7 @@ func (s *RepoTestSuite) TestCreateRepositoryInvalidDBCreateErr() {
s.Fixtures.CreateRepoParams.Name,
s.Fixtures.CreateRepoParams.CredentialsName,
s.Fixtures.CreateRepoParams.WebhookSecret,
params.PoolBalancerTypeRoundRobin,
)
s.assertSQLMockExpectations()

52
database/sql/util.go
View file

@ -113,11 +113,16 @@ func (s *sqlDatabase) sqlToCommonOrganization(org Organization) (params.Organiza
}
ret := params.Organization{
ID: org.ID.String(),
Name: org.Name,
CredentialsName: org.CredentialsName,
Pools: make([]params.Pool, len(org.Pools)),
WebhookSecret: string(secret),
ID: org.ID.String(),
Name: org.Name,
CredentialsName: org.CredentialsName,
Pools: make([]params.Pool, len(org.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: org.PoolBalancerType,
}
if ret.PoolBalancerType == "" {
ret.PoolBalancerType = params.PoolBalancerTypeRoundRobin
}
for idx, pool := range org.Pools {
@ -140,11 +145,16 @@ func (s *sqlDatabase) sqlToCommonEnterprise(enterprise Enterprise) (params.Enter
}
ret := params.Enterprise{
ID: enterprise.ID.String(),
Name: enterprise.Name,
CredentialsName: enterprise.CredentialsName,
Pools: make([]params.Pool, len(enterprise.Pools)),
WebhookSecret: string(secret),
ID: enterprise.ID.String(),
Name: enterprise.Name,
CredentialsName: enterprise.CredentialsName,
Pools: make([]params.Pool, len(enterprise.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: enterprise.PoolBalancerType,
}
if ret.PoolBalancerType == "" {
ret.PoolBalancerType = params.PoolBalancerTypeRoundRobin
}
for idx, pool := range enterprise.Pools {
@ -176,6 +186,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) (params.Pool, error) {
RunnerBootstrapTimeout: pool.RunnerBootstrapTimeout,
ExtraSpecs: json.RawMessage(pool.ExtraSpecs),
GitHubRunnerGroup: pool.GitHubRunnerGroup,
Priority: pool.Priority,
}
if pool.RepoID != nil {
@ -227,12 +238,17 @@ func (s *sqlDatabase) sqlToCommonRepository(repo Repository) (params.Repository,
}
ret := params.Repository{
ID: repo.ID.String(),
Name: repo.Name,
Owner: repo.Owner,
CredentialsName: repo.CredentialsName,
Pools: make([]params.Pool, len(repo.Pools)),
WebhookSecret: string(secret),
ID: repo.ID.String(),
Name: repo.Name,
Owner: repo.Owner,
CredentialsName: repo.CredentialsName,
Pools: make([]params.Pool, len(repo.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: repo.PoolBalancerType,
}
if ret.PoolBalancerType == "" {
ret.PoolBalancerType = params.PoolBalancerTypeRoundRobin
}
for idx, pool := range repo.Pools {
@ -324,6 +340,10 @@ func (s *sqlDatabase) updatePool(pool Pool, param params.UpdatePoolParams) (para
pool.GitHubRunnerGroup = *param.GitHubRunnerGroup
}
if param.Priority != nil {
pool.Priority = *param.Priority
}
if q := s.conn.Save(&pool); q.Error != nil {
return params.Pool{}, errors.Wrap(q.Error, "saving database entry")
}