DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add extra specs on pools

Browse source 
Extra specs is an opaque valid JSON that can be set on a pool and which
will be passed along to the provider as part of instance bootstrap params.

This field is meant to allow operators to send extra configuration values
to external or built-in providers. The extra specs is not interpreted or
useful in any way to garm itself, but it may be useful to the provider
which interacts with the IaaS.

The extra specs are not meant to be used for secrets. Adding sensitive
information to this field is highly discouraged. This field is meant as a    
means to add fine tuning knobs to the providers, on a per pool basis.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2023-01-30 00:40:01 +00:00
parent 1867bbaad6
commit f25951decb
126 changed files with 8771 additions and 5232 deletions
Download patch file Download diff file Expand all files Collapse all files

34
database/sql/enterprise.go
View file

@ -9,6 +9,7 @@ import (
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
)
@ -152,6 +153,10 @@ func (s *sqlDatabase) CreateEnterprisePool(ctx context.Context, enterpriseID str
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
}
if len(param.ExtraSpecs) > 0 {
newPool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
}
_, err = s.getEnterprisePoolByUniqueFields(ctx, enterpriseID, newPool.ProviderName, newPool.Image, newPool.Flavor)
if err != nil {
if !errors.Is(err, runnerErrors.ErrNotFound) {
@ -312,7 +317,7 @@ func (s *sqlDatabase) getEnterprisePoolByUniqueFields(ctx context.Context, enter
}
func (s *sqlDatabase) getEnterprisePool(ctx context.Context, enterpriseID, poolID string, preload ...string) (Pool, error) {
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID)
_, err := s.getEnterpriseByID(ctx, enterpriseID)
if err != nil {
return Pool{}, errors.Wrap(err, "fetching enterprise")
}
@ -329,33 +334,38 @@ func (s *sqlDatabase) getEnterprisePool(ctx context.Context, enterpriseID, poolI
}
}
var pool []Pool
err = q.Model(&enterprise).Association("Pools").Find(&pool, "id = ?", u)
var pool Pool
err = q.Model(&Pool{}).
Where("id = ? and enterprise_id = ?", u, enterpriseID).
First(&pool).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return Pool{}, errors.Wrap(runnerErrors.ErrNotFound, "finding pool")
}
return Pool{}, errors.Wrap(err, "fetching pool")
}
if len(pool) == 0 {
return Pool{}, runnerErrors.ErrNotFound
}
return pool[0], nil
return pool, nil
}
func (s *sqlDatabase) getEnterprisePools(ctx context.Context, enterpriseID string, preload ...string) ([]Pool, error) {
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID)
_, err := s.getEnterpriseByID(ctx, enterpriseID)
if err != nil {
return nil, errors.Wrap(err, "fetching enterprise")
}
var pools []Pool
q := s.conn.Model(&enterprise)
q := s.conn
if len(preload) > 0 {
for _, item := range preload {
q = q.Preload(item)
}
}
err = q.Association("Pools").Find(&pools)
var pools []Pool
err = q.Model(&Pool{}).Where("enterprise_id = ?", enterpriseID).
Omit("extra_specs").
Find(&pools).Error
if err != nil {
return nil, errors.Wrap(err, "fetching pool")

6
database/sql/enterprise_test.go
View file

@ -701,7 +701,7 @@ func (s *EnterpriseTestSuite) TestDeleteEnterprisePool() {
s.Require().Nil(err)
_, err = s.Store.GetEnterprisePool(context.Background(), s.Fixtures.Enterprises[0].ID, pool.ID)
s.Require().Equal("fetching pool: not found", err.Error())
s.Require().Equal("fetching pool: finding pool: not found", err.Error())
}
func (s *EnterpriseTestSuite) TestDeleteEnterprisePoolInvalidEnterpriseID() {
@ -722,8 +722,8 @@ func (s *EnterpriseTestSuite) TestDeleteEnterprisePoolDBDeleteErr() {
WithArgs(s.Fixtures.Enterprises[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Enterprises[0].ID))
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`enterprise_id` = ? AND id = ? AND `pools`.`deleted_at` IS NULL")).
WithArgs(s.Fixtures.Enterprises[0].ID, pool.ID).
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE (id = ? and enterprise_id = ?) AND `pools`.`deleted_at` IS NULL ORDER BY `pools`.`id` LIMIT 1")).
WithArgs(pool.ID, s.Fixtures.Enterprises[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"enterprise_id", "id"}).AddRow(s.Fixtures.Enterprises[0].ID, pool.ID))
s.Fixtures.SQLMock.ExpectBegin()
s.Fixtures.SQLMock.

5
database/sql/models.go
View file

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
)
@ -66,6 +67,10 @@ type Pool struct {
OSArch config.OSArch
Tags []*Tag `gorm:"many2many:pool_tags;"`
Enabled bool
// ExtraSpecs is an opaque json that gets sent to the provider
// as part of the bootstrap params for instances. It can contain
// any kind of data needed by providers.
ExtraSpecs datatypes.JSON
RepoID uuid.UUID `gorm:"index"`
Repository Repository `gorm:"foreignKey:RepoID"`

35
database/sql/organizations.go
View file

@ -24,6 +24,7 @@ import (
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
)
@ -169,6 +170,10 @@ func (s *sqlDatabase) CreateOrganizationPool(ctx context.Context, orgId string,
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
}
if len(param.ExtraSpecs) > 0 {
newPool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
}
_, err = s.getOrgPoolByUniqueFields(ctx, orgId, newPool.ProviderName, newPool.Image, newPool.Flavor)
if err != nil {
if !errors.Is(err, runnerErrors.ErrNotFound) {
@ -296,7 +301,7 @@ func (s *sqlDatabase) getPoolByID(ctx context.Context, poolID string, preload ..
}
func (s *sqlDatabase) getOrgPool(ctx context.Context, orgID, poolID string, preload ...string) (Pool, error) {
org, err := s.getOrgByID(ctx, orgID)
_, err := s.getOrgByID(ctx, orgID)
if err != nil {
return Pool{}, errors.Wrap(err, "fetching org")
}
@ -313,33 +318,39 @@ func (s *sqlDatabase) getOrgPool(ctx context.Context, orgID, poolID string, prel
}
}
var pool []Pool
err = q.Model(&org).Association("Pools").Find(&pool, "id = ?", u)
var pool Pool
err = q.Model(&Pool{}).
Where("id = ? and org_id = ?", u, orgID).
First(&pool).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return Pool{}, errors.Wrap(runnerErrors.ErrNotFound, "finding pool")
}
return Pool{}, errors.Wrap(err, "fetching pool")
}
if len(pool) == 0 {
return Pool{}, runnerErrors.ErrNotFound
}
return pool[0], nil
return pool, nil
}
func (s *sqlDatabase) getOrgPools(ctx context.Context, orgID string, preload ...string) ([]Pool, error) {
org, err := s.getOrgByID(ctx, orgID)
_, err := s.getOrgByID(ctx, orgID)
if err != nil {
return nil, errors.Wrap(err, "fetching org")
}
var pools []Pool
q := s.conn.Model(&org)
q := s.conn
if len(preload) > 0 {
for _, item := range preload {
q = q.Preload(item)
}
}
err = q.Association("Pools").Find(&pools)
var pools []Pool
err = q.Model(&Pool{}).
Where("org_id = ?", orgID).
Omit("extra_specs").
Find(&pools).Error
if err != nil {
return nil, errors.Wrap(err, "fetching pool")

6
database/sql/organizations_test.go
View file

@ -701,7 +701,7 @@ func (s *OrgTestSuite) TestDeleteOrganizationPool() {
s.Require().Nil(err)
_, err = s.Store.GetOrganizationPool(context.Background(), s.Fixtures.Orgs[0].ID, pool.ID)
s.Require().Equal("fetching pool: not found", err.Error())
s.Require().Equal("fetching pool: finding pool: not found", err.Error())
}
func (s *OrgTestSuite) TestDeleteOrganizationPoolInvalidOrgID() {
@ -722,8 +722,8 @@ func (s *OrgTestSuite) TestDeleteOrganizationPoolDBDeleteErr() {
WithArgs(s.Fixtures.Orgs[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Orgs[0].ID))
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`org_id` = ? AND id = ? AND `pools`.`deleted_at` IS NULL")).
WithArgs(s.Fixtures.Orgs[0].ID, pool.ID).
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE (id = ? and org_id = ?) AND `pools`.`deleted_at` IS NULL ORDER BY `pools`.`id` LIMIT 1")).
WithArgs(pool.ID, s.Fixtures.Orgs[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"org_id", "id"}).AddRow(s.Fixtures.Orgs[0].ID, pool.ID))
s.Fixtures.SQLMock.ExpectBegin()
s.Fixtures.SQLMock.

1
database/sql/pools.go
View file

@ -30,6 +30,7 @@ func (s *sqlDatabase) ListAllPools(ctx context.Context) ([]params.Pool, error) {
Preload("Organization").
Preload("Repository").
Preload("Enterprise").
Omit("extra_specs").
Find(&pools)
if q.Error != nil {
return nil, errors.Wrap(q.Error, "fetching all pools")

2
database/sql/pools_test.go
View file

@ -127,7 +127,7 @@ func (s *PoolsTestSuite) TestListAllPools() {
func (s *PoolsTestSuite) TestListAllPoolsDBFetchErr() {
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")).
WillReturnError(fmt.Errorf("mocked fetching all pools error"))
_, err := s.StoreSQLMocked.ListAllPools(context.Background())

33
database/sql/repositories.go
View file

@ -24,6 +24,7 @@ import (
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
)
@ -169,6 +170,10 @@ func (s *sqlDatabase) CreateRepositoryPool(ctx context.Context, repoId string, p
RunnerBootstrapTimeout: param.RunnerBootstrapTimeout,
}
if len(param.ExtraSpecs) > 0 {
newPool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
}
_, err = s.getRepoPoolByUniqueFields(ctx, repoId, newPool.ProviderName, newPool.Image, newPool.Flavor)
if err != nil {
if !errors.Is(err, runnerErrors.ErrNotFound) {
@ -318,7 +323,7 @@ func (s *sqlDatabase) findPoolByTags(id, poolType string, tags []string) (params
}
func (s *sqlDatabase) getRepoPool(ctx context.Context, repoID, poolID string, preload ...string) (Pool, error) {
repo, err := s.getRepoByID(ctx, repoID)
_, err := s.getRepoByID(ctx, repoID)
if err != nil {
return Pool{}, errors.Wrap(err, "fetching repo")
}
@ -335,16 +340,19 @@ func (s *sqlDatabase) getRepoPool(ctx context.Context, repoID, poolID string, pr
}
}
var pool []Pool
err = q.Model(&repo).Association("Pools").Find(&pool, "id = ?", u)
var pool Pool
err = q.Model(&Pool{}).
Where("id = ? and repo_id = ?", u, repoID).
First(&pool).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return Pool{}, errors.Wrap(runnerErrors.ErrNotFound, "finding pool")
}
return Pool{}, errors.Wrap(err, "fetching pool")
}
if len(pool) == 0 {
return Pool{}, runnerErrors.ErrNotFound
}
return pool[0], nil
return pool, nil
}
func (s *sqlDatabase) getRepoPoolByUniqueFields(ctx context.Context, repoID string, provider, image, flavor string) (Pool, error) {
@ -367,19 +375,22 @@ func (s *sqlDatabase) getRepoPoolByUniqueFields(ctx context.Context, repoID stri
}
func (s *sqlDatabase) getRepoPools(ctx context.Context, repoID string, preload ...string) ([]Pool, error) {
repo, err := s.getRepoByID(ctx, repoID)
_, err := s.getRepoByID(ctx, repoID)
if err != nil {
return nil, errors.Wrap(err, "fetching repo")
}
var pools []Pool
q := s.conn.Model(&repo)
q := s.conn
if len(preload) > 0 {
for _, item := range preload {
q = q.Preload(item)
}
}
err = q.Association("Pools").Find(&pools)
var pools []Pool
err = q.Model(&Pool{}).Where("repo_id = ?", repoID).
Omit("extra_specs").
Find(&pools).Error
if err != nil {
return nil, errors.Wrap(err, "fetching pool")
}

4
database/sql/repositories_test.go
View file

@ -759,8 +759,8 @@ func (s *RepoTestSuite) TestDeleteRepositoryPoolDBDeleteErr() {
WithArgs(s.Fixtures.Repos[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Repos[0].ID))
s.Fixtures.SQLMock.
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`repo_id` = ? AND id = ? AND `pools`.`deleted_at` IS NULL")).
WithArgs(s.Fixtures.Repos[0].ID, pool.ID).
ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE (id = ? and repo_id = ?) AND `pools`.`deleted_at` IS NULL ORDER BY `pools`.`id` LIMIT 1")).
WithArgs(pool.ID, s.Fixtures.Repos[0].ID).
WillReturnRows(sqlmock.NewRows([]string{"repo_id", "id"}).AddRow(s.Fixtures.Repos[0].ID, pool.ID))
s.Fixtures.SQLMock.ExpectBegin()
s.Fixtures.SQLMock.

6
database/sql/util.go
View file

@ -22,6 +22,7 @@ import (
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"gorm.io/datatypes"
"gorm.io/gorm"
)
@ -141,6 +142,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) params.Pool {
Tags: make([]params.Tag, len(pool.Tags)),
Instances: make([]params.Instance, len(pool.Instances)),
RunnerBootstrapTimeout: pool.RunnerBootstrapTimeout,
ExtraSpecs: []byte(pool.ExtraSpecs.String()),
}
if pool.RepoID != uuid.Nil {
@ -270,6 +272,10 @@ func (s *sqlDatabase) updatePool(pool Pool, param params.UpdatePoolParams) (para
pool.OSType = param.OSType
}
if param.ExtraSpecs != nil {
pool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
}
if param.RunnerBootstrapTimeout != nil && *param.RunnerBootstrapTimeout > 0 {
pool.RunnerBootstrapTimeout = *param.RunnerBootstrapTimeout
}