DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add rudimentary database watcher

Browse source 
Adds a simple database watcher. At this point it's just one process, but
the plan is to allow different implementations that inform the local running
workers of changes that have occured on entities of interest in the database.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2024-04-03 14:46:32 +00:00
parent 214cb05072
commit 8d57fc8fa2
18 changed files with 514 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

20
database/sql/instances.go
View file

@ -25,29 +25,9 @@ import (
"gorm.io/gorm/clause"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) marshalAndSeal(data interface{}) ([]byte, error) {
enc, err := json.Marshal(data)
if err != nil {
return nil, errors.Wrap(err, "marshalling data")
}
return util.Seal(enc, []byte(s.cfg.Passphrase))
}
func (s *sqlDatabase) unsealAndUnmarshal(data []byte, target interface{}) error {
decrypted, err := util.Unseal(data, []byte(s.cfg.Passphrase))
if err != nil {
return errors.Wrap(err, "decrypting data")
}
if err := json.Unmarshal(decrypted, target); err != nil {
return errors.Wrap(err, "unmarshalling data")
}
return nil
}
func (s *sqlDatabase) CreateInstance(_ context.Context, poolID string, param params.CreateInstanceParams) (params.Instance, error) {
pool, err := s.getPoolByID(s.conn, poolID)
if err != nil {

33
database/sql/pools.go
View file

@ -24,6 +24,7 @@ import (
"gorm.io/gorm"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
)
@ -66,12 +67,18 @@ func (s *sqlDatabase) GetPoolByID(_ context.Context, poolID string) (params.Pool
return s.sqlToCommonPool(pool)
}
func (s *sqlDatabase) DeletePoolByID(_ context.Context, poolID string) error {
func (s *sqlDatabase) DeletePoolByID(_ context.Context, poolID string) (err error) {
pool, err := s.getPoolByID(s.conn, poolID)
if err != nil {
return errors.Wrap(err, "fetching pool by ID")
}
defer func() {
if err == nil {
s.sendNotify(common.PoolEntityType, common.DeleteOperation, pool)
}
}()
if q := s.conn.Unscoped().Delete(&pool); q.Error != nil {
return errors.Wrap(q.Error, "removing pool")
}
@ -247,11 +254,17 @@ func (s *sqlDatabase) FindPoolsMatchingAllTags(_ context.Context, entityType par
return pools, nil
}
func (s *sqlDatabase) CreateEntityPool(_ context.Context, entity params.GithubEntity, param params.CreatePoolParams) (params.Pool, error) {
func (s *sqlDatabase) CreateEntityPool(_ context.Context, entity params.GithubEntity, param params.CreatePoolParams) (pool params.Pool, err error) {
if len(param.Tags) == 0 {
return params.Pool{}, runnerErrors.NewBadRequestError("no tags specified")
}
defer func() {
if err == nil {
s.sendNotify(common.PoolEntityType, common.CreateOperation, pool)
}
}()
newPool := Pool{
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
@ -313,12 +326,12 @@ func (s *sqlDatabase) CreateEntityPool(_ context.Context, entity params.GithubEn
return params.Pool{}, err
}
pool, err := s.getPoolByID(s.conn, newPool.ID.String(), "Tags", "Instances", "Enterprise", "Organization", "Repository")
dbPool, err := s.getPoolByID(s.conn, newPool.ID.String(), "Tags", "Instances", "Enterprise", "Organization", "Repository")
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching pool")
}
return s.sqlToCommonPool(pool)
return s.sqlToCommonPool(dbPool)
}
func (s *sqlDatabase) GetEntityPool(_ context.Context, entity params.GithubEntity, poolID string) (params.Pool, error) {
@ -329,12 +342,21 @@ func (s *sqlDatabase) GetEntityPool(_ context.Context, entity params.GithubEntit
return s.sqlToCommonPool(pool)
}
func (s *sqlDatabase) DeleteEntityPool(_ context.Context, entity params.GithubEntity, poolID string) error {
func (s *sqlDatabase) DeleteEntityPool(_ context.Context, entity params.GithubEntity, poolID string) (err error) {
entityID, err := uuid.Parse(entity.ID)
if err != nil {
return errors.Wrap(runnerErrors.ErrBadRequest, "parsing id")
}
defer func() {
if err == nil {
pool := params.Pool{
ID: poolID,
}
s.sendNotify(common.PoolEntityType, common.DeleteOperation, pool)
}
}()
poolUUID, err := uuid.Parse(poolID)
if err != nil {
return errors.Wrap(runnerErrors.ErrBadRequest, "parsing pool id")
@ -374,6 +396,7 @@ func (s *sqlDatabase) UpdateEntityPool(_ context.Context, entity params.GithubEn
if err != nil {
return params.Pool{}, err
}
s.sendNotify(common.PoolEntityType, common.UpdateOperation, updatedPool)
return updatedPool, nil
}

34
database/sql/repositories.go
View file

@ -24,10 +24,17 @@ import (
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Repository, error) {
func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (param params.Repository, err error) {
defer func() {
if err == nil {
s.sendNotify(common.RepositoryEntityType, common.CreateOperation, param)
}
}()
if webhookSecret == "" {
return params.Repository{}, errors.New("creating repo: missing secret")
}
@ -68,7 +75,7 @@ func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, credent
return params.Repository{}, errors.Wrap(err, "creating repository")
}
param, err := s.sqlToCommonRepository(newRepo, true)
param, err = s.sqlToCommonRepository(newRepo, true)
if err != nil {
return params.Repository{}, errors.Wrap(err, "creating repository")
}
@ -113,12 +120,21 @@ func (s *sqlDatabase) ListRepositories(_ context.Context) ([]params.Repository,
return ret, nil
}
func (s *sqlDatabase) DeleteRepository(ctx context.Context, repoID string) error {
func (s *sqlDatabase) DeleteRepository(ctx context.Context, repoID string) (err error) {
repo, err := s.getRepoByID(ctx, s.conn, repoID)
if err != nil {
return errors.Wrap(err, "fetching repo")
}
defer func(repo Repository) {
if err == nil {
asParam, innerErr := s.sqlToCommonRepository(repo, true)
if innerErr == nil {
s.sendNotify(common.RepositoryEntityType, common.DeleteOperation, asParam)
}
}
}(repo)
q := s.conn.Unscoped().Delete(&repo)
if q.Error != nil && !errors.Is(q.Error, gorm.ErrRecordNotFound) {
return errors.Wrap(q.Error, "deleting repo")
@ -127,10 +143,15 @@ func (s *sqlDatabase) DeleteRepository(ctx context.Context, repoID string) error
return nil
}
func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param params.UpdateEntityParams) (params.Repository, error) {
func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param params.UpdateEntityParams) (newParams params.Repository, err error) {
defer func() {
if err == nil {
s.sendNotify(common.RepositoryEntityType, common.UpdateOperation, newParams)
}
}()
var repo Repository
var creds GithubCredentials
err := s.conn.Transaction(func(tx *gorm.DB) error {
err = s.conn.Transaction(func(tx *gorm.DB) error {
var err error
repo, err = s.getRepoByID(ctx, tx, repoID)
if err != nil {
@ -186,7 +207,8 @@ func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param
if err != nil {
return params.Repository{}, errors.Wrap(err, "updating enterprise")
}
newParams, err := s.sqlToCommonRepository(repo, true)
newParams, err = s.sqlToCommonRepository(repo, true)
if err != nil {
return params.Repository{}, errors.Wrap(err, "saving repo")
}

7
database/sql/repositories_test.go
View file

@ -30,6 +30,7 @@ import (
"github.com/cloudbase/garm/auth"
dbCommon "github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/database/watcher"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
)
@ -827,5 +828,11 @@ func (s *RepoTestSuite) TestUpdateRepositoryPoolInvalidRepoID() {
func TestRepoTestSuite(t *testing.T) {
t.Parallel()
watcher.SetWatcher(&garmTesting.MockWatcher{})
suite.Run(t, new(RepoTestSuite))
}
func init() {
watcher.SetWatcher(&garmTesting.MockWatcher{})
}

19
database/sql/sql.go
View file

@ -31,6 +31,7 @@ import (
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/database/watcher"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util/appdefaults"
)
@ -68,10 +69,15 @@ func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, err
if err != nil {
return nil, errors.Wrap(err, "creating DB connection")
}
producer, err := watcher.RegisterProducer("sql")
if err != nil {
return nil, errors.Wrap(err, "registering producer")
}
db := &sqlDatabase{
conn: conn,
ctx: ctx,
cfg: cfg,
conn: conn,
ctx: ctx,
cfg: cfg,
producer: producer,
}
if err := db.migrateDB(); err != nil {
@ -81,9 +87,10 @@ func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, err
}
type sqlDatabase struct {
conn *gorm.DB
ctx context.Context
cfg config.Database
conn *gorm.DB
ctx context.Context
cfg config.Database
producer common.Producer
}
var renameTemplate = `

29
database/sql/util.go
View file

@ -26,6 +26,7 @@ import (
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm-provider-common/util"
dbCommon "github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
)
@ -467,3 +468,31 @@ func (s *sqlDatabase) hasGithubEntity(tx *gorm.DB, entityType params.GithubEntit
}
return nil
}
func (s *sqlDatabase) marshalAndSeal(data interface{}) ([]byte, error) {
enc, err := json.Marshal(data)
if err != nil {
return nil, errors.Wrap(err, "marshalling data")
}
return util.Seal(enc, []byte(s.cfg.Passphrase))
}
func (s *sqlDatabase) unsealAndUnmarshal(data []byte, target interface{}) error {
decrypted, err := util.Unseal(data, []byte(s.cfg.Passphrase))
if err != nil {
return errors.Wrap(err, "decrypting data")
}
if err := json.Unmarshal(decrypted, target); err != nil {
return errors.Wrap(err, "unmarshalling data")
}
return nil
}
func (s *sqlDatabase) sendNotify(entityType dbCommon.DatabaseEntityType, op dbCommon.OperationType, payload interface{}) {
message := dbCommon.ChangePayload{
Operation: op,
Payload: payload,
EntityType: entityType,
}
s.producer.Notify(message)
}