From 2be56536836c7631c335731326a1d9a134cf374c Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 25 Apr 2022 00:03:26 +0000 Subject: [PATCH] Added some DB logic --- config/config.go | 39 +++- database/common/common.go | 30 +++ database/sql/models.go | 57 +++++ database/sql/sql.go | 451 ++++++++++++++++++++++++++++++++++++++ go.mod | 8 + go.sum | 18 ++ params/params.go | 43 ++++ runner/pool/repository.go | 49 +++-- util/util.go | 96 ++++++++ 9 files changed, 765 insertions(+), 26 deletions(-) create mode 100644 database/common/common.go create mode 100644 database/sql/models.go create mode 100644 database/sql/sql.go diff --git a/config/config.go b/config/config.go index 79dff5f5..38c6614c 100644 --- a/config/config.go +++ b/config/config.go @@ -232,7 +232,9 @@ func (p *Provider) Validate() error { // it has, the image it runs on and the size of the compute system that was // requested. type Runner struct { - // Name is the name of this runner. + // Name is the name of this runner. The name needs to be unique within a provider, + // and is used as an ID. If you wish to change the name, you must make sure all + // runners of this type are deleted. Name string `toml:"name" json:"name"` // Labels is a list of labels that will be set for this runner in github. // The labels will be used in workflows to request a particular kind of @@ -241,12 +243,13 @@ type Runner struct { // MaxRunners is the maximum number of self hosted action runners // of any type that are spun up for this repo. If current worker count // is not enough to handle jobs comming in, a new runner will be spun up, - // until MaxWorkers count is hit. + // until MaxWorkers count is hit. Set this to 0 to disable MaxRunners. MaxRunners int `toml:"max_runners" json:"max-runners"` - // MinRunners is the minimum number of self hosted runners that will - // be maintained for this repo. If no jobs are sent to the workers, - // idle workers will be removed until the MinWorkers setting is reached. - MinRunners int `toml:"min_runners" json:"min-runners"` + // MinIdleRunners is the minimum number of idle self hosted runners that will + // be maintained for this repo. Ensuring a few idle runners, speeds up jobs, especially + // on providers where cold boot takes a long time. The pool will attempt to maintain at + // least this many idle workers, unless MaxRunners is hit. Set this to 0, for on-demand. + MinIdleRunners int `toml:"min_idle_runners" json:"min-runners"` // Flavor is the size of the VM that will be spun up. Flavor string `toml:"flavor" json:"flavor"` @@ -264,6 +267,21 @@ type Runner struct { OSArch OSArch `toml:"os_arch" json:"os-arch"` } +func (r *Runner) HasAllLabels(labels []string) bool { + hashed := map[string]struct{}{} + for _, val := range r.Labels { + hashed[val] = struct{}{} + } + + for _, val := range labels { + if _, ok := hashed[val]; !ok { + return false + } + } + + return true +} + // TODO: validate rest func (r *Runner) Validate() error { if len(r.Labels) == 0 { @@ -352,6 +370,12 @@ type Database struct { DbBackend DBBackendType `toml:"backend" json:"backend"` MySQL MySQL `toml:"mysql" json:"mysql"` SQLite SQLite `toml:"sqlite3" json:"sqlite3"` + // Passphrase is used to encrypt any sensitive info before + // inserting it into the database. This is just temporary until + // we move to something like vault or barbican for secrets storage. + // Don't lose or change this. It will invalidate all encrypted data + // in the DB. This field must be set and must be exactly 32 characters. + Passphrase string `toml:"passphrase"` } // GormParams returns the database type and connection URI @@ -382,6 +406,9 @@ func (d *Database) Validate() error { if d.DbBackend == "" { return fmt.Errorf("invalid databse configuration: backend is required") } + if len(d.Passphrase) != 32 { + return fmt.Errorf("passphrase must be set and it must be a string of 32 characters (aes 256)") + } switch d.DbBackend { case MySQLBackend: if err := d.MySQL.Validate(); err != nil { diff --git a/database/common/common.go b/database/common/common.go new file mode 100644 index 00000000..5dd0bdd4 --- /dev/null +++ b/database/common/common.go @@ -0,0 +1,30 @@ +package common + +import ( + "context" + "runner-manager/params" +) + +type Store interface { + CreateRepository(ctx context.Context, owner, name, webhookSecret string) (params.Repository, error) + GetRepository(ctx context.Context, id string) (params.Repository, error) + ListRepositories(ctx context.Context) ([]params.Repository, error) + DeleteRepository(ctx context.Context, id string) error + + CreateOrganization(ctx context.Context, name, webhookSecret string) (params.Organization, error) + GetOrganization(ctx context.Context, id string) (params.Organization, error) + ListOrganizations(ctx context.Context) ([]params.Organization, error) + DeleteOrganization(ctx context.Context, id string) error + + CreateRepositoryPool(ctx context.Context, repoId string, param params.CreatePoolParams) (params.Pool, error) + CreateOrganizationPool(ctx context.Context, orgId string, param params.CreatePoolParams) (params.Pool, error) + + GetRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) + GetOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) + + DeleteRepositoryPool(ctx context.Context, repoID, poolID string) error + DeleteOrganizationPool(ctx context.Context, orgID, poolID string) error + + UpdateRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) + UpdateOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) +} diff --git a/database/sql/models.go b/database/sql/models.go new file mode 100644 index 00000000..16ba82c5 --- /dev/null +++ b/database/sql/models.go @@ -0,0 +1,57 @@ +package sql + +import ( + "runner-manager/config" + "time" + + uuid "github.com/satori/go.uuid" + "gorm.io/gorm" +) + +type Base struct { + ID uuid.UUID `gorm:"type:uuid;primary_key;"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt `gorm:"index"` +} + +func (b *Base) BeforeCreate(tx *gorm.DB) error { + b.ID = uuid.NewV4() + return nil +} + +type Tag struct { + Base + + Name string `gorm:"type:varchar(64);uniqueIndex"` +} + +type Pool struct { + Base + + ProviderName string `gorm:"index:idx_pool_type,unique"` + MaxRunners uint + MinIdleRunners uint + Image string `gorm:"index:idx_pool_type,unique"` + Flavor string `gorm:"index:idx_pool_type,unique"` + OSType config.OSType + OSArch config.OSArch + Tags []Tag `gorm:"foreignKey:id"` +} + +type Repository struct { + Base + + Owner string `gorm:"index:idx_owner,unique"` + Name string `gorm:"index:idx_owner,unique"` + WebhookSecret []byte + Pools []Pool `gorm:"foreignKey:id"` +} + +type Organization struct { + Base + + Name string `gorm:"uniqueIndex"` + WebhookSecret []byte + Pools []Pool `gorm:"foreignKey:id"` +} diff --git a/database/sql/sql.go b/database/sql/sql.go new file mode 100644 index 00000000..91c02dfd --- /dev/null +++ b/database/sql/sql.go @@ -0,0 +1,451 @@ +package sql + +import ( + "context" + "fmt" + "runner-manager/config" + "runner-manager/database/common" + runnerErrors "runner-manager/errors" + "runner-manager/params" + "runner-manager/util" + + "github.com/pborman/uuid" + "github.com/pkg/errors" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, error) { + conn, err := util.NewDBConn(cfg) + if err != nil { + return nil, errors.Wrap(err, "creating DB connection") + } + db := &sqlDatabase{ + conn: conn, + ctx: ctx, + cfg: cfg, + } + + if err := db.migrateDB(); err != nil { + return nil, errors.Wrap(err, "migrating database") + } + return db, nil +} + +type sqlDatabase struct { + conn *gorm.DB + ctx context.Context + cfg config.Database +} + +func (s *sqlDatabase) migrateDB() error { + if err := s.conn.AutoMigrate( + &Tag{}, + // &Runner{}, + &Pool{}, + &Repository{}, + &Organization{}, + ); err != nil { + return err + } + + return nil +} + +func (s *sqlDatabase) sqlToCommonTags(tag Tag) params.Tag { + return params.Tag{ + ID: tag.ID.String(), + Name: tag.Name, + } +} + +// func (s *sqlDatabase) sqlToCommonRunner(runner Runner) params.Runner { +// ret := params.Runner{ +// ID: runner.ID.String(), +// MaxRunners: runner.MaxRunners, +// MinIdleRunners: runner.MinIdleRunners, +// Image: runner.Image, +// Flavor: runner.Flavor, +// OSArch: runner.OSArch, +// OSType: runner.OSType, +// Tags: make([]params.Tag, len(runner.Tags)), +// } + +// for idx, val := range runner.Tags { +// ret.Tags[idx] = s.sqlToCommonTags(val) +// } + +// return ret +// } + +func (s *sqlDatabase) sqlToCommonPool(pool Pool) params.Pool { + ret := params.Pool{ + ID: pool.ID.String(), + ProviderName: pool.ProviderName, + MaxRunners: pool.MaxRunners, + MinIdleRunners: pool.MinIdleRunners, + Image: pool.Image, + Flavor: pool.Flavor, + OSArch: pool.OSArch, + OSType: pool.OSType, + Tags: make([]params.Tag, len(pool.Tags)), + } + + for idx, val := range pool.Tags { + ret.Tags[idx] = s.sqlToCommonTags(val) + } + + return ret +} + +func (s *sqlDatabase) sqlToCommonRepository(repo Repository) params.Repository { + ret := params.Repository{ + ID: repo.ID.String(), + Name: repo.Name, + Owner: repo.Owner, + Pools: make([]params.Pool, len(repo.Pools)), + } + + for idx, pool := range repo.Pools { + ret.Pools[idx] = s.sqlToCommonPool(pool) + } + + return ret +} + +func (s *sqlDatabase) sqlToCommonOrganization(org Organization) params.Organization { + ret := params.Organization{ + ID: org.ID.String(), + Name: org.Name, + Pools: make([]params.Pool, len(org.Pools)), + } + + return ret +} + +func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, webhookSecret string) (params.Repository, error) { + secret := []byte{} + var err error + if webhookSecret != "" { + secret, err = util.Aes256EncodeString(webhookSecret, s.cfg.Passphrase) + if err != nil { + return params.Repository{}, fmt.Errorf("failed to encrypt string") + } + } + newRepo := Repository{ + Name: name, + Owner: owner, + WebhookSecret: secret, + } + + q := s.conn.Create(&newRepo) + if q.Error != nil { + return params.Repository{}, errors.Wrap(q.Error, "creating repository") + } + + param := s.sqlToCommonRepository(newRepo) + param.WebhookSecret = webhookSecret + + return param, nil +} + +func (s *sqlDatabase) getRepo(ctx context.Context, id string) (Repository, error) { + u := uuid.Parse(id) + if u == nil { + return Repository{}, errors.Wrap(runnerErrors.NewBadRequestError(""), "parsing id") + } + var repo Repository + q := s.conn.Preload(clause.Associations).Where("id = ?", u).First(&repo) + if q.Error != nil { + if errors.Is(q.Error, gorm.ErrRecordNotFound) { + return Repository{}, runnerErrors.ErrNotFound + } + return Repository{}, errors.Wrap(q.Error, "fetching repository from database") + } + return repo, nil +} + +func (s *sqlDatabase) GetRepository(ctx context.Context, id string) (params.Repository, error) { + repo, err := s.getRepo(ctx, id) + if err != nil { + return params.Repository{}, errors.Wrap(err, "fetching repo") + } + + param := s.sqlToCommonRepository(repo) + secret, err := util.Aes256DecodeString(repo.WebhookSecret, s.cfg.Passphrase) + if err != nil { + return params.Repository{}, errors.Wrap(err, "decrypting secret") + } + param.WebhookSecret = secret + + return param, nil +} + +func (s *sqlDatabase) ListRepositories(ctx context.Context) ([]params.Repository, error) { + var repos []Repository + q := s.conn.Find(&repos) + if q.Error != nil { + return []params.Repository{}, errors.Wrap(q.Error, "fetching user from database") + } + + ret := make([]params.Repository, len(repos)) + for idx, val := range repos { + ret[idx] = s.sqlToCommonRepository(val) + } + + return ret, nil +} + +func (s *sqlDatabase) DeleteRepository(ctx context.Context, id string) error { + repo, err := s.getRepo(ctx, id) + if err != nil { + if err == runnerErrors.ErrNotFound { + return nil + } + return errors.Wrap(err, "fetching repo") + } + + q := s.conn.Delete(&repo) + if q.Error != nil && !errors.Is(q.Error, gorm.ErrRecordNotFound) { + return errors.Wrap(q.Error, "deleting repo") + } + + return nil +} + +func (s *sqlDatabase) CreateOrganization(ctx context.Context, name, webhookSecret string) (params.Organization, error) { + secret := []byte{} + var err error + if webhookSecret != "" { + secret, err = util.Aes256EncodeString(webhookSecret, s.cfg.Passphrase) + if err != nil { + return params.Organization{}, fmt.Errorf("failed to encrypt string") + } + } + newOrg := Organization{ + Name: name, + WebhookSecret: secret, + } + + q := s.conn.Create(&newOrg) + if q.Error != nil { + return params.Organization{}, errors.Wrap(q.Error, "creating org") + } + + param := s.sqlToCommonOrganization(newOrg) + param.WebhookSecret = webhookSecret + + return param, nil +} + +func (s *sqlDatabase) getOrg(ctx context.Context, id string) (Organization, error) { + u := uuid.Parse(id) + if u == nil { + return Organization{}, errors.Wrap(runnerErrors.NewBadRequestError(""), "parsing id") + } + var org Organization + q := s.conn.Preload(clause.Associations).Where("id = ?", u).First(&org) + if q.Error != nil { + if errors.Is(q.Error, gorm.ErrRecordNotFound) { + return Organization{}, runnerErrors.ErrNotFound + } + return Organization{}, errors.Wrap(q.Error, "fetching org from database") + } + return org, nil +} + +func (s *sqlDatabase) GetOrganization(ctx context.Context, id string) (params.Organization, error) { + org, err := s.getOrg(ctx, id) + if err != nil { + return params.Organization{}, errors.Wrap(err, "fetching repo") + } + + param := s.sqlToCommonOrganization(org) + secret, err := util.Aes256DecodeString(org.WebhookSecret, s.cfg.Passphrase) + if err != nil { + return params.Organization{}, errors.Wrap(err, "decrypting secret") + } + param.WebhookSecret = secret + + return param, nil +} + +func (s *sqlDatabase) ListOrganizations(ctx context.Context) ([]params.Organization, error) { + var orgs []Organization + q := s.conn.Find(&orgs) + if q.Error != nil { + return []params.Organization{}, errors.Wrap(q.Error, "fetching user from database") + } + + ret := make([]params.Organization, len(orgs)) + for idx, val := range orgs { + ret[idx] = s.sqlToCommonOrganization(val) + } + + return ret, nil +} + +func (s *sqlDatabase) DeleteOrganization(ctx context.Context, id string) error { + org, err := s.getOrg(ctx, id) + if err != nil { + if err == runnerErrors.ErrNotFound { + return nil + } + return errors.Wrap(err, "fetching repo") + } + + q := s.conn.Delete(&org) + if q.Error != nil && !errors.Is(q.Error, gorm.ErrRecordNotFound) { + return errors.Wrap(q.Error, "deleting org") + } + + return nil +} + +func (s *sqlDatabase) getOrCreateTag(tagName string) (Tag, error) { + var tag Tag + q := s.conn.Where("name = ?", tagName).First(&tag) + if q.Error == nil { + return tag, nil + } + if !errors.Is(q.Error, gorm.ErrRecordNotFound) { + return Tag{}, errors.Wrap(q.Error, "fetching tag from database") + } + newTag := Tag{ + Name: tagName, + } + + q = s.conn.Create(&newTag) + if q.Error != nil { + return Tag{}, errors.Wrap(q.Error, "creating tag") + } + return newTag, nil +} + +func (s *sqlDatabase) CreateRepositoryPool(ctx context.Context, repoId string, param params.CreatePoolParams) (params.Pool, error) { + if len(param.Tags) == 0 { + return params.Pool{}, runnerErrors.NewBadRequestError("no tags specified") + } + + repo, err := s.getRepo(ctx, repoId) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching repo") + } + + newPool := Pool{ + ProviderName: param.ProviderName, + MaxRunners: param.MaxRunners, + MinIdleRunners: param.MinIdleRunners, + Image: param.Image, + Flavor: param.Flavor, + OSType: param.OSType, + OSArch: param.OSArch, + } + + tags := make([]Tag, len(param.Tags)) + for idx, val := range param.Tags { + t, err := s.getOrCreateTag(val) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching tag") + } + tags[idx] = t + } + + err = s.conn.Model(&repo).Association("Pools").Append(&newPool) + if err != nil { + return params.Pool{}, errors.Wrap(err, "adding pool") + } + return s.sqlToCommonPool(newPool), nil +} + +func (s *sqlDatabase) CreateOrganizationPool(ctx context.Context, orgId string, param params.CreatePoolParams) (params.Pool, error) { + if len(param.Tags) == 0 { + return params.Pool{}, runnerErrors.NewBadRequestError("no tags specified") + } + + org, err := s.getOrg(ctx, orgId) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching org") + } + + newPool := Pool{ + ProviderName: param.ProviderName, + MaxRunners: param.MaxRunners, + MinIdleRunners: param.MinIdleRunners, + Image: param.Image, + Flavor: param.Flavor, + OSType: param.OSType, + OSArch: param.OSArch, + } + + tags := make([]Tag, len(param.Tags)) + for idx, val := range param.Tags { + t, err := s.getOrCreateTag(val) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching tag") + } + tags[idx] = t + } + + err = s.conn.Model(&org).Association("Pools").Append(&newPool) + if err != nil { + return params.Pool{}, errors.Wrap(err, "adding pool") + } + return s.sqlToCommonPool(newPool), nil +} + +func (s *sqlDatabase) GetRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) { + repo, err := s.getRepo(ctx, repoID) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching repo") + } + u := uuid.Parse(poolID) + if u == nil { + return params.Pool{}, fmt.Errorf("invalid pool id") + } + var pool []Pool + err = s.conn.Model(&repo).Association("Pools").Find(&pool, "id = ?", u) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching pool") + } + if len(pool) == 0 { + return params.Pool{}, runnerErrors.ErrNotFound + } + return s.sqlToCommonPool(pool[0]), nil +} + +func (s *sqlDatabase) GetOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) { + org, err := s.getOrg(ctx, orgID) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching org") + } + u := uuid.Parse(poolID) + if u == nil { + return params.Pool{}, fmt.Errorf("invalid pool id") + } + var pool []Pool + err = s.conn.Model(&org).Association("Pools").Find(&pool, "id = ?", u) + if err != nil { + return params.Pool{}, errors.Wrap(err, "fetching pool") + } + if len(pool) == 0 { + return params.Pool{}, runnerErrors.ErrNotFound + } + return s.sqlToCommonPool(pool[0]), nil +} + +func (s *sqlDatabase) DeleteRepositoryPool(ctx context.Context, repoID, poolID string) error { + return nil +} + +func (s *sqlDatabase) DeleteOrganizationPool(ctx context.Context, orgID, poolID string) error { + return nil +} + +func (s *sqlDatabase) UpdateRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) { + return params.Pool{}, nil +} + +func (s *sqlDatabase) UpdateOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) { + return params.Pool{}, nil +} diff --git a/go.mod b/go.mod index f9e00fc6..1637437c 100644 --- a/go.mod +++ b/go.mod @@ -10,24 +10,32 @@ require ( github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124 github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 + github.com/satori/go.uuid v1.2.0 golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gorm.io/driver/mysql v1.3.3 + gorm.io/driver/sqlite v1.3.2 + gorm.io/gorm v1.23.4 ) require ( github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect github.com/go-macaroon-bakery/macaroonpb v1.0.0 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/juju/webbrowser v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-sqlite3 v1.14.12 // indirect github.com/pkg/sftp v1.13.4 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/fastuuid v1.2.0 // indirect diff --git a/go.sum b/go.sum index b3fa3e30..b2133731 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebP github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/go-macaroon-bakery/macaroonpb v1.0.0 h1:It9exBaRMZ9iix1iJ6gwzfwsDE6ExNuwtAJ9e09v6XE= github.com/go-macaroon-bakery/macaroonpb v1.0.0/go.mod h1:UzrGOcbiwTXISFP2XDLDPjfhMINZa+fX/7A2lMd31zc= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -60,6 +62,11 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/juju/mgotest v1.0.1/go.mod h1:vTaDufYul+Ps8D7bgseHjq87X8eu0ivlKLp9mVc/Bfc= github.com/juju/postgrestest v1.1.0/go.mod h1:/n17Y2T6iFozzXwSCO0JYJ5gSiz2caEtSwAjh/uLXDM= github.com/juju/qthttptest v0.0.1/go.mod h1://LCf/Ls22/rPw2u1yWukUJvYtfPY4nYpWUl2uZhryo= @@ -88,6 +95,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124 h1:EmjWCASxSUz+ymsEJfiWN3yx3yTypoKJrnOSSzAWYds= github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124/go.mod h1:T4xjj62BmFg1L5JfY2wRyPZtKbBeTFgo/GLwV8DFZ8M= +github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0= +github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= @@ -104,6 +113,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -220,5 +231,12 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.3.3 h1:jXG9ANrwBc4+bMvBcSl8zCfPBaVoPyBEBshA8dA93X8= +gorm.io/driver/mysql v1.3.3/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U= +gorm.io/driver/sqlite v1.3.2 h1:nWTy4cE52K6nnMhv23wLmur9Y3qWbZvOBz+V4PrGAxg= +gorm.io/driver/sqlite v1.3.2/go.mod h1:B+8GyC9K7VgzJAcrcXMRPdnMcck+8FgJynEehEPM16U= +gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.4 h1:1BKWM67O6CflSLcwGQR7ccfmC4ebOxQrTfOQGRE9wjg= +gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/params/params.go b/params/params.go index 2a45de38..ceac8454 100644 --- a/params/params.go +++ b/params/params.go @@ -59,3 +59,46 @@ type BootstrapInstance struct { Image string `json:"image"` Labels []string `json:"labels"` } + +type Tag struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Pool struct { + ID string `json:"id"` + ProviderName string `json:"provider_name"` + MaxRunners uint `json:"max_runners"` + MinIdleRunners uint `json:"min_idle_runners"` + Image string `json:"image"` + Flavor string `json:"flavor"` + OSType config.OSType `json:"os_type"` + OSArch config.OSArch `json:"os_arch"` + Tags []Tag `json:"tags"` +} + +type Repository struct { + ID string `json:"id"` + Owner string `json:"owner"` + Name string `json:"name"` + WebhookSecret string `json:"-"` + Pools []Pool `json:"pool,omitempty"` +} + +type Organization struct { + ID string `json:"id"` + Name string `json:"name"` + WebhookSecret string `json:"-"` + Pools []Pool `json:"pool,omitempty"` +} + +type CreatePoolParams struct { + ProviderName string `json:"provider_name"` + MaxRunners uint `json:"max_runners"` + MinIdleRunners uint `json:"min_idle_runners"` + Image string `json:"image"` + Flavor string `json:"flavor"` + OSType config.OSType `json:"os_type"` + OSArch config.OSArch `json:"os_arch"` + Tags []string `json:"tags"` +} diff --git a/runner/pool/repository.go b/runner/pool/repository.go index f4c95233..664967b1 100644 --- a/runner/pool/repository.go +++ b/runner/pool/repository.go @@ -19,19 +19,20 @@ import ( // test that we implement PoolManager var _ common.PoolManager = &Repository{} -func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, ghcli *github.Client, provider common.Provider) (common.PoolManager, error) { +func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, provider common.Provider, ghcli *github.Client, controllerID string) (common.PoolManager, error) { queueSize := cfg.Pool.QueueSize if queueSize == 0 { queueSize = config.DefaultPoolQueueSize } repo := &Repository{ - ctx: ctx, - cfg: cfg, - ghcli: ghcli, - provider: provider, - jobQueue: make(chan params.WorkflowJob, queueSize), - quit: make(chan struct{}), - done: make(chan struct{}), + ctx: ctx, + cfg: cfg, + ghcli: ghcli, + provider: provider, + controllerID: controllerID, + jobQueue: make(chan params.WorkflowJob, queueSize), + quit: make(chan struct{}), + done: make(chan struct{}), } if err := repo.fetchTools(); err != nil { @@ -41,19 +42,25 @@ func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, ghcli * } type Repository struct { - ctx context.Context - cfg config.Repository - ghcli *github.Client - provider common.Provider - tools []*github.RunnerApplicationDownload - jobQueue chan params.WorkflowJob - quit chan struct{} - done chan struct{} - mux sync.Mutex + ctx context.Context + controllerID string + cfg config.Repository + ghcli *github.Client + provider common.Provider + tools []*github.RunnerApplicationDownload + jobQueue chan params.WorkflowJob + quit chan struct{} + done chan struct{} + mux sync.Mutex } -func (r *Repository) getGithubRunners() ([]github.Runner, error) { - return nil, nil +func (r *Repository) getGithubRunners() ([]*github.Runner, error) { + runners, _, err := r.ghcli.Actions.ListRunners(r.ctx, r.cfg.Owner, r.cfg.Name, nil) + if err != nil { + return nil, errors.Wrap(err, "fetching runners") + } + + return runners.Runners, nil } func (r *Repository) getProviderInstances() ([]params.Instance, error) { @@ -92,7 +99,7 @@ func (r *Repository) Wait() error { func (r *Repository) loop() { defer close(r.done) - // TODO: Consolidate runners on loop start. Local runners must match runners + // TODO: Consolidate runners on loop start. Provider runners must match runners // in github and DB. When a Workflow job is received, we will first create/update // an entity in the database, before sending the request to the provider to create/delete // an instance. If a "queued" job is received, we create an entity in the db with @@ -116,6 +123,8 @@ func (r *Repository) loop() { // Create instance. case "completed": // Remove instance. + case "in_progress": + // update state } fmt.Println(job) case <-time.After(3 * time.Hour): diff --git a/util/util.go b/util/util.go index 30ffb772..8106d41f 100644 --- a/util/util.go +++ b/util/util.go @@ -3,6 +3,8 @@ package util import ( "bytes" "context" + "crypto/aes" + "crypto/cipher" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -20,6 +22,9 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/oauth2" lumberjack "gopkg.in/natefinch/lumberjack.v2" + "gorm.io/driver/mysql" + "gorm.io/driver/sqlite" + "gorm.io/gorm" "runner-manager/cloudconfig" "runner-manager/config" @@ -27,6 +32,8 @@ import ( "runner-manager/params" ) +const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + var ( OSToOSTypeMap map[string]config.OSType = map[string]config.OSType{ "ubuntu": config.Linux, @@ -158,3 +165,92 @@ func GetCloudConfig(bootstrapParams params.BootstrapInstance, tools github.Runne } return asStr, nil } + +// NewDBConn returns a new gorm db connection, given the config +func NewDBConn(dbCfg config.Database) (conn *gorm.DB, err error) { + dbType, connURI, err := dbCfg.GormParams() + if err != nil { + return nil, errors.Wrap(err, "getting DB URI string") + } + switch dbType { + case config.MySQLBackend: + conn, err = gorm.Open(mysql.Open(connURI), &gorm.Config{}) + case config.SQLiteBackend: + conn, err = gorm.Open(sqlite.Open(connURI), &gorm.Config{}) + } + if err != nil { + return nil, errors.Wrap(err, "connecting to database") + } + + if dbCfg.Debug { + conn = conn.Debug() + } + return conn, nil +} + +// GetRandomString returns a secure random string +func GetRandomString(n int) (string, error) { + data := make([]byte, n) + _, err := rand.Read(data) + if err != nil { + return "", errors.Wrap(err, "getting random data") + } + for i, b := range data { + data[i] = alphanumeric[b%byte(len(alphanumeric))] + } + + return string(data), nil +} + +func Aes256EncodeString(target string, passphrase string) ([]byte, error) { + if len(passphrase) != 32 { + return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)") + } + + toEncrypt := []byte(target) + block, err := aes.NewCipher([]byte(passphrase)) + if err != nil { + return nil, errors.Wrap(err, "creating cipher") + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Wrap(err, "creating new aead") + } + + nonce := make([]byte, aesgcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errors.Wrap(err, "creating nonce") + } + + ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil) + return ciphertext, nil +} + +func Aes256DecodeString(target []byte, passphrase string) (string, error) { + if len(passphrase) != 32 { + return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)") + } + + block, err := aes.NewCipher([]byte(passphrase)) + if err != nil { + return "", errors.Wrap(err, "creating cipher") + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return "", errors.Wrap(err, "creating new aead") + } + + nonceSize := aesgcm.NonceSize() + if len(target) < nonceSize { + return "", fmt.Errorf("failed to decrypt text") + } + + nonce, ciphertext := target[:nonceSize], target[nonceSize:] + plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", fmt.Errorf("failed to decrypt text") + } + return string(plaintext), nil +}