DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Added some DB logic

Browse source
This commit is contained in:
Gabriel Adrian Samfira 2022-04-25 00:03:26 +00:00
parent ee207b0b54
commit 2be5653683
9 changed files with 765 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

39
config/config.go
View file

@ -232,7 +232,9 @@ func (p *Provider) Validate() error {
// it has, the image it runs on and the size of the compute system that was
// requested.
type Runner struct {
// Name is the name of this runner.
// Name is the name of this runner. The name needs to be unique within a provider,
// and is used as an ID. If you wish to change the name, you must make sure all
// runners of this type are deleted.
Name string `toml:"name" json:"name"`
// Labels is a list of labels that will be set for this runner in github.
// The labels will be used in workflows to request a particular kind of
@ -241,12 +243,13 @@ type Runner struct {
// MaxRunners is the maximum number of self hosted action runners
// of any type that are spun up for this repo. If current worker count
// is not enough to handle jobs comming in, a new runner will be spun up,
// until MaxWorkers count is hit.
// until MaxWorkers count is hit. Set this to 0 to disable MaxRunners.
MaxRunners int `toml:"max_runners" json:"max-runners"`
// MinRunners is the minimum number of self hosted runners that will
// be maintained for this repo. If no jobs are sent to the workers,
// idle workers will be removed until the MinWorkers setting is reached.
MinRunners int `toml:"min_runners" json:"min-runners"`
// MinIdleRunners is the minimum number of idle self hosted runners that will
// be maintained for this repo. Ensuring a few idle runners, speeds up jobs, especially
// on providers where cold boot takes a long time. The pool will attempt to maintain at
// least this many idle workers, unless MaxRunners is hit. Set this to 0, for on-demand.
MinIdleRunners int `toml:"min_idle_runners" json:"min-runners"`
// Flavor is the size of the VM that will be spun up.
Flavor string `toml:"flavor" json:"flavor"`
@ -264,6 +267,21 @@ type Runner struct {
OSArch OSArch `toml:"os_arch" json:"os-arch"`
}
func (r *Runner) HasAllLabels(labels []string) bool {
hashed := map[string]struct{}{}
for _, val := range r.Labels {
hashed[val] = struct{}{}
}
for _, val := range labels {
if _, ok := hashed[val]; !ok {
return false
}
}
return true
}
// TODO: validate rest
func (r *Runner) Validate() error {
if len(r.Labels) == 0 {
@ -352,6 +370,12 @@ type Database struct {
DbBackend DBBackendType `toml:"backend" json:"backend"`
MySQL MySQL `toml:"mysql" json:"mysql"`
SQLite SQLite `toml:"sqlite3" json:"sqlite3"`
// Passphrase is used to encrypt any sensitive info before
// inserting it into the database. This is just temporary until
// we move to something like vault or barbican for secrets storage.
// Don't lose or change this. It will invalidate all encrypted data
// in the DB. This field must be set and must be exactly 32 characters.
Passphrase string `toml:"passphrase"`
}
// GormParams returns the database type and connection URI
@ -382,6 +406,9 @@ func (d *Database) Validate() error {
if d.DbBackend == "" {
return fmt.Errorf("invalid databse configuration: backend is required")
}
if len(d.Passphrase) != 32 {
return fmt.Errorf("passphrase must be set and it must be a string of 32 characters (aes 256)")
}
switch d.DbBackend {
case MySQLBackend:
if err := d.MySQL.Validate(); err != nil {

30
database/common/common.go Normal file
View file

@ -0,0 +1,30 @@
package common
import (
"context"
"runner-manager/params"
)
type Store interface {
CreateRepository(ctx context.Context, owner, name, webhookSecret string) (params.Repository, error)
GetRepository(ctx context.Context, id string) (params.Repository, error)
ListRepositories(ctx context.Context) ([]params.Repository, error)
DeleteRepository(ctx context.Context, id string) error
CreateOrganization(ctx context.Context, name, webhookSecret string) (params.Organization, error)
GetOrganization(ctx context.Context, id string) (params.Organization, error)
ListOrganizations(ctx context.Context) ([]params.Organization, error)
DeleteOrganization(ctx context.Context, id string) error
CreateRepositoryPool(ctx context.Context, repoId string, param params.CreatePoolParams) (params.Pool, error)
CreateOrganizationPool(ctx context.Context, orgId string, param params.CreatePoolParams) (params.Pool, error)
GetRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error)
GetOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error)
DeleteRepositoryPool(ctx context.Context, repoID, poolID string) error
DeleteOrganizationPool(ctx context.Context, orgID, poolID string) error
UpdateRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error)
UpdateOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error)
}

57
database/sql/models.go Normal file
View file

@ -0,0 +1,57 @@
package sql
import (
"runner-manager/config"
"time"
uuid "github.com/satori/go.uuid"
"gorm.io/gorm"
)
type Base struct {
ID uuid.UUID `gorm:"type:uuid;primary_key;"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt `gorm:"index"`
}
func (b *Base) BeforeCreate(tx *gorm.DB) error {
b.ID = uuid.NewV4()
return nil
}
type Tag struct {
Base
Name string `gorm:"type:varchar(64);uniqueIndex"`
}
type Pool struct {
Base
ProviderName string `gorm:"index:idx_pool_type,unique"`
MaxRunners uint
MinIdleRunners uint
Image string `gorm:"index:idx_pool_type,unique"`
Flavor string `gorm:"index:idx_pool_type,unique"`
OSType config.OSType
OSArch config.OSArch
Tags []Tag `gorm:"foreignKey:id"`
}
type Repository struct {
Base
Owner string `gorm:"index:idx_owner,unique"`
Name string `gorm:"index:idx_owner,unique"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:id"`
}
type Organization struct {
Base
Name string `gorm:"uniqueIndex"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:id"`
}

451
database/sql/sql.go Normal file
View file

@ -0,0 +1,451 @@
package sql
import (
"context"
"fmt"
"runner-manager/config"
"runner-manager/database/common"
runnerErrors "runner-manager/errors"
"runner-manager/params"
"runner-manager/util"
"github.com/pborman/uuid"
"github.com/pkg/errors"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
func NewSQLDatabase(ctx context.Context, cfg config.Database) (common.Store, error) {
conn, err := util.NewDBConn(cfg)
if err != nil {
return nil, errors.Wrap(err, "creating DB connection")
}
db := &sqlDatabase{
conn: conn,
ctx: ctx,
cfg: cfg,
}
if err := db.migrateDB(); err != nil {
return nil, errors.Wrap(err, "migrating database")
}
return db, nil
}
type sqlDatabase struct {
conn *gorm.DB
ctx context.Context
cfg config.Database
}
func (s *sqlDatabase) migrateDB() error {
if err := s.conn.AutoMigrate(
&Tag{},
// &Runner{},
&Pool{},
&Repository{},
&Organization{},
); err != nil {
return err
}
return nil
}
func (s *sqlDatabase) sqlToCommonTags(tag Tag) params.Tag {
return params.Tag{
ID: tag.ID.String(),
Name: tag.Name,
}
}
// func (s *sqlDatabase) sqlToCommonRunner(runner Runner) params.Runner {
// ret := params.Runner{
// ID: runner.ID.String(),
// MaxRunners: runner.MaxRunners,
// MinIdleRunners: runner.MinIdleRunners,
// Image: runner.Image,
// Flavor: runner.Flavor,
// OSArch: runner.OSArch,
// OSType: runner.OSType,
// Tags: make([]params.Tag, len(runner.Tags)),
// }
// for idx, val := range runner.Tags {
// ret.Tags[idx] = s.sqlToCommonTags(val)
// }
// return ret
// }
func (s *sqlDatabase) sqlToCommonPool(pool Pool) params.Pool {
ret := params.Pool{
ID: pool.ID.String(),
ProviderName: pool.ProviderName,
MaxRunners: pool.MaxRunners,
MinIdleRunners: pool.MinIdleRunners,
Image: pool.Image,
Flavor: pool.Flavor,
OSArch: pool.OSArch,
OSType: pool.OSType,
Tags: make([]params.Tag, len(pool.Tags)),
}
for idx, val := range pool.Tags {
ret.Tags[idx] = s.sqlToCommonTags(val)
}
return ret
}
func (s *sqlDatabase) sqlToCommonRepository(repo Repository) params.Repository {
ret := params.Repository{
ID: repo.ID.String(),
Name: repo.Name,
Owner: repo.Owner,
Pools: make([]params.Pool, len(repo.Pools)),
}
for idx, pool := range repo.Pools {
ret.Pools[idx] = s.sqlToCommonPool(pool)
}
return ret
}
func (s *sqlDatabase) sqlToCommonOrganization(org Organization) params.Organization {
ret := params.Organization{
ID: org.ID.String(),
Name: org.Name,
Pools: make([]params.Pool, len(org.Pools)),
}
return ret
}
func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, webhookSecret string) (params.Repository, error) {
secret := []byte{}
var err error
if webhookSecret != "" {
secret, err = util.Aes256EncodeString(webhookSecret, s.cfg.Passphrase)
if err != nil {
return params.Repository{}, fmt.Errorf("failed to encrypt string")
}
}
newRepo := Repository{
Name: name,
Owner: owner,
WebhookSecret: secret,
}
q := s.conn.Create(&newRepo)
if q.Error != nil {
return params.Repository{}, errors.Wrap(q.Error, "creating repository")
}
param := s.sqlToCommonRepository(newRepo)
param.WebhookSecret = webhookSecret
return param, nil
}
func (s *sqlDatabase) getRepo(ctx context.Context, id string) (Repository, error) {
u := uuid.Parse(id)
if u == nil {
return Repository{}, errors.Wrap(runnerErrors.NewBadRequestError(""), "parsing id")
}
var repo Repository
q := s.conn.Preload(clause.Associations).Where("id = ?", u).First(&repo)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {
return Repository{}, runnerErrors.ErrNotFound
}
return Repository{}, errors.Wrap(q.Error, "fetching repository from database")
}
return repo, nil
}
func (s *sqlDatabase) GetRepository(ctx context.Context, id string) (params.Repository, error) {
repo, err := s.getRepo(ctx, id)
if err != nil {
return params.Repository{}, errors.Wrap(err, "fetching repo")
}
param := s.sqlToCommonRepository(repo)
secret, err := util.Aes256DecodeString(repo.WebhookSecret, s.cfg.Passphrase)
if err != nil {
return params.Repository{}, errors.Wrap(err, "decrypting secret")
}
param.WebhookSecret = secret
return param, nil
}
func (s *sqlDatabase) ListRepositories(ctx context.Context) ([]params.Repository, error) {
var repos []Repository
q := s.conn.Find(&repos)
if q.Error != nil {
return []params.Repository{}, errors.Wrap(q.Error, "fetching user from database")
}
ret := make([]params.Repository, len(repos))
for idx, val := range repos {
ret[idx] = s.sqlToCommonRepository(val)
}
return ret, nil
}
func (s *sqlDatabase) DeleteRepository(ctx context.Context, id string) error {
repo, err := s.getRepo(ctx, id)
if err != nil {
if err == runnerErrors.ErrNotFound {
return nil
}
return errors.Wrap(err, "fetching repo")
}
q := s.conn.Delete(&repo)
if q.Error != nil && !errors.Is(q.Error, gorm.ErrRecordNotFound) {
return errors.Wrap(q.Error, "deleting repo")
}
return nil
}
func (s *sqlDatabase) CreateOrganization(ctx context.Context, name, webhookSecret string) (params.Organization, error) {
secret := []byte{}
var err error
if webhookSecret != "" {
secret, err = util.Aes256EncodeString(webhookSecret, s.cfg.Passphrase)
if err != nil {
return params.Organization{}, fmt.Errorf("failed to encrypt string")
}
}
newOrg := Organization{
Name: name,
WebhookSecret: secret,
}
q := s.conn.Create(&newOrg)
if q.Error != nil {
return params.Organization{}, errors.Wrap(q.Error, "creating org")
}
param := s.sqlToCommonOrganization(newOrg)
param.WebhookSecret = webhookSecret
return param, nil
}
func (s *sqlDatabase) getOrg(ctx context.Context, id string) (Organization, error) {
u := uuid.Parse(id)
if u == nil {
return Organization{}, errors.Wrap(runnerErrors.NewBadRequestError(""), "parsing id")
}
var org Organization
q := s.conn.Preload(clause.Associations).Where("id = ?", u).First(&org)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {
return Organization{}, runnerErrors.ErrNotFound
}
return Organization{}, errors.Wrap(q.Error, "fetching org from database")
}
return org, nil
}
func (s *sqlDatabase) GetOrganization(ctx context.Context, id string) (params.Organization, error) {
org, err := s.getOrg(ctx, id)
if err != nil {
return params.Organization{}, errors.Wrap(err, "fetching repo")
}
param := s.sqlToCommonOrganization(org)
secret, err := util.Aes256DecodeString(org.WebhookSecret, s.cfg.Passphrase)
if err != nil {
return params.Organization{}, errors.Wrap(err, "decrypting secret")
}
param.WebhookSecret = secret
return param, nil
}
func (s *sqlDatabase) ListOrganizations(ctx context.Context) ([]params.Organization, error) {
var orgs []Organization
q := s.conn.Find(&orgs)
if q.Error != nil {
return []params.Organization{}, errors.Wrap(q.Error, "fetching user from database")
}
ret := make([]params.Organization, len(orgs))
for idx, val := range orgs {
ret[idx] = s.sqlToCommonOrganization(val)
}
return ret, nil
}
func (s *sqlDatabase) DeleteOrganization(ctx context.Context, id string) error {
org, err := s.getOrg(ctx, id)
if err != nil {
if err == runnerErrors.ErrNotFound {
return nil
}
return errors.Wrap(err, "fetching repo")
}
q := s.conn.Delete(&org)
if q.Error != nil && !errors.Is(q.Error, gorm.ErrRecordNotFound) {
return errors.Wrap(q.Error, "deleting org")
}
return nil
}
func (s *sqlDatabase) getOrCreateTag(tagName string) (Tag, error) {
var tag Tag
q := s.conn.Where("name = ?", tagName).First(&tag)
if q.Error == nil {
return tag, nil
}
if !errors.Is(q.Error, gorm.ErrRecordNotFound) {
return Tag{}, errors.Wrap(q.Error, "fetching tag from database")
}
newTag := Tag{
Name: tagName,
}
q = s.conn.Create(&newTag)
if q.Error != nil {
return Tag{}, errors.Wrap(q.Error, "creating tag")
}
return newTag, nil
}
func (s *sqlDatabase) CreateRepositoryPool(ctx context.Context, repoId string, param params.CreatePoolParams) (params.Pool, error) {
if len(param.Tags) == 0 {
return params.Pool{}, runnerErrors.NewBadRequestError("no tags specified")
}
repo, err := s.getRepo(ctx, repoId)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching repo")
}
newPool := Pool{
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
MinIdleRunners: param.MinIdleRunners,
Image: param.Image,
Flavor: param.Flavor,
OSType: param.OSType,
OSArch: param.OSArch,
}
tags := make([]Tag, len(param.Tags))
for idx, val := range param.Tags {
t, err := s.getOrCreateTag(val)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching tag")
}
tags[idx] = t
}
err = s.conn.Model(&repo).Association("Pools").Append(&newPool)
if err != nil {
return params.Pool{}, errors.Wrap(err, "adding pool")
}
return s.sqlToCommonPool(newPool), nil
}
func (s *sqlDatabase) CreateOrganizationPool(ctx context.Context, orgId string, param params.CreatePoolParams) (params.Pool, error) {
if len(param.Tags) == 0 {
return params.Pool{}, runnerErrors.NewBadRequestError("no tags specified")
}
org, err := s.getOrg(ctx, orgId)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching org")
}
newPool := Pool{
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
MinIdleRunners: param.MinIdleRunners,
Image: param.Image,
Flavor: param.Flavor,
OSType: param.OSType,
OSArch: param.OSArch,
}
tags := make([]Tag, len(param.Tags))
for idx, val := range param.Tags {
t, err := s.getOrCreateTag(val)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching tag")
}
tags[idx] = t
}
err = s.conn.Model(&org).Association("Pools").Append(&newPool)
if err != nil {
return params.Pool{}, errors.Wrap(err, "adding pool")
}
return s.sqlToCommonPool(newPool), nil
}
func (s *sqlDatabase) GetRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) {
repo, err := s.getRepo(ctx, repoID)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching repo")
}
u := uuid.Parse(poolID)
if u == nil {
return params.Pool{}, fmt.Errorf("invalid pool id")
}
var pool []Pool
err = s.conn.Model(&repo).Association("Pools").Find(&pool, "id = ?", u)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching pool")
}
if len(pool) == 0 {
return params.Pool{}, runnerErrors.ErrNotFound
}
return s.sqlToCommonPool(pool[0]), nil
}
func (s *sqlDatabase) GetOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) {
org, err := s.getOrg(ctx, orgID)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching org")
}
u := uuid.Parse(poolID)
if u == nil {
return params.Pool{}, fmt.Errorf("invalid pool id")
}
var pool []Pool
err = s.conn.Model(&org).Association("Pools").Find(&pool, "id = ?", u)
if err != nil {
return params.Pool{}, errors.Wrap(err, "fetching pool")
}
if len(pool) == 0 {
return params.Pool{}, runnerErrors.ErrNotFound
}
return s.sqlToCommonPool(pool[0]), nil
}
func (s *sqlDatabase) DeleteRepositoryPool(ctx context.Context, repoID, poolID string) error {
return nil
}
func (s *sqlDatabase) DeleteOrganizationPool(ctx context.Context, orgID, poolID string) error {
return nil
}
func (s *sqlDatabase) UpdateRepositoryPool(ctx context.Context, repoID, poolID string) (params.Pool, error) {
return params.Pool{}, nil
}
func (s *sqlDatabase) UpdateOrganizationPool(ctx context.Context, orgID, poolID string) (params.Pool, error) {
return params.Pool{}, nil
}

8
go.mod
View file

@ -10,24 +10,32 @@ require (
github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124
github.com/pborman/uuid v1.2.1
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gorm.io/driver/mysql v1.3.3
gorm.io/driver/sqlite v1.3.2
gorm.io/gorm v1.23.4
)
require (
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect
github.com/go-macaroon-bakery/macaroonpb v1.0.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/juju/webbrowser v1.0.0 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/mattn/go-sqlite3 v1.14.12 // indirect
github.com/pkg/sftp v1.13.4 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rogpeppe/fastuuid v1.2.0 // indirect

18
go.sum
View file

@ -22,6 +22,8 @@ github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebP
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/go-macaroon-bakery/macaroonpb v1.0.0 h1:It9exBaRMZ9iix1iJ6gwzfwsDE6ExNuwtAJ9e09v6XE=
github.com/go-macaroon-bakery/macaroonpb v1.0.0/go.mod h1:UzrGOcbiwTXISFP2XDLDPjfhMINZa+fX/7A2lMd31zc=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -60,6 +62,11 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/juju/mgotest v1.0.1/go.mod h1:vTaDufYul+Ps8D7bgseHjq87X8eu0ivlKLp9mVc/Bfc=
github.com/juju/postgrestest v1.1.0/go.mod h1:/n17Y2T6iFozzXwSCO0JYJ5gSiz2caEtSwAjh/uLXDM=
github.com/juju/qthttptest v0.0.1/go.mod h1://LCf/Ls22/rPw2u1yWukUJvYtfPY4nYpWUl2uZhryo=
@ -88,6 +95,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124 h1:EmjWCASxSUz+ymsEJfiWN3yx3yTypoKJrnOSSzAWYds=
github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124/go.mod h1:T4xjj62BmFg1L5JfY2wRyPZtKbBeTFgo/GLwV8DFZ8M=
github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
@ -104,6 +113,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -220,5 +231,12 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.3.3 h1:jXG9ANrwBc4+bMvBcSl8zCfPBaVoPyBEBshA8dA93X8=
gorm.io/driver/mysql v1.3.3/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
gorm.io/driver/sqlite v1.3.2 h1:nWTy4cE52K6nnMhv23wLmur9Y3qWbZvOBz+V4PrGAxg=
gorm.io/driver/sqlite v1.3.2/go.mod h1:B+8GyC9K7VgzJAcrcXMRPdnMcck+8FgJynEehEPM16U=
gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.4 h1:1BKWM67O6CflSLcwGQR7ccfmC4ebOxQrTfOQGRE9wjg=
gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

43
params/params.go
View file

@ -59,3 +59,46 @@ type BootstrapInstance struct {
Image string `json:"image"`
Labels []string `json:"labels"`
}
type Tag struct {
ID string `json:"id"`
Name string `json:"name"`
}
type Pool struct {
ID string `json:"id"`
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType config.OSType `json:"os_type"`
OSArch config.OSArch `json:"os_arch"`
Tags []Tag `json:"tags"`
}
type Repository struct {
ID string `json:"id"`
Owner string `json:"owner"`
Name string `json:"name"`
WebhookSecret string `json:"-"`
Pools []Pool `json:"pool,omitempty"`
}
type Organization struct {
ID string `json:"id"`
Name string `json:"name"`
WebhookSecret string `json:"-"`
Pools []Pool `json:"pool,omitempty"`
}
type CreatePoolParams struct {
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType config.OSType `json:"os_type"`
OSArch config.OSArch `json:"os_arch"`
Tags []string `json:"tags"`
}

49
runner/pool/repository.go
View file

@ -19,19 +19,20 @@ import (
// test that we implement PoolManager
var _ common.PoolManager = &Repository{}
func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, ghcli *github.Client, provider common.Provider) (common.PoolManager, error) {
func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, provider common.Provider, ghcli *github.Client, controllerID string) (common.PoolManager, error) {
queueSize := cfg.Pool.QueueSize
if queueSize == 0 {
queueSize = config.DefaultPoolQueueSize
}
repo := &Repository{
ctx: ctx,
cfg: cfg,
ghcli: ghcli,
provider: provider,
jobQueue: make(chan params.WorkflowJob, queueSize),
quit: make(chan struct{}),
done: make(chan struct{}),
ctx: ctx,
cfg: cfg,
ghcli: ghcli,
provider: provider,
controllerID: controllerID,
jobQueue: make(chan params.WorkflowJob, queueSize),
quit: make(chan struct{}),
done: make(chan struct{}),
}
if err := repo.fetchTools(); err != nil {
@ -41,19 +42,25 @@ func NewRepositoryRunnerPool(ctx context.Context, cfg config.Repository, ghcli *
}
type Repository struct {
ctx context.Context
cfg config.Repository
ghcli *github.Client
provider common.Provider
tools []*github.RunnerApplicationDownload
jobQueue chan params.WorkflowJob
quit chan struct{}
done chan struct{}
mux sync.Mutex
ctx context.Context
controllerID string
cfg config.Repository
ghcli *github.Client
provider common.Provider
tools []*github.RunnerApplicationDownload
jobQueue chan params.WorkflowJob
quit chan struct{}
done chan struct{}
mux sync.Mutex
}
func (r *Repository) getGithubRunners() ([]github.Runner, error) {
return nil, nil
func (r *Repository) getGithubRunners() ([]*github.Runner, error) {
runners, _, err := r.ghcli.Actions.ListRunners(r.ctx, r.cfg.Owner, r.cfg.Name, nil)
if err != nil {
return nil, errors.Wrap(err, "fetching runners")
}
return runners.Runners, nil
}
func (r *Repository) getProviderInstances() ([]params.Instance, error) {
@ -92,7 +99,7 @@ func (r *Repository) Wait() error {
func (r *Repository) loop() {
defer close(r.done)
// TODO: Consolidate runners on loop start. Local runners must match runners
// TODO: Consolidate runners on loop start. Provider runners must match runners
// in github and DB. When a Workflow job is received, we will first create/update
// an entity in the database, before sending the request to the provider to create/delete
// an instance. If a "queued" job is received, we create an entity in the db with
@ -116,6 +123,8 @@ func (r *Repository) loop() {
// Create instance.
case "completed":
// Remove instance.
case "in_progress":
// update state
}
fmt.Println(job)
case <-time.After(3 * time.Hour):

96
util/util.go
View file

@ -3,6 +3,8 @@ package util
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
@ -20,6 +22,9 @@ import (
"golang.org/x/crypto/ssh"
"golang.org/x/oauth2"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
"gorm.io/driver/mysql"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"runner-manager/cloudconfig"
"runner-manager/config"
@ -27,6 +32,8 @@ import (
"runner-manager/params"
)
const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
var (
OSToOSTypeMap map[string]config.OSType = map[string]config.OSType{
"ubuntu": config.Linux,
@ -158,3 +165,92 @@ func GetCloudConfig(bootstrapParams params.BootstrapInstance, tools github.Runne
}
return asStr, nil
}
// NewDBConn returns a new gorm db connection, given the config
func NewDBConn(dbCfg config.Database) (conn *gorm.DB, err error) {
dbType, connURI, err := dbCfg.GormParams()
if err != nil {
return nil, errors.Wrap(err, "getting DB URI string")
}
switch dbType {
case config.MySQLBackend:
conn, err = gorm.Open(mysql.Open(connURI), &gorm.Config{})
case config.SQLiteBackend:
conn, err = gorm.Open(sqlite.Open(connURI), &gorm.Config{})
}
if err != nil {
return nil, errors.Wrap(err, "connecting to database")
}
if dbCfg.Debug {
conn = conn.Debug()
}
return conn, nil
}
// GetRandomString returns a secure random string
func GetRandomString(n int) (string, error) {
data := make([]byte, n)
_, err := rand.Read(data)
if err != nil {
return "", errors.Wrap(err, "getting random data")
}
for i, b := range data {
data[i] = alphanumeric[b%byte(len(alphanumeric))]
}
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}