DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Use database for github creds

Browse source 
Add database models that deal with github credentials. This change
adds models for github endpoints (github.com, GHES, etc). This change
also adds code to migrate config credntials to the DB.

Tests need to be fixed and new tests need to be written. This will come
in a later commit.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2024-04-15 08:32:19 +00:00
parent 834c3bb798
commit 90870c11be
22 changed files with 1312 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

12
database/sql/enterprise.go
View file

@ -54,7 +54,7 @@ func (s *sqlDatabase) GetEnterprise(ctx context.Context, name string) (params.En
}
func (s *sqlDatabase) GetEnterpriseByID(ctx context.Context, enterpriseID string) (params.Enterprise, error) {
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID, "Pools")
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID, "Pools", "Credentials", "Endpoint")
if err != nil {
return params.Enterprise{}, errors.Wrap(err, "fetching enterprise")
}
@ -68,7 +68,7 @@ func (s *sqlDatabase) GetEnterpriseByID(ctx context.Context, enterpriseID string
func (s *sqlDatabase) ListEnterprises(_ context.Context) ([]params.Enterprise, error) {
var enterprises []Enterprise
q := s.conn.Find(&enterprises)
q := s.conn.Preload("Credentials").Find(&enterprises)
if q.Error != nil {
return []params.Enterprise{}, errors.Wrap(q.Error, "fetching enterprises")
}
@ -100,7 +100,7 @@ func (s *sqlDatabase) DeleteEnterprise(ctx context.Context, enterpriseID string)
}
func (s *sqlDatabase) UpdateEnterprise(ctx context.Context, enterpriseID string, param params.UpdateEntityParams) (params.Enterprise, error) {
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID)
enterprise, err := s.getEnterpriseByID(ctx, enterpriseID, "Credentials", "Endpoint")
if err != nil {
return params.Enterprise{}, errors.Wrap(err, "fetching enterprise")
}
@ -136,8 +136,10 @@ func (s *sqlDatabase) UpdateEnterprise(ctx context.Context, enterpriseID string,
func (s *sqlDatabase) getEnterprise(_ context.Context, name string) (Enterprise, error) {
var enterprise Enterprise
q := s.conn.Where("name = ? COLLATE NOCASE", name)
q = q.First(&enterprise)
q := s.conn.Where("name = ? COLLATE NOCASE", name).
Preload("Credentials").
Preload("Endpoint").
First(&enterprise)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {
return Enterprise{}, runnerErrors.ErrNotFound

4
database/sql/enterprise_test.go
View file

@ -173,7 +173,7 @@ func (s *EnterpriseTestSuite) TestCreateEnterprise() {
s.FailNow(fmt.Sprintf("failed to get enterprise by id: %v", err))
}
s.Require().Equal(storeEnterprise.Name, enterprise.Name)
s.Require().Equal(storeEnterprise.CredentialsName, enterprise.CredentialsName)
s.Require().Equal(storeEnterprise.Credentials.Name, enterprise.Credentials.Name)
s.Require().Equal(storeEnterprise.WebhookSecret, enterprise.WebhookSecret)
}
@ -313,7 +313,7 @@ func (s *EnterpriseTestSuite) TestUpdateEnterprise() {
enterprise, err := s.Store.UpdateEnterprise(context.Background(), s.Fixtures.Enterprises[0].ID, s.Fixtures.UpdateRepoParams)
s.Require().Nil(err)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, enterprise.CredentialsName)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, enterprise.Credentials.Name)
s.Require().Equal(s.Fixtures.UpdateRepoParams.WebhookSecret, enterprise.WebhookSecret)
}

473
database/sql/github.go Normal file
View file

@ -0,0 +1,473 @@
package sql
import (
"context"
"github.com/google/uuid"
"github.com/pkg/errors"
"gorm.io/gorm"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) sqlToCommonGithubCredentials(creds GithubCredentials) (params.GithubCredentials, error) {
data, err := util.Unseal(creds.Payload, []byte(s.cfg.Passphrase))
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "unsealing credentials")
}
commonCreds := params.GithubCredentials{
ID: creds.ID,
Name: creds.Name,
Description: creds.Description,
APIBaseURL: creds.Endpoint.APIBaseURL,
BaseURL: creds.Endpoint.BaseURL,
UploadBaseURL: creds.Endpoint.UploadBaseURL,
CABundle: creds.Endpoint.CACertBundle,
AuthType: creds.AuthType,
Endpoint: creds.Endpoint.Name,
CredentialsPayload: data,
}
for _, repo := range creds.Repositories {
commonRepo, err := s.sqlToCommonRepository(repo)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "converting github repository")
}
commonCreds.Repositories = append(commonCreds.Repositories, commonRepo)
}
for _, org := range creds.Organizations {
commonOrg, err := s.sqlToCommonOrganization(org)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "converting github organization")
}
commonCreds.Organizations = append(commonCreds.Organizations, commonOrg)
}
for _, ent := range creds.Enterprises {
commonEnt, err := s.sqlToCommonEnterprise(ent)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "converting github enterprise")
}
commonCreds.Enterprises = append(commonCreds.Enterprises, commonEnt)
}
return commonCreds, nil
}
func (s *sqlDatabase) sqlToCommonGithubEndpoint(ep GithubEndpoint) (params.GithubEndpoint, error) {
return params.GithubEndpoint{
Name: ep.Name,
Description: ep.Description,
APIBaseURL: ep.APIBaseURL,
BaseURL: ep.BaseURL,
UploadBaseURL: ep.UploadBaseURL,
CACertBundle: ep.CACertBundle,
}, nil
}
func getUIDFromContext(ctx context.Context) (uuid.UUID, error) {
userID := auth.UserID(ctx)
if userID == "" {
return uuid.Nil, errors.Wrap(runnerErrors.ErrUnauthorized, "creating github endpoint")
}
asUUID, err := uuid.Parse(userID)
if err != nil {
return uuid.Nil, errors.Wrap(runnerErrors.ErrUnauthorized, "creating github endpoint")
}
return asUUID, nil
}
func (s *sqlDatabase) CreateGithubEndpoint(_ context.Context, param params.CreateGithubEndpointParams) (params.GithubEndpoint, error) {
var endpoint GithubEndpoint
err := s.conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("name = ?", param.Name).First(&endpoint).Error; err == nil {
return errors.Wrap(runnerErrors.ErrDuplicateEntity, "github endpoint already exists")
}
endpoint = GithubEndpoint{
Name: param.Name,
Description: param.Description,
APIBaseURL: param.APIBaseURL,
BaseURL: param.BaseURL,
UploadBaseURL: param.UploadBaseURL,
CACertBundle: param.CACertBundle,
}
if err := tx.Create(&endpoint).Error; err != nil {
return errors.Wrap(err, "creating github endpoint")
}
return nil
})
if err != nil {
return params.GithubEndpoint{}, errors.Wrap(err, "creating github endpoint")
}
return s.sqlToCommonGithubEndpoint(endpoint)
}
func (s *sqlDatabase) ListGithubEndpoints(_ context.Context) ([]params.GithubEndpoint, error) {
var endpoints []GithubEndpoint
err := s.conn.Find(&endpoints).Error
if err != nil {
return nil, errors.Wrap(err, "fetching github endpoints")
}
var ret []params.GithubEndpoint
for _, ep := range endpoints {
commonEp, err := s.sqlToCommonGithubEndpoint(ep)
if err != nil {
return nil, errors.Wrap(err, "converting github endpoint")
}
ret = append(ret, commonEp)
}
return ret, nil
}
func (s *sqlDatabase) UpdateGithubEndpoint(_ context.Context, name string, param params.UpdateGithubEndpointParams) (params.GithubEndpoint, error) {
var endpoint GithubEndpoint
err := s.conn.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("name = ?", name).First(&endpoint).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errors.Wrap(runnerErrors.ErrNotFound, "github endpoint not found")
}
return errors.Wrap(err, "fetching github endpoint")
}
if param.APIBaseURL != nil {
endpoint.APIBaseURL = *param.APIBaseURL
}
if param.BaseURL != nil {
endpoint.BaseURL = *param.BaseURL
}
if param.UploadBaseURL != nil {
endpoint.UploadBaseURL = *param.UploadBaseURL
}
if param.CACertBundle != nil {
endpoint.CACertBundle = param.CACertBundle
}
if param.Description != nil {
endpoint.Description = *param.Description
}
if err := tx.Save(&endpoint).Error; err != nil {
return errors.Wrap(err, "updating github endpoint")
}
return nil
})
if err != nil {
return params.GithubEndpoint{}, errors.Wrap(err, "updating github endpoint")
}
return s.sqlToCommonGithubEndpoint(endpoint)
}
func (s *sqlDatabase) GetGithubEndpoint(_ context.Context, name string) (params.GithubEndpoint, error) {
var endpoint GithubEndpoint
err := s.conn.Where("name = ?", name).First(&endpoint).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return params.GithubEndpoint{}, errors.Wrap(err, "github endpoint not found")
}
return params.GithubEndpoint{}, errors.Wrap(err, "fetching github endpoint")
}
return s.sqlToCommonGithubEndpoint(endpoint)
}
func (s *sqlDatabase) DeleteGithubEndpoint(_ context.Context, name string) error {
err := s.conn.Transaction(func(tx *gorm.DB) error {
var endpoint GithubEndpoint
if err := tx.Where("name = ?", name).First(&endpoint).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return errors.Wrap(err, "fetching github endpoint")
}
var credsCount int64
if err := tx.Model(&GithubCredentials{}).Where("endpoint_name = ?", endpoint.Name).Count(&credsCount).Error; err != nil {
if !errors.Is(err, gorm.ErrRecordNotFound) {
return errors.Wrap(err, "fetching github credentials")
}
}
if credsCount > 0 {
return errors.New("cannot delete endpoint with credentials")
}
if err := tx.Unscoped().Delete(&endpoint).Error; err != nil {
return errors.Wrap(err, "deleting github endpoint")
}
return nil
})
if err != nil {
return errors.Wrap(err, "deleting github endpoint")
}
return nil
}
func (s *sqlDatabase) CreateGithubCredentials(ctx context.Context, endpointName string, param params.CreateGithubCredentialsParams) (params.GithubCredentials, error) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "creating github credentials")
}
var creds GithubCredentials
err = s.conn.Transaction(func(tx *gorm.DB) error {
var endpoint GithubEndpoint
if err := tx.Where("name = ?", endpointName).First(&endpoint).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errors.Wrap(runnerErrors.ErrNotFound, "github endpoint not found")
}
return errors.Wrap(err, "fetching github endpoint")
}
if err := tx.Where("name = ?", param.Name).First(&creds).Error; err == nil {
return errors.New("github credentials already exists")
}
var data []byte
var err error
switch param.AuthType {
case params.GithubAuthTypePAT:
data, err = s.marshalAndSeal(param.PAT)
case params.GithubAuthTypeApp:
data, err = s.marshalAndSeal(param.App)
}
if err != nil {
return errors.Wrap(err, "marshaling and sealing credentials")
}
creds = GithubCredentials{
Name: param.Name,
Description: param.Description,
EndpointName: &endpoint.Name,
AuthType: param.AuthType,
Payload: data,
UserID: &userID,
}
if err := tx.Create(&creds).Error; err != nil {
return errors.Wrap(err, "creating github credentials")
}
// Skip making an extra query.
creds.Endpoint = endpoint
return nil
})
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "creating github credentials")
}
return s.sqlToCommonGithubCredentials(creds)
}
func (s *sqlDatabase) getGithubCredentialsByName(ctx context.Context, tx *gorm.DB, name string, detailed bool) (GithubCredentials, error) {
var creds GithubCredentials
q := tx.Preload("Endpoint")
if detailed {
q = q.
Preload("Repositories").
Preload("Organizations").
Preload("Enterprises")
}
if !auth.IsAdmin(ctx) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return GithubCredentials{}, errors.Wrap(err, "fetching github credentials")
}
q = q.Where("user_id = ?", userID)
}
err := q.Where("name = ?", name).First(&creds).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return GithubCredentials{}, errors.Wrap(runnerErrors.ErrNotFound, "github credentials not found")
}
return GithubCredentials{}, errors.Wrap(err, "fetching github credentials")
}
return creds, nil
}
func (s *sqlDatabase) GetGithubCredentialsByName(ctx context.Context, name string, detailed bool) (params.GithubCredentials, error) {
creds, err := s.getGithubCredentialsByName(ctx, s.conn, name, detailed)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "fetching github credentials")
}
return s.sqlToCommonGithubCredentials(creds)
}
func (s *sqlDatabase) GetGithubCredentials(ctx context.Context, id uint, detailed bool) (params.GithubCredentials, error) {
var creds GithubCredentials
q := s.conn.Preload("Endpoint")
if detailed {
q = q.
Preload("Repositories").
Preload("Organizations").
Preload("Enterprises")
}
if !auth.IsAdmin(ctx) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "fetching github credentials")
}
q = q.Where("user_id = ?", userID)
}
err := q.Where("id = ?", id).First(&creds).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return params.GithubCredentials{}, errors.Wrap(runnerErrors.ErrNotFound, "github credentials not found")
}
return params.GithubCredentials{}, errors.Wrap(err, "fetching github credentials")
}
return s.sqlToCommonGithubCredentials(creds)
}
func (s *sqlDatabase) ListGithubCredentials(ctx context.Context) ([]params.GithubCredentials, error) {
q := s.conn.Preload("Endpoint")
if !auth.IsAdmin(ctx) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return nil, errors.Wrap(err, "fetching github credentials")
}
q = q.Where("user_id = ?", userID)
}
var creds []GithubCredentials
err := q.Preload("Endpoint").Find(&creds).Error
if err != nil {
return nil, errors.Wrap(err, "fetching github credentials")
}
var ret []params.GithubCredentials
for _, c := range creds {
commonCreds, err := s.sqlToCommonGithubCredentials(c)
if err != nil {
return nil, errors.Wrap(err, "converting github credentials")
}
ret = append(ret, commonCreds)
}
return ret, nil
}
func (s *sqlDatabase) UpdateGithubCredentials(ctx context.Context, id uint, param params.UpdateGithubCredentialsParams) (params.GithubCredentials, error) {
var creds GithubCredentials
err := s.conn.Transaction(func(tx *gorm.DB) error {
q := tx.Preload("Endpoint")
if !auth.IsAdmin(ctx) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return errors.Wrap(err, "updating github credentials")
}
q = q.Where("user_id = ?", userID)
}
if err := q.Where("id = ?", id).First(&creds).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errors.Wrap(runnerErrors.ErrNotFound, "github credentials not found")
}
return errors.Wrap(err, "fetching github credentials")
}
if param.Name != nil {
creds.Name = *param.Name
}
if param.Description != nil {
creds.Description = *param.Description
}
var data []byte
var err error
switch creds.AuthType {
case params.GithubAuthTypePAT:
if param.PAT != nil {
data, err = s.marshalAndSeal(param.PAT)
}
if param.App != nil {
return errors.New("cannot update app credentials for PAT")
}
case params.GithubAuthTypeApp:
if param.App != nil {
data, err = s.marshalAndSeal(param.App)
}
if param.PAT != nil {
return errors.New("cannot update PAT credentials for app")
}
}
if err != nil {
return errors.Wrap(err, "marshaling and sealing credentials")
}
if len(data) > 0 {
creds.Payload = data
}
if err := tx.Save(&creds).Error; err != nil {
return errors.Wrap(err, "updating github credentials")
}
return nil
})
if err != nil {
return params.GithubCredentials{}, errors.Wrap(err, "updating github credentials")
}
return s.sqlToCommonGithubCredentials(creds)
}
func (s *sqlDatabase) DeleteGithubCredentials(ctx context.Context, id uint) error {
err := s.conn.Transaction(func(tx *gorm.DB) error {
q := tx.Where("id = ?", id).
Preload("Repositories").
Preload("Organizations").
Preload("Enterprises")
if !auth.IsAdmin(ctx) {
userID, err := getUIDFromContext(ctx)
if err != nil {
return errors.Wrap(err, "deleting github credentials")
}
q = q.Where("user_id = ?", userID)
}
var creds GithubCredentials
err := q.First(&creds).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return errors.Wrap(err, "fetching github credentials")
}
if len(creds.Repositories) > 0 {
return errors.New("cannot delete credentials with repositories")
}
if len(creds.Organizations) > 0 {
return errors.New("cannot delete credentials with organizations")
}
if len(creds.Enterprises) > 0 {
return errors.New("cannot delete credentials with enterprises")
}
if err := tx.Unscoped().Delete(&creds).Error; err != nil {
return errors.Wrap(err, "deleting github credentials")
}
return nil
})
if err != nil {
return errors.Wrap(err, "deleting github credentials")
}
return nil
}

59
database/sql/models.go
View file

@ -89,35 +89,56 @@ type Pool struct {
type Repository struct {
Base
CredentialsName string
CredentialsName string
CredentialsID *uint `gorm:"index"`
Credentials GithubCredentials `gorm:"foreignKey:CredentialsID;constraint:OnDelete:SET NULL"`
Owner string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
Name string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:RepoID"`
Jobs []WorkflowJob `gorm:"foreignKey:RepoID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
EndpointName *string `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
}
type Organization struct {
Base
CredentialsName string
CredentialsName string
CredentialsID *uint `gorm:"index"`
Credentials GithubCredentials `gorm:"foreignKey:CredentialsID;constraint:OnDelete:SET NULL"`
Name string `gorm:"index:idx_org_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:OrgID"`
Jobs []WorkflowJob `gorm:"foreignKey:OrgID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
EndpointName *string `gorm:"index"`
Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
}
type Enterprise struct {
Base
CredentialsName string
CredentialsName string
CredentialsID *uint `gorm:"index"`
Credentials GithubCredentials `gorm:"foreignKey:CredentialsID;constraint:OnDelete:SET NULL"`
Name string `gorm:"index:idx_ent_name_nocase,collate:nocase"`
WebhookSecret []byte
Pools []Pool `gorm:"foreignKey:EnterpriseID"`
Jobs []WorkflowJob `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:SET NULL"`
PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
EndpointName *string `gorm:"index"`
Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
}
type Address struct {
@ -246,3 +267,35 @@ type WorkflowJob struct {
UpdatedAt time.Time
DeletedAt gorm.DeletedAt `gorm:"index"`
}
type GithubEndpoint struct {
Name string `gorm:"type:varchar(64) collate nocase;primary_key;"`
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt `gorm:"index"`
Description string `gorm:"type:text"`
APIBaseURL string `gorm:"type:text collate nocase"`
UploadBaseURL string `gorm:"type:text collate nocase"`
BaseURL string `gorm:"type:text collate nocase"`
CACertBundle []byte `gorm:"type:longblob"`
}
type GithubCredentials struct {
gorm.Model
Name string `gorm:"index:idx_github_credentials,unique;type:varchar(64) collate nocase"`
UserID *uuid.UUID `gorm:"index:idx_github_credentials,unique"`
User User `gorm:"foreignKey:UserID"`
Description string `gorm:"type:text"`
AuthType params.GithubAuthType `gorm:"index"`
Payload []byte `gorm:"type:longblob"`
Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName"`
EndpointName *string `gorm:"index"`
Repositories []Repository `gorm:"foreignKey:CredentialsID"`
Organizations []Organization `gorm:"foreignKey:CredentialsID"`
Enterprises []Enterprise `gorm:"foreignKey:CredentialsID"`
}

12
database/sql/organizations.go
View file

@ -72,7 +72,7 @@ func (s *sqlDatabase) GetOrganization(ctx context.Context, name string) (params.
func (s *sqlDatabase) ListOrganizations(_ context.Context) ([]params.Organization, error) {
var orgs []Organization
q := s.conn.Find(&orgs)
q := s.conn.Preload("Credentials").Find(&orgs)
if q.Error != nil {
return []params.Organization{}, errors.Wrap(q.Error, "fetching org from database")
}
@ -104,7 +104,7 @@ func (s *sqlDatabase) DeleteOrganization(ctx context.Context, orgID string) erro
}
func (s *sqlDatabase) UpdateOrganization(ctx context.Context, orgID string, param params.UpdateEntityParams) (params.Organization, error) {
org, err := s.getOrgByID(ctx, orgID)
org, err := s.getOrgByID(ctx, orgID, "Credentials", "Endpoint")
if err != nil {
return params.Organization{}, errors.Wrap(err, "fetching org")
}
@ -138,7 +138,7 @@ func (s *sqlDatabase) UpdateOrganization(ctx context.Context, orgID string, para
}
func (s *sqlDatabase) GetOrganizationByID(ctx context.Context, orgID string) (params.Organization, error) {
org, err := s.getOrgByID(ctx, orgID, "Pools")
org, err := s.getOrgByID(ctx, orgID, "Pools", "Credentials", "Endpoint")
if err != nil {
return params.Organization{}, errors.Wrap(err, "fetching org")
}
@ -177,8 +177,10 @@ func (s *sqlDatabase) getOrgByID(_ context.Context, id string, preload ...string
func (s *sqlDatabase) getOrg(_ context.Context, name string) (Organization, error) {
var org Organization
q := s.conn.Where("name = ? COLLATE NOCASE", name)
q = q.First(&org)
q := s.conn.Where("name = ? COLLATE NOCASE", name).
Preload("Credentials").
Preload("Endpoint").
First(&org)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {
return Organization{}, runnerErrors.ErrNotFound

4
database/sql/organizations_test.go
View file

@ -173,7 +173,7 @@ func (s *OrgTestSuite) TestCreateOrganization() {
s.FailNow(fmt.Sprintf("failed to get organization by id: %v", err))
}
s.Require().Equal(storeOrg.Name, org.Name)
s.Require().Equal(storeOrg.CredentialsName, org.CredentialsName)
s.Require().Equal(storeOrg.Credentials.Name, org.Credentials.Name)
s.Require().Equal(storeOrg.WebhookSecret, org.WebhookSecret)
}
@ -313,7 +313,7 @@ func (s *OrgTestSuite) TestUpdateOrganization() {
org, err := s.Store.UpdateOrganization(context.Background(), s.Fixtures.Orgs[0].ID, s.Fixtures.UpdateRepoParams)
s.Require().Nil(err)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, org.CredentialsName)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, org.Credentials.Name)
s.Require().Equal(s.Fixtures.UpdateRepoParams.WebhookSecret, org.WebhookSecret)
}

42
database/sql/repositories.go
View file

@ -27,7 +27,7 @@ import (
"github.com/cloudbase/garm/params"
)
func (s *sqlDatabase) CreateRepository(_ context.Context, owner, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Repository, error) {
func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name, credentialsName, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Repository, error) {
if webhookSecret == "" {
return params.Repository{}, errors.New("creating repo: missing secret")
}
@ -35,17 +35,29 @@ func (s *sqlDatabase) CreateRepository(_ context.Context, owner, name, credentia
if err != nil {
return params.Repository{}, fmt.Errorf("failed to encrypt string")
}
newRepo := Repository{
Name: name,
Owner: owner,
WebhookSecret: secret,
CredentialsName: credentialsName,
PoolBalancerType: poolBalancerType,
}
q := s.conn.Create(&newRepo)
if q.Error != nil {
return params.Repository{}, errors.Wrap(q.Error, "creating repository")
var newRepo Repository
err = s.conn.Transaction(func(tx *gorm.DB) error {
creds, err := s.getGithubCredentialsByName(ctx, tx, credentialsName, false)
if err != nil {
return errors.Wrap(err, "creating repository")
}
newRepo.Name = name
newRepo.Owner = owner
newRepo.WebhookSecret = secret
newRepo.CredentialsID = &creds.ID
newRepo.PoolBalancerType = poolBalancerType
q := tx.Create(&newRepo)
if q.Error != nil {
return errors.Wrap(q.Error, "creating repository")
}
return nil
})
if err != nil {
return params.Repository{}, errors.Wrap(err, "creating repository")
}
param, err := s.sqlToCommonRepository(newRepo)
@ -72,7 +84,7 @@ func (s *sqlDatabase) GetRepository(ctx context.Context, owner, name string) (pa
func (s *sqlDatabase) ListRepositories(_ context.Context) ([]params.Repository, error) {
var repos []Repository
q := s.conn.Find(&repos)
q := s.conn.Preload("Credentials").Find(&repos)
if q.Error != nil {
return []params.Repository{}, errors.Wrap(q.Error, "fetching user from database")
}
@ -104,7 +116,7 @@ func (s *sqlDatabase) DeleteRepository(ctx context.Context, repoID string) error
}
func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param params.UpdateEntityParams) (params.Repository, error) {
repo, err := s.getRepoByID(ctx, repoID)
repo, err := s.getRepoByID(ctx, repoID, "Credentials", "Endpoint")
if err != nil {
return params.Repository{}, errors.Wrap(err, "fetching repo")
}
@ -138,7 +150,7 @@ func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param
}
func (s *sqlDatabase) GetRepositoryByID(ctx context.Context, repoID string) (params.Repository, error) {
repo, err := s.getRepoByID(ctx, repoID, "Pools")
repo, err := s.getRepoByID(ctx, repoID, "Pools", "Credentials", "Endpoint")
if err != nil {
return params.Repository{}, errors.Wrap(err, "fetching repo")
}
@ -154,6 +166,8 @@ func (s *sqlDatabase) getRepo(_ context.Context, owner, name string) (Repository
var repo Repository
q := s.conn.Where("name = ? COLLATE NOCASE and owner = ? COLLATE NOCASE", name, owner).
Preload("Credentials").
Preload("Endpoint").
First(&repo)
q = q.First(&repo)

4
database/sql/repositories_test.go
View file

@ -188,7 +188,7 @@ func (s *RepoTestSuite) TestCreateRepository() {
}
s.Require().Equal(storeRepo.Owner, repo.Owner)
s.Require().Equal(storeRepo.Name, repo.Name)
s.Require().Equal(storeRepo.CredentialsName, repo.CredentialsName)
s.Require().Equal(storeRepo.Credentials.Name, repo.Credentials.Name)
s.Require().Equal(storeRepo.WebhookSecret, repo.WebhookSecret)
}
@ -352,7 +352,7 @@ func (s *RepoTestSuite) TestUpdateRepository() {
repo, err := s.Store.UpdateRepository(context.Background(), s.Fixtures.Repos[0].ID, s.Fixtures.UpdateRepoParams)
s.Require().Nil(err)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, repo.CredentialsName)
s.Require().Equal(s.Fixtures.UpdateRepoParams.CredentialsName, repo.Credentials.Name)
s.Require().Equal(s.Fixtures.UpdateRepoParams.WebhookSecret, repo.WebhookSecret)
}

168
database/sql/sql.go
View file

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"log/slog"
"net/url"
"strings"
"github.com/pkg/errors"
@ -26,8 +27,12 @@ import (
"gorm.io/gorm"
"gorm.io/gorm/logger"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util/appdefaults"
)
// newDBConn returns a new gorm db connection, given the config
@ -190,6 +195,154 @@ func (s *sqlDatabase) cascadeMigration() error {
return nil
}
func (s *sqlDatabase) migrateCredentialsToDB() (err error) {
s.conn.Exec("PRAGMA foreign_keys = OFF")
defer s.conn.Exec("PRAGMA foreign_keys = ON")
adminUser, err := s.GetAdminUser(s.ctx)
if err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
// Admin user doesn't exist. This is a new deploy. Nothing to migrate.
return nil
}
return errors.Wrap(err, "getting admin user")
}
// Impersonate the admin user. We're migrating from config credentials to
// database credentials. At this point, there is no other user than the admin
// user. GARM is not yet multi-user, so it's safe to assume we only have this
// one user.
adminCtx := context.Background()
adminCtx = auth.PopulateContext(adminCtx, adminUser)
slog.Info("migrating credentials to DB")
slog.Info("creating github endpoints table")
if err := s.conn.AutoMigrate(&GithubEndpoint{}); err != nil {
return errors.Wrap(err, "migrating github endpoints")
}
defer func() {
if err != nil {
slog.With(slog.Any("error", err)).Error("rolling back github github endpoints table")
s.conn.Migrator().DropTable(&GithubEndpoint{})
}
}()
slog.Info("creating github credentials table")
if err := s.conn.AutoMigrate(&GithubCredentials{}); err != nil {
return errors.Wrap(err, "migrating github credentials")
}
defer func() {
if err != nil {
slog.With(slog.Any("error", err)).Error("rolling back github github credentials table")
s.conn.Migrator().DropTable(&GithubCredentials{})
}
}()
// Create the default Github endpoint.
createEndpointParams := params.CreateGithubEndpointParams{
Name: "github.com",
Description: "The github.com endpoint",
APIBaseURL: appdefaults.GithubDefaultBaseURL,
BaseURL: appdefaults.DefaultGithubURL,
UploadBaseURL: appdefaults.GithubDefaultUploadBaseURL,
}
if _, err := s.CreateGithubEndpoint(adminCtx, createEndpointParams); err != nil {
if !errors.Is(err, runnerErrors.ErrDuplicateEntity) {
return errors.Wrap(err, "creating default github endpoint")
}
}
// Nothing to migrate.
if len(s.cfg.MigrateCredentials) == 0 {
return nil
}
slog.Info("importing credentials from config")
for _, cred := range s.cfg.MigrateCredentials {
slog.Info("importing credential", "name", cred.Name)
parsed, err := url.Parse(cred.BaseEndpoint())
if err != nil {
return errors.Wrap(err, "parsing base URL")
}
certBundle, err := cred.CACertBundle()
if err != nil {
return errors.Wrap(err, "getting CA cert bundle")
}
hostname := parsed.Hostname()
createParams := params.CreateGithubEndpointParams{
Name: hostname,
Description: fmt.Sprintf("Endpoint for %s", hostname),
APIBaseURL: cred.APIEndpoint(),
BaseURL: cred.BaseEndpoint(),
UploadBaseURL: cred.UploadEndpoint(),
CACertBundle: certBundle,
}
var endpoint params.GithubEndpoint
endpoint, err = s.GetGithubEndpoint(adminCtx, hostname)
if err != nil {
if !errors.Is(err, runnerErrors.ErrNotFound) {
return errors.Wrap(err, "getting github endpoint")
}
endpoint, err = s.CreateGithubEndpoint(adminCtx, createParams)
if err != nil {
return errors.Wrap(err, "creating default github endpoint")
}
}
credParams := params.CreateGithubCredentialsParams{
Name: cred.Name,
Description: cred.Description,
AuthType: params.GithubAuthType(cred.AuthType),
}
switch credParams.AuthType {
case params.GithubAuthTypeApp:
keyBytes, err := cred.App.PrivateKeyBytes()
if err != nil {
return errors.Wrap(err, "getting private key bytes")
}
credParams.App = params.GithubApp{
AppID: cred.App.AppID,
InstallationID: cred.App.InstallationID,
PrivateKeyBytes: keyBytes,
}
if err := credParams.App.Validate(); err != nil {
return errors.Wrap(err, "validating app credentials")
}
case params.GithubAuthTypePAT:
if cred.PAT.OAuth2Token == "" {
return errors.New("missing OAuth2 token")
}
credParams.PAT = params.GithubPAT{
OAuth2Token: cred.PAT.OAuth2Token,
}
}
creds, err := s.CreateGithubCredentials(adminCtx, endpoint.Name, credParams)
if err != nil {
return errors.Wrap(err, "creating github credentials")
}
if err := s.conn.Exec("update repositories set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint, creds.Name).Error; err != nil {
return errors.Wrap(err, "updating repositories")
}
if err := s.conn.Exec("update organizations set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint, creds.Name).Error; err != nil {
return errors.Wrap(err, "updating organizations")
}
if err := s.conn.Exec("update enterprises set credentials_id = ?,endpoint_name = ? where credentials_name = ?", creds.ID, creds.Endpoint, creds.Name).Error; err != nil {
return errors.Wrap(err, "updating enterprises")
}
}
return nil
}
func (s *sqlDatabase) migrateDB() error {
if s.conn.Migrator().HasIndex(&Organization{}, "idx_organizations_name") {
if err := s.conn.Migrator().DropIndex(&Organization{}, "idx_organizations_name"); err != nil {
@ -234,7 +387,15 @@ func (s *sqlDatabase) migrateDB() error {
}
}
var needsCredentialMigration bool
if !s.conn.Migrator().HasTable(&GithubCredentials{}) || !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
needsCredentialMigration = true
}
s.conn.Exec("PRAGMA foreign_keys = OFF")
if err := s.conn.AutoMigrate(
&User{},
&GithubEndpoint{},
&GithubCredentials{},
&Tag{},
&Pool{},
&Repository{},
@ -244,11 +405,16 @@ func (s *sqlDatabase) migrateDB() error {
&InstanceStatusUpdate{},
&Instance{},
&ControllerInfo{},
&User{},
&WorkflowJob{},
); err != nil {
return errors.Wrap(err, "running auto migrate")
}
s.conn.Exec("PRAGMA foreign_keys = ON")
if needsCredentialMigration {
if err := s.migrateCredentialsToDB(); err != nil {
return errors.Wrap(err, "migrating credentials")
}
}
return nil
}

17
database/sql/users.go
View file

@ -67,6 +67,10 @@ func (s *sqlDatabase) CreateUser(_ context.Context, user params.NewUserParams) (
return params.User{}, runnerErrors.NewConflictError("email already exists")
}
if s.HasAdminUser(context.Background()) && user.IsAdmin {
return params.User{}, runnerErrors.NewBadRequestError("admin user already exists")
}
newUser := User{
Username: user.Username,
Password: user.Password,
@ -129,3 +133,16 @@ func (s *sqlDatabase) UpdateUser(_ context.Context, user string, param params.Up
return s.sqlToParamsUser(dbUser), nil
}
// GetAdminUser returns the system admin user. This is only for internal use.
func (s *sqlDatabase) GetAdminUser(_ context.Context) (params.User, error) {
var user User
q := s.conn.Model(&User{}).Where("is_admin = ?", true).First(&user)
if q.Error != nil {
if errors.Is(q.Error, gorm.ErrRecordNotFound) {
return params.User{}, runnerErrors.ErrNotFound
}
return params.User{}, errors.Wrap(q.Error, "fetching admin user")
}
return s.sqlToParamsUser(user), nil
}

22
database/sql/util.go
View file

@ -114,10 +114,16 @@ func (s *sqlDatabase) sqlToCommonOrganization(org Organization) (params.Organiza
return params.Organization{}, errors.Wrap(err, "decrypting secret")
}
creds, err := s.sqlToCommonGithubCredentials(org.Credentials)
if err != nil {
return params.Organization{}, errors.Wrap(err, "converting credentials")
}
ret := params.Organization{
ID: org.ID.String(),
Name: org.Name,
CredentialsName: org.CredentialsName,
CredentialsName: creds.Name,
Credentials: creds,
Pools: make([]params.Pool, len(org.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: org.PoolBalancerType,
@ -146,10 +152,15 @@ func (s *sqlDatabase) sqlToCommonEnterprise(enterprise Enterprise) (params.Enter
return params.Enterprise{}, errors.Wrap(err, "decrypting secret")
}
creds, err := s.sqlToCommonGithubCredentials(enterprise.Credentials)
if err != nil {
return params.Enterprise{}, errors.Wrap(err, "converting credentials")
}
ret := params.Enterprise{
ID: enterprise.ID.String(),
Name: enterprise.Name,
CredentialsName: enterprise.CredentialsName,
CredentialsName: creds.Name,
Credentials: creds,
Pools: make([]params.Pool, len(enterprise.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: enterprise.PoolBalancerType,
@ -239,11 +250,16 @@ func (s *sqlDatabase) sqlToCommonRepository(repo Repository) (params.Repository,
return params.Repository{}, errors.Wrap(err, "decrypting secret")
}
creds, err := s.sqlToCommonGithubCredentials(repo.Credentials)
if err != nil {
return params.Repository{}, errors.Wrap(err, "converting credentials")
}
ret := params.Repository{
ID: repo.ID.String(),
Name: repo.Name,
Owner: repo.Owner,
CredentialsName: repo.CredentialsName,
CredentialsName: creds.Name,
Credentials: creds,
Pools: make([]params.Pool, len(repo.Pools)),
WebhookSecret: string(secret),
PoolBalancerType: repo.PoolBalancerType,