From 9921a7bfc8d0845056e3439a488b23400de5147d Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 22 May 2025 18:43:32 +0000 Subject: [PATCH] Fix AddInstanceEvent and expose events * We were passing the wrong type to GORM for events * We now expose entity events in the API and CLI Signed-off-by: Gabriel Adrian Samfira --- cmd/garm-cli/cmd/enterprise.go | 9 +++- cmd/garm-cli/cmd/organization.go | 8 ++- cmd/garm-cli/cmd/pool.go | 6 ++- cmd/garm-cli/cmd/repository.go | 10 +++- database/common/mocks/Store.go | 8 +-- database/sql/enterprise.go | 13 +++-- database/sql/enterprise_test.go | 26 ++++++++++ database/sql/models.go | 6 +-- database/sql/organizations.go | 20 +++++--- database/sql/organizations_test.go | 26 ++++++++++ database/sql/pools.go | 15 +++++- database/sql/repositories.go | 20 +++++--- database/sql/repositories_test.go | 27 ++++++++++ database/sql/util.go | 80 ++++++++++++++++++++++-------- params/params.go | 14 ++++++ util/github/client.go | 1 - workers/cache/cache.go | 4 +- workers/cache/tool_cache.go | 40 +++++++++++++-- 18 files changed, 273 insertions(+), 60 deletions(-) diff --git a/cmd/garm-cli/cmd/enterprise.go b/cmd/garm-cli/cmd/enterprise.go index eabfad26..1e6c3930 100644 --- a/cmd/garm-cli/cmd/enterprise.go +++ b/cmd/garm-cli/cmd/enterprise.go @@ -16,6 +16,7 @@ package cmd import ( "fmt" + "strings" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" @@ -250,9 +251,15 @@ func formatOneEnterprise(enterprise params.Enterprise) { t.AppendRow(table.Row{"Pools", pool.ID}, rowConfigAutoMerge) } } + + if len(enterprise.Events) > 0 { + for _, event := range enterprise.Events { + t.AppendRow(table.Row{"Events", fmt.Sprintf("%s %s: %s", event.CreatedAt.Format("2006-01-02T15:04:05"), strings.ToUpper(string(event.EventLevel)), event.Message)}, rowConfigAutoMerge) + } + } t.SetColumnConfigs([]table.ColumnConfig{ {Number: 1, AutoMerge: true}, - {Number: 2, AutoMerge: false}, + {Number: 2, AutoMerge: false, WidthMax: 100}, }) fmt.Println(t.Render()) diff --git a/cmd/garm-cli/cmd/organization.go b/cmd/garm-cli/cmd/organization.go index c35fd75b..58110053 100644 --- a/cmd/garm-cli/cmd/organization.go +++ b/cmd/garm-cli/cmd/organization.go @@ -16,6 +16,7 @@ package cmd import ( "fmt" + "strings" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" @@ -394,9 +395,14 @@ func formatOneOrganization(org params.Organization) { t.AppendRow(table.Row{"Pools", pool.ID}, rowConfigAutoMerge) } } + if len(org.Events) > 0 { + for _, event := range org.Events { + t.AppendRow(table.Row{"Events", fmt.Sprintf("%s %s: %s", event.CreatedAt.Format("2006-01-02T15:04:05"), strings.ToUpper(string(event.EventLevel)), event.Message)}, rowConfigAutoMerge) + } + } t.SetColumnConfigs([]table.ColumnConfig{ {Number: 1, AutoMerge: true}, - {Number: 2, AutoMerge: false}, + {Number: 2, AutoMerge: false, WidthMax: 100}, }) fmt.Println(t.Render()) diff --git a/cmd/garm-cli/cmd/pool.go b/cmd/garm-cli/cmd/pool.go index 0b891e96..b2c324ea 100644 --- a/cmd/garm-cli/cmd/pool.go +++ b/cmd/garm-cli/cmd/pool.go @@ -476,7 +476,7 @@ func formatPools(pools []params.Pool) { t.SetColumnConfigs([]table.ColumnConfig{ {Number: 2, WidthMax: 40}, }) - header := table.Row{"ID", "Image", "Flavor", "Tags", "Belongs to", "Enabled"} + header := table.Row{"ID", "Image", "Flavor", "Tags", "Belongs to", "Endpoint", "Forge Type", "Enabled"} if long { header = append(header, "Level", "Created At", "Updated at", "Runner Prefix", "Priority") } @@ -501,7 +501,7 @@ func formatPools(pools []params.Pool) { belongsTo = pool.EnterpriseName level = entityTypeEnterprise } - row := table.Row{pool.ID, pool.Image, pool.Flavor, strings.Join(tags, " "), belongsTo, pool.Enabled} + row := table.Row{pool.ID, pool.Image, pool.Flavor, strings.Join(tags, " "), belongsTo, pool.Endpoint.Name, pool.Endpoint.EndpointType, pool.Enabled} if long { row = append(row, level, pool.CreatedAt, pool.UpdatedAt, pool.GetRunnerPrefix(), pool.Priority) } @@ -561,6 +561,8 @@ func formatOnePool(pool params.Pool) { t.AppendRow(table.Row{"Runner Prefix", pool.GetRunnerPrefix()}) t.AppendRow(table.Row{"Extra specs", string(pool.ExtraSpecs)}) t.AppendRow(table.Row{"GitHub Runner Group", pool.GitHubRunnerGroup}) + t.AppendRow(table.Row{"Forge Type", pool.Endpoint.EndpointType}) + t.AppendRow(table.Row{"Endpoint Name", pool.Endpoint.Name}) if len(pool.Instances) > 0 { for _, instance := range pool.Instances { diff --git a/cmd/garm-cli/cmd/repository.go b/cmd/garm-cli/cmd/repository.go index c94495cd..96f214fd 100644 --- a/cmd/garm-cli/cmd/repository.go +++ b/cmd/garm-cli/cmd/repository.go @@ -16,6 +16,7 @@ package cmd import ( "fmt" + "strings" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" @@ -404,9 +405,16 @@ func formatOneRepository(repo params.Repository) { t.AppendRow(table.Row{"Pools", pool.ID}, rowConfigAutoMerge) } } + + if len(repo.Events) > 0 { + for _, event := range repo.Events { + t.AppendRow(table.Row{"Events", fmt.Sprintf("%s %s: %s", event.CreatedAt.Format("2006-01-02T15:04:05"), strings.ToUpper(string(event.EventLevel)), event.Message)}, rowConfigAutoMerge) + } + } + t.SetColumnConfigs([]table.ColumnConfig{ {Number: 1, AutoMerge: true}, - {Number: 2, AutoMerge: false}, + {Number: 2, AutoMerge: false, WidthMax: 100}, }) fmt.Println(t.Render()) diff --git a/database/common/mocks/Store.go b/database/common/mocks/Store.go index c5994b87..97da1c06 100644 --- a/database/common/mocks/Store.go +++ b/database/common/mocks/Store.go @@ -97,7 +97,7 @@ func (_m *Store) ControllerInfo() (params.ControllerInfo, error) { } // CreateEnterprise provides a mock function with given fields: ctx, name, credentialsName, webhookSecret, poolBalancerType -func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsName string, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Enterprise, error) { +func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Enterprise, error) { ret := _m.Called(ctx, name, credentialsName, webhookSecret, poolBalancerType) if len(ret) == 0 { @@ -106,16 +106,16 @@ func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsN var r0 params.Enterprise var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, params.PoolBalancerType) (params.Enterprise, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Enterprise, error)); ok { return rf(ctx, name, credentialsName, webhookSecret, poolBalancerType) } - if rf, ok := ret.Get(0).(func(context.Context, string, string, string, params.PoolBalancerType) params.Enterprise); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) params.Enterprise); ok { r0 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType) } else { r0 = ret.Get(0).(params.Enterprise) } - if rf, ok := ret.Get(1).(func(context.Context, string, string, string, params.PoolBalancerType) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) error); ok { r1 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType) } else { r1 = ret.Error(1) diff --git a/database/sql/enterprise.go b/database/sql/enterprise.go index e9c2ed08..41d95b26 100644 --- a/database/sql/enterprise.go +++ b/database/sql/enterprise.go @@ -70,12 +70,12 @@ func (s *sqlDatabase) CreateEnterprise(ctx context.Context, name string, credent return params.Enterprise{}, errors.Wrap(err, "creating enterprise") } - paramEnt, err = s.sqlToCommonEnterprise(newEnterprise, true) + ret, err := s.GetEnterpriseByID(ctx, newEnterprise.ID.String()) if err != nil { return params.Enterprise{}, errors.Wrap(err, "creating enterprise") } - return paramEnt, nil + return ret, nil } func (s *sqlDatabase) GetEnterprise(ctx context.Context, name, endpointName string) (params.Enterprise, error) { @@ -92,7 +92,14 @@ func (s *sqlDatabase) GetEnterprise(ctx context.Context, name, endpointName stri } func (s *sqlDatabase) GetEnterpriseByID(ctx context.Context, enterpriseID string) (params.Enterprise, error) { - enterprise, err := s.getEnterpriseByID(ctx, s.conn, enterpriseID, "Pools", "Credentials", "Endpoint", "Credentials.Endpoint") + preloadList := []string{ + "Pools", + "Credentials", + "Endpoint", + "Credentials.Endpoint", + "Events", + } + enterprise, err := s.getEnterpriseByID(ctx, s.conn, enterpriseID, preloadList...) if err != nil { return params.Enterprise{}, errors.Wrap(err, "fetching enterprise") } diff --git a/database/sql/enterprise_test.go b/database/sql/enterprise_test.go index 24e7bee7..79b298d5 100644 --- a/database/sql/enterprise_test.go +++ b/database/sql/enterprise_test.go @@ -441,6 +441,10 @@ func (s *EnterpriseTestSuite) TestGetEnterpriseByIDDBDecryptingErr() { ExpectQuery(regexp.QuoteMeta("SELECT * FROM `enterprises` WHERE id = ? AND `enterprises`.`deleted_at` IS NULL ORDER BY `enterprises`.`id` LIMIT ?")). WithArgs(s.Fixtures.Enterprises[0].ID, 1). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Enterprises[0].ID)) + s.Fixtures.SQLMock. + ExpectQuery(regexp.QuoteMeta("SELECT * FROM `enterprise_events` WHERE `enterprise_events`.`enterprise_id` = ? AND `enterprise_events`.`deleted_at` IS NULL")). + WithArgs(s.Fixtures.Enterprises[0].ID). + WillReturnRows(sqlmock.NewRows([]string{"enterprise_id"}).AddRow(s.Fixtures.Enterprises[0].ID)) s.Fixtures.SQLMock. ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`enterprise_id` = ? AND `pools`.`deleted_at` IS NULL")). WithArgs(s.Fixtures.Enterprises[0].ID). @@ -773,6 +777,28 @@ func (s *EnterpriseTestSuite) TestUpdateEnterprisePoolInvalidEnterpriseID() { s.Require().Equal("fetching pool: parsing id: invalid request", err.Error()) } +func (s *EnterpriseTestSuite) TestAddRepoEntityEvent() { + enterprise, err := s.Store.CreateEnterprise( + s.adminCtx, + s.Fixtures.CreateEnterpriseParams.Name, + s.testCreds, + s.Fixtures.CreateEnterpriseParams.WebhookSecret, + params.PoolBalancerTypeRoundRobin) + + s.Require().Nil(err) + entity, err := enterprise.GetEntity() + s.Require().Nil(err) + err = s.Store.AddEntityEvent(s.adminCtx, entity, params.StatusEvent, params.EventInfo, "this is a test", 20) + s.Require().Nil(err) + + enterprise, err = s.Store.GetEnterpriseByID(s.adminCtx, enterprise.ID) + s.Require().Nil(err) + s.Require().Equal(1, len(enterprise.Events)) + s.Require().Equal(params.StatusEvent, enterprise.Events[0].EventType) + s.Require().Equal(params.EventInfo, enterprise.Events[0].EventLevel) + s.Require().Equal("this is a test", enterprise.Events[0].Message) +} + func TestEnterpriseTestSuite(t *testing.T) { suite.Run(t, new(EnterpriseTestSuite)) } diff --git a/database/sql/models.go b/database/sql/models.go index 154fb51d..4cdb9b8b 100644 --- a/database/sql/models.go +++ b/database/sql/models.go @@ -185,7 +185,7 @@ type Repository struct { EndpointName *string `gorm:"index:idx_owner_nocase,unique,collate:nocase"` Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"` - Events []*RepositoryEvent `gorm:"foreignKey:RepoID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` + Events []RepositoryEvent `gorm:"foreignKey:RepoID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` } type OrganizationEvent struct { @@ -217,7 +217,7 @@ type Organization struct { EndpointName *string `gorm:"index:idx_org_name_nocase,collate:nocase"` Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"` - Events []*OrganizationEvent `gorm:"foreignKey:OrgID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` + Events []OrganizationEvent `gorm:"foreignKey:OrgID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` } type EnterpriseEvent struct { @@ -247,7 +247,7 @@ type Enterprise struct { EndpointName *string `gorm:"index:idx_ent_name_nocase,collate:nocase"` Endpoint GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"` - Events []*EnterpriseEvent `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` + Events []EnterpriseEvent `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"` } type Address struct { diff --git a/database/sql/organizations.go b/database/sql/organizations.go index 6f8eaa10..73456362 100644 --- a/database/sql/organizations.go +++ b/database/sql/organizations.go @@ -70,17 +70,12 @@ func (s *sqlDatabase) CreateOrganization(ctx context.Context, name string, crede return params.Organization{}, errors.Wrap(err, "creating org") } - org, err := s.getOrgByID(ctx, s.conn, newOrg.ID.String(), "Pools", "Endpoint", "Credentials", "GiteaCredentials", "Credentials.Endpoint", "GiteaCredentials.Endpoint") + ret, err := s.GetOrganizationByID(ctx, newOrg.ID.String()) if err != nil { return params.Organization{}, errors.Wrap(err, "creating org") } - param, err = s.sqlToCommonOrganization(org, true) - if err != nil { - return params.Organization{}, errors.Wrap(err, "creating org") - } - - return param, nil + return ret, nil } func (s *sqlDatabase) GetOrganization(ctx context.Context, name, endpointName string) (params.Organization, error) { @@ -215,7 +210,16 @@ func (s *sqlDatabase) UpdateOrganization(ctx context.Context, orgID string, para } func (s *sqlDatabase) GetOrganizationByID(ctx context.Context, orgID string) (params.Organization, error) { - org, err := s.getOrgByID(ctx, s.conn, orgID, "Pools", "Credentials", "Endpoint", "Credentials.Endpoint", "GiteaCredentials", "GiteaCredentials.Endpoint") + preloadList := []string{ + "Pools", + "Credentials", + "Endpoint", + "Credentials.Endpoint", + "GiteaCredentials", + "GiteaCredentials.Endpoint", + "Events", + } + org, err := s.getOrgByID(ctx, s.conn, orgID, preloadList...) if err != nil { return params.Organization{}, errors.Wrap(err, "fetching org") } diff --git a/database/sql/organizations_test.go b/database/sql/organizations_test.go index a93ef372..5c053cec 100644 --- a/database/sql/organizations_test.go +++ b/database/sql/organizations_test.go @@ -502,6 +502,10 @@ func (s *OrgTestSuite) TestGetOrganizationByIDDBDecryptingErr() { ExpectQuery(regexp.QuoteMeta("SELECT * FROM `organizations` WHERE id = ? AND `organizations`.`deleted_at` IS NULL ORDER BY `organizations`.`id` LIMIT ?")). WithArgs(s.Fixtures.Orgs[0].ID, 1). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Orgs[0].ID)) + s.Fixtures.SQLMock. + ExpectQuery(regexp.QuoteMeta("SELECT * FROM `organization_events` WHERE `organization_events`.`org_id` = ? AND `organization_events`.`deleted_at` IS NULL")). + WithArgs(s.Fixtures.Orgs[0].ID). + WillReturnRows(sqlmock.NewRows([]string{"org_id"}).AddRow(s.Fixtures.Orgs[0].ID)) s.Fixtures.SQLMock. ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`org_id` = ? AND `pools`.`deleted_at` IS NULL")). WithArgs(s.Fixtures.Orgs[0].ID). @@ -826,6 +830,28 @@ func (s *OrgTestSuite) TestUpdateOrganizationPool() { s.Require().Equal(s.Fixtures.UpdatePoolParams.Flavor, pool.Flavor) } +func (s *OrgTestSuite) TestAddOrgEntityEvent() { + org, err := s.Store.CreateOrganization( + s.adminCtx, + s.Fixtures.CreateOrgParams.Name, + s.testCreds, + s.Fixtures.CreateOrgParams.WebhookSecret, + params.PoolBalancerTypeRoundRobin) + + s.Require().Nil(err) + entity, err := org.GetEntity() + s.Require().Nil(err) + err = s.Store.AddEntityEvent(s.adminCtx, entity, params.StatusEvent, params.EventInfo, "this is a test", 20) + s.Require().Nil(err) + + org, err = s.Store.GetOrganizationByID(s.adminCtx, org.ID) + s.Require().Nil(err) + s.Require().Equal(1, len(org.Events)) + s.Require().Equal(params.StatusEvent, org.Events[0].EventType) + s.Require().Equal(params.EventInfo, org.Events[0].EventLevel) + s.Require().Equal("this is a test", org.Events[0].Message) +} + func (s *OrgTestSuite) TestUpdateOrganizationPoolInvalidOrgID() { entity := params.ForgeEntity{ ID: "dummy-org-id", diff --git a/database/sql/pools.go b/database/sql/pools.go index 24476fe8..a4b3354e 100644 --- a/database/sql/pools.go +++ b/database/sql/pools.go @@ -40,8 +40,11 @@ func (s *sqlDatabase) ListAllPools(_ context.Context) ([]params.Pool, error) { q := s.conn.Model(&Pool{}). Preload("Tags"). Preload("Organization"). + Preload("Organization.Endpoint"). Preload("Repository"). + Preload("Repository.Endpoint"). Preload("Enterprise"). + Preload("Enterprise.Endpoint"). Omit("extra_specs"). Find(&pools) if q.Error != nil { @@ -60,7 +63,17 @@ func (s *sqlDatabase) ListAllPools(_ context.Context) ([]params.Pool, error) { } func (s *sqlDatabase) GetPoolByID(_ context.Context, poolID string) (params.Pool, error) { - pool, err := s.getPoolByID(s.conn, poolID, "Tags", "Instances", "Enterprise", "Organization", "Repository") + preloadList := []string{ + "Tags", + "Instances", + "Enterprise", + "Enterprise.Endpoint", + "Organization", + "Organization.Endpoint", + "Repository", + "Repository.Endpoint", + } + pool, err := s.getPoolByID(s.conn, poolID, preloadList...) if err != nil { return params.Pool{}, errors.Wrap(err, "fetching pool by ID") } diff --git a/database/sql/repositories.go b/database/sql/repositories.go index d7419070..03452df6 100644 --- a/database/sql/repositories.go +++ b/database/sql/repositories.go @@ -71,17 +71,12 @@ func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name string, return params.Repository{}, errors.Wrap(err, "creating repository") } - repo, err := s.getRepoByID(ctx, s.conn, newRepo.ID.String(), "Endpoint", "Credentials", "GiteaCredentials", "Credentials.Endpoint", "GiteaCredentials.Endpoint") + ret, err := s.GetRepositoryByID(ctx, newRepo.ID.String()) if err != nil { return params.Repository{}, errors.Wrap(err, "creating repository") } - param, err = s.sqlToCommonRepository(repo, true) - if err != nil { - return params.Repository{}, errors.Wrap(err, "creating repository") - } - - return param, nil + return ret, nil } func (s *sqlDatabase) GetRepository(ctx context.Context, owner, name, endpointName string) (params.Repository, error) { @@ -217,7 +212,16 @@ func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param } func (s *sqlDatabase) GetRepositoryByID(ctx context.Context, repoID string) (params.Repository, error) { - repo, err := s.getRepoByID(ctx, s.conn, repoID, "Pools", "Credentials", "Endpoint", "Credentials.Endpoint", "GiteaCredentials", "GiteaCredentials.Endpoint") + preloadList := []string{ + "Pools", + "Credentials", + "Endpoint", + "Credentials.Endpoint", + "GiteaCredentials", + "GiteaCredentials.Endpoint", + "Events", + } + repo, err := s.getRepoByID(ctx, s.conn, repoID, preloadList...) if err != nil { return params.Repository{}, errors.Wrap(err, "fetching repo") } diff --git a/database/sql/repositories_test.go b/database/sql/repositories_test.go index f27e10b5..f593ddce 100644 --- a/database/sql/repositories_test.go +++ b/database/sql/repositories_test.go @@ -558,6 +558,10 @@ func (s *RepoTestSuite) TestGetRepositoryByIDDBDecryptingErr() { ExpectQuery(regexp.QuoteMeta("SELECT * FROM `repositories` WHERE id = ? AND `repositories`.`deleted_at` IS NULL ORDER BY `repositories`.`id` LIMIT ?")). WithArgs(s.Fixtures.Repos[0].ID, 1). WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(s.Fixtures.Repos[0].ID)) + s.Fixtures.SQLMock. + ExpectQuery(regexp.QuoteMeta("SELECT * FROM `repository_events` WHERE `repository_events`.`repo_id` = ? AND `repository_events`.`deleted_at` IS NULL")). + WithArgs(s.Fixtures.Repos[0].ID). + WillReturnRows(sqlmock.NewRows([]string{"repo_id"}).AddRow(s.Fixtures.Repos[0].ID)) s.Fixtures.SQLMock. ExpectQuery(regexp.QuoteMeta("SELECT * FROM `pools` WHERE `pools`.`repo_id` = ? AND `pools`.`deleted_at` IS NULL")). WithArgs(s.Fixtures.Repos[0].ID). @@ -894,6 +898,29 @@ func (s *RepoTestSuite) TestUpdateRepositoryPoolInvalidRepoID() { s.Require().Equal("fetching pool: parsing id: invalid request", err.Error()) } +func (s *RepoTestSuite) TestAddRepoEntityEvent() { + repo, err := s.Store.CreateRepository( + s.adminCtx, + s.Fixtures.CreateRepoParams.Owner, + s.Fixtures.CreateRepoParams.Name, + s.testCreds, + s.Fixtures.CreateRepoParams.WebhookSecret, + params.PoolBalancerTypeRoundRobin) + + s.Require().Nil(err) + entity, err := repo.GetEntity() + s.Require().Nil(err) + err = s.Store.AddEntityEvent(s.adminCtx, entity, params.StatusEvent, params.EventInfo, "this is a test", 20) + s.Require().Nil(err) + + repo, err = s.Store.GetRepositoryByID(s.adminCtx, repo.ID) + s.Require().Nil(err) + s.Require().Equal(1, len(repo.Events)) + s.Require().Equal(params.StatusEvent, repo.Events[0].EventType) + s.Require().Equal(params.EventInfo, repo.Events[0].EventLevel) + s.Require().Equal("this is a test", repo.Events[0].Message) +} + func TestRepoTestSuite(t *testing.T) { suite.Run(t, new(RepoTestSuite)) } diff --git a/database/sql/util.go b/database/sql/util.go index 11d338ba..d55e0174 100644 --- a/database/sql/util.go +++ b/database/sql/util.go @@ -166,6 +166,19 @@ func (s *sqlDatabase) sqlToCommonOrganization(org Organization, detailed bool) ( return params.Organization{}, errors.Wrap(err, "converting credentials") } + if len(org.Events) > 0 { + ret.Events = make([]params.EntityEvent, len(org.Events)) + for idx, event := range org.Events { + ret.Events[idx] = params.EntityEvent{ + ID: event.ID, + Message: event.Message, + EventType: event.EventType, + EventLevel: event.EventLevel, + CreatedAt: event.CreatedAt, + } + } + } + if detailed { ret.Credentials = forgeCreds ret.CredentialsName = forgeCreds.Name @@ -214,6 +227,19 @@ func (s *sqlDatabase) sqlToCommonEnterprise(enterprise Enterprise, detailed bool ret.CredentialsID = *enterprise.CredentialsID } + if len(enterprise.Events) > 0 { + ret.Events = make([]params.EntityEvent, len(enterprise.Events)) + for idx, event := range enterprise.Events { + ret.Events[idx] = params.EntityEvent{ + ID: event.ID, + Message: event.Message, + EventType: event.EventType, + EventLevel: event.EventLevel, + CreatedAt: event.CreatedAt, + } + } + } + if detailed { creds, err := s.sqlToCommonForgeCredentials(enterprise.Credentials) if err != nil { @@ -260,28 +286,37 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) (params.Pool, error) { UpdatedAt: pool.UpdatedAt, } + var ep GithubEndpoint if pool.RepoID != nil { ret.RepoID = pool.RepoID.String() if pool.Repository.Owner != "" && pool.Repository.Name != "" { ret.RepoName = fmt.Sprintf("%s/%s", pool.Repository.Owner, pool.Repository.Name) } + ep = pool.Repository.Endpoint } if pool.OrgID != nil && pool.Organization.Name != "" { ret.OrgID = pool.OrgID.String() ret.OrgName = pool.Organization.Name + ep = pool.Organization.Endpoint } if pool.EnterpriseID != nil && pool.Enterprise.Name != "" { ret.EnterpriseID = pool.EnterpriseID.String() ret.EnterpriseName = pool.Enterprise.Name + ep = pool.Enterprise.Endpoint } + endpoint, err := s.sqlToCommonGithubEndpoint(ep) + if err != nil { + return params.Pool{}, errors.Wrap(err, "converting endpoint") + } + ret.Endpoint = endpoint + for idx, val := range pool.Tags { ret.Tags[idx] = s.sqlToCommonTags(*val) } - var err error for idx, inst := range pool.Instances { ret.Instances[idx], err = s.sqlToParamsInstance(inst) if err != nil { @@ -399,6 +434,19 @@ func (s *sqlDatabase) sqlToCommonRepository(repo Repository, detailed bool) (par return params.Repository{}, errors.Wrap(err, "converting credentials") } + if len(repo.Events) > 0 { + ret.Events = make([]params.EntityEvent, len(repo.Events)) + for idx, event := range repo.Events { + ret.Events[idx] = params.EntityEvent{ + ID: event.ID, + Message: event.Message, + EventType: event.EventType, + EventLevel: event.EventLevel, + CreatedAt: event.CreatedAt, + } + } + } + if detailed { ret.Credentials = forgeCreds ret.CredentialsName = forgeCreds.Name @@ -654,7 +702,7 @@ func (s *sqlDatabase) GetForgeEntity(_ context.Context, entityType params.ForgeE } func (s *sqlDatabase) addRepositoryEvent(ctx context.Context, repoID string, event params.EventType, eventLevel params.EventLevel, statusMessage string, maxEvents int) error { - repo, err := s.GetRepositoryByID(ctx, repoID) + repo, err := s.getRepoByID(ctx, s.conn, repoID) if err != nil { return errors.Wrap(err, "updating instance") } @@ -670,20 +718,16 @@ func (s *sqlDatabase) addRepositoryEvent(ctx context.Context, repoID string, eve } if maxEvents > 0 { - repoID, err := uuid.Parse(repo.ID) - if err != nil { - return errors.Wrap(runnerErrors.ErrBadRequest, "parsing id") - } var latestEvents []RepositoryEvent q := s.conn.Model(&RepositoryEvent{}). Limit(maxEvents).Order("id desc"). - Where("repo_id = ?", repoID).Find(&latestEvents) + Where("repo_id = ?", repo.ID).Find(&latestEvents) if q.Error != nil { return errors.Wrap(q.Error, "fetching latest events") } if len(latestEvents) == maxEvents { lastInList := latestEvents[len(latestEvents)-1] - if err := s.conn.Where("repo_id = ? and id < ?", repoID, lastInList.ID).Unscoped().Delete(&RepositoryEvent{}).Error; err != nil { + if err := s.conn.Where("repo_id = ? and id < ?", repo.ID, lastInList.ID).Unscoped().Delete(&RepositoryEvent{}).Error; err != nil { return errors.Wrap(err, "deleting old events") } } @@ -692,7 +736,7 @@ func (s *sqlDatabase) addRepositoryEvent(ctx context.Context, repoID string, eve } func (s *sqlDatabase) addOrgEvent(ctx context.Context, orgID string, event params.EventType, eventLevel params.EventLevel, statusMessage string, maxEvents int) error { - org, err := s.GetOrganizationByID(ctx, orgID) + org, err := s.getOrgByID(ctx, s.conn, orgID) if err != nil { return errors.Wrap(err, "updating instance") } @@ -708,20 +752,16 @@ func (s *sqlDatabase) addOrgEvent(ctx context.Context, orgID string, event param } if maxEvents > 0 { - orgID, err := uuid.Parse(org.ID) - if err != nil { - return errors.Wrap(runnerErrors.ErrBadRequest, "parsing id") - } var latestEvents []OrganizationEvent q := s.conn.Model(&OrganizationEvent{}). Limit(maxEvents).Order("id desc"). - Where("org_id = ?", orgID).Find(&latestEvents) + Where("org_id = ?", org.ID).Find(&latestEvents) if q.Error != nil { return errors.Wrap(q.Error, "fetching latest events") } if len(latestEvents) == maxEvents { lastInList := latestEvents[len(latestEvents)-1] - if err := s.conn.Where("org_id = ? and id < ?", orgID, lastInList.ID).Unscoped().Delete(&OrganizationEvent{}).Error; err != nil { + if err := s.conn.Where("org_id = ? and id < ?", org.ID, lastInList.ID).Unscoped().Delete(&OrganizationEvent{}).Error; err != nil { return errors.Wrap(err, "deleting old events") } } @@ -730,7 +770,7 @@ func (s *sqlDatabase) addOrgEvent(ctx context.Context, orgID string, event param } func (s *sqlDatabase) addEnterpriseEvent(ctx context.Context, entID string, event params.EventType, eventLevel params.EventLevel, statusMessage string, maxEvents int) error { - ent, err := s.GetEnterpriseByID(ctx, entID) + ent, err := s.getEnterpriseByID(ctx, s.conn, entID) if err != nil { return errors.Wrap(err, "updating instance") } @@ -746,20 +786,16 @@ func (s *sqlDatabase) addEnterpriseEvent(ctx context.Context, entID string, even } if maxEvents > 0 { - entID, err := uuid.Parse(ent.ID) - if err != nil { - return errors.Wrap(runnerErrors.ErrBadRequest, "parsing id") - } var latestEvents []EnterpriseEvent q := s.conn.Model(&EnterpriseEvent{}). Limit(maxEvents).Order("id desc"). - Where("enterprise_id = ?", entID).Find(&latestEvents) + Where("enterprise_id = ?", ent.ID).Find(&latestEvents) if q.Error != nil { return errors.Wrap(q.Error, "fetching latest events") } if len(latestEvents) == maxEvents { lastInList := latestEvents[len(latestEvents)-1] - if err := s.conn.Where("enterprise_id = ? and id < ?", entID, lastInList.ID).Unscoped().Delete(&EnterpriseEvent{}).Error; err != nil { + if err := s.conn.Where("enterprise_id = ? and id < ?", ent.ID, lastInList.ID).Unscoped().Delete(&EnterpriseEvent{}).Error; err != nil { return errors.Wrap(err, "deleting old events") } } diff --git a/params/params.go b/params/params.go index e154b2df..a127d760 100644 --- a/params/params.go +++ b/params/params.go @@ -177,6 +177,15 @@ type StatusMessage struct { EventLevel EventLevel `json:"event_level,omitempty"` } +type EntityEvent struct { + ID uint `json:"id,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + + EventType EventType `json:"event_type,omitempty"` + EventLevel EventLevel `json:"event_level,omitempty"` + Message string `json:"message,omitempty"` +} + type Instance struct { // ID is the database ID of this instance. ID string `json:"id,omitempty"` @@ -365,6 +374,8 @@ type Pool struct { EnterpriseID string `json:"enterprise_id,omitempty"` EnterpriseName string `json:"enterprise_name,omitempty"` + Endpoint ForgeEndpoint `json:"forge_type,omitempty"` + RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` @@ -600,6 +611,7 @@ type Repository struct { Endpoint ForgeEndpoint `json:"endpoint,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` + Events []EntityEvent `json:"events,omitempty"` // Do not serialize sensitive info. WebhookSecret string `json:"-"` } @@ -669,6 +681,7 @@ type Organization struct { Endpoint ForgeEndpoint `json:"endpoint,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` + Events []EntityEvent `json:"events,omitempty"` // Do not serialize sensitive info. WebhookSecret string `json:"-"` } @@ -726,6 +739,7 @@ type Enterprise struct { Endpoint ForgeEndpoint `json:"endpoint,omitempty"` CreatedAt time.Time `json:"created_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` + Events []EntityEvent `json:"events,omitempty"` // Do not serialize sensitive info. WebhookSecret string `json:"-"` } diff --git a/util/github/client.go b/util/github/client.go index a553e1d8..35d846ab 100644 --- a/util/github/client.go +++ b/util/github/client.go @@ -245,7 +245,6 @@ func parseError(response *github.Response, err error) error { statusCode = response.StatusCode } - slog.Debug("parsing error", "status_code", statusCode, "response", response, "error", err) switch statusCode { case http.StatusNotFound: return runnerErrors.ErrNotFound diff --git a/workers/cache/cache.go b/workers/cache/cache.go index 8f53cb67..a00c7667 100644 --- a/workers/cache/cache.go +++ b/workers/cache/cache.go @@ -130,7 +130,7 @@ func (w *Worker) loadAllEntities() error { } for _, entity := range cache.GetAllEntities() { - worker := newToolsUpdater(w.ctx, entity) + worker := newToolsUpdater(w.ctx, entity, w.store) if err := worker.Start(); err != nil { return fmt.Errorf("starting tools updater: %w", err) } @@ -286,7 +286,7 @@ func (w *Worker) handleEntityEvent(entityGetter params.EntityGetter, op common.O cache.SetEntity(entity) worker, ok := w.toolsWorkes[entity.ID] if !ok { - worker = newToolsUpdater(w.ctx, entity) + worker = newToolsUpdater(w.ctx, entity, w.store) if err := worker.Start(); err != nil { slog.ErrorContext(w.ctx, "starting tools updater", "error", err) return diff --git a/workers/cache/tool_cache.go b/workers/cache/tool_cache.go index 2e91bf50..3df103ec 100644 --- a/workers/cache/tool_cache.go +++ b/workers/cache/tool_cache.go @@ -25,16 +25,18 @@ import ( commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/cache" + "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/params" garmUtil "github.com/cloudbase/garm/util" "github.com/cloudbase/garm/util/github" ) -func newToolsUpdater(ctx context.Context, entity params.ForgeEntity) *toolsUpdater { +func newToolsUpdater(ctx context.Context, entity params.ForgeEntity, store common.Store) *toolsUpdater { return &toolsUpdater{ ctx: ctx, entity: entity, quit: make(chan struct{}), + store: store, } } @@ -44,6 +46,7 @@ type toolsUpdater struct { entity params.ForgeEntity tools []commonParams.RunnerApplicationDownload lastUpdate time.Time + store common.Store mux sync.Mutex running bool @@ -157,7 +160,14 @@ func (t *toolsUpdater) giteaUpdateLoop() { } t.sleepWithCancel(time.Duration(randInt.Int64()) * time.Millisecond) tools, err := getTools() - if err == nil { + if err != nil { + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventError, fmt.Sprintf("failed to update gitea tools: %q", err), 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } + } else { + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventInfo, "successfully updated tools", 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } cache.SetGithubToolsCache(t.entity, tools) } @@ -174,9 +184,15 @@ func (t *toolsUpdater) giteaUpdateLoop() { case <-ticker.C: tools, err := getTools() if err != nil { + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventError, fmt.Sprintf("failed to update gitea tools: %q", err), 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } slog.DebugContext(t.ctx, "failed to update gitea tools", "error", err) continue } + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventInfo, "successfully updated tools", 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } cache.SetGithubToolsCache(t.entity, tools) } } @@ -197,12 +213,18 @@ func (t *toolsUpdater) loop() { now := time.Now().UTC() if now.After(t.lastUpdate.Add(40 * time.Minute)) { if err := t.updateTools(); err != nil { + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventError, fmt.Sprintf("failed to update tools: %q", err), 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } slog.ErrorContext(t.ctx, "initial tools update error", "error", err) resetTime = now.Add(5 * time.Minute) slog.ErrorContext(t.ctx, "initial tools update error", "error", err) } else { // Tools are usually valid for 1 hour. resetTime = t.lastUpdate.Add(40 * time.Minute) + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventInfo, "successfully updated tools", 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } } } @@ -224,12 +246,18 @@ func (t *toolsUpdater) loop() { case <-timer.C: slog.DebugContext(t.ctx, "updating tools") now = time.Now().UTC() - if err := t.updateTools(); err == nil { + if err := t.updateTools(); err != nil { slog.ErrorContext(t.ctx, "updating tools", "error", err) + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventError, fmt.Sprintf("failed to update tools: %q", err), 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } resetTime = now.Add(5 * time.Minute) } else { // Tools are usually valid for 1 hour. resetTime = t.lastUpdate.Add(40 * time.Minute) + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventInfo, "successfully updated tools", 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } } case <-t.reset: slog.DebugContext(t.ctx, "resetting tools updater") @@ -237,10 +265,16 @@ func (t *toolsUpdater) loop() { now = time.Now().UTC() if err := t.updateTools(); err != nil { slog.ErrorContext(t.ctx, "updating tools", "error", err) + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventError, fmt.Sprintf("failed to update tools: %q", err), 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } resetTime = now.Add(5 * time.Minute) } else { // Tools are usually valid for 1 hour. resetTime = t.lastUpdate.Add(40 * time.Minute) + if err := t.store.AddEntityEvent(t.ctx, t.entity, params.StatusEvent, params.EventInfo, "successfully updated tools", 30); err != nil { + slog.ErrorContext(t.ctx, "failed to add entity event", "error", err) + } } } timer.Stop()