Add agent mode
This change adds a new "agent mode" to GARM. The agent enables GARM to set up a persistent websocket connection between the garm server and the runners it spawns. The goal is to be able to easier keep track of state, even without subsequent webhooks from the forge. The Agent will report via websockets when the runner is actually online, when it started a job and when it finished a job. Additionally, the agent allows us to enable optional remote shell between the user and any runner that is spun up using agent mode. The remote shell is multiplexed over the same persistent websocket connection the agent sets up with the server (the agent never listens on a port). Enablement has also been done in the web UI for this functionality. Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira
Gabriel
parent
3b132e4233
commit
42cfd1b3c6
246 changed files with 11042 additions and 672 deletions
|
|
@ -21,6 +21,7 @@ import (
|
|||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gorm.io/driver/mysql"
|
||||
|
|
@ -470,6 +471,11 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
return fmt.Errorf("failed to get linux template for gitea: %w", err)
|
||||
}
|
||||
|
||||
giteaWindowsData, err := templates.GetTemplateContent(commonParams.Windows, params.GiteaEndpointType)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get windows template for gitea: %w", err)
|
||||
}
|
||||
|
||||
adminCtx := auth.GetAdminContext(s.ctx)
|
||||
|
||||
githubWindowsParams := params.CreateTemplateParams{
|
||||
|
|
@ -478,8 +484,9 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
OSType: commonParams.Windows,
|
||||
ForgeType: params.GithubEndpointType,
|
||||
Data: githubWindowsData,
|
||||
IsSystem: true,
|
||||
}
|
||||
githubWindowsSystemTemplate, err := s.createSystemTemplate(adminCtx, githubWindowsParams)
|
||||
githubWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, githubWindowsParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create github windows template: %w", err)
|
||||
}
|
||||
|
|
@ -490,8 +497,9 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
OSType: commonParams.Linux,
|
||||
ForgeType: params.GithubEndpointType,
|
||||
Data: githubLinuxData,
|
||||
IsSystem: true,
|
||||
}
|
||||
githubLinuxSystemTemplate, err := s.createSystemTemplate(adminCtx, githubLinuxParams)
|
||||
githubLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, githubLinuxParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create github linux template: %w", err)
|
||||
}
|
||||
|
|
@ -502,12 +510,26 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
OSType: commonParams.Linux,
|
||||
ForgeType: params.GiteaEndpointType,
|
||||
Data: giteaLinuxData,
|
||||
IsSystem: true,
|
||||
}
|
||||
giteaLinuxSystemTemplate, err := s.createSystemTemplate(adminCtx, giteaLinuxParams)
|
||||
giteaLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, giteaLinuxParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gitea linux template: %w", err)
|
||||
}
|
||||
|
||||
giteaWindowsParams := params.CreateTemplateParams{
|
||||
Name: "gitea_windows",
|
||||
Description: "Default Windows runner install template for Gitea",
|
||||
OSType: commonParams.Windows,
|
||||
ForgeType: params.GiteaEndpointType,
|
||||
Data: giteaWindowsData,
|
||||
IsSystem: true,
|
||||
}
|
||||
giteaWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, giteaWindowsParams)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gitea windows template: %w", err)
|
||||
}
|
||||
|
||||
getTplID := func(forgeType params.EndpointType, osType commonParams.OSType) uint {
|
||||
var templateID uint
|
||||
switch forgeType {
|
||||
|
|
@ -515,6 +537,8 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
switch osType {
|
||||
case commonParams.Linux:
|
||||
templateID = giteaLinuxSystemTemplate.ID
|
||||
case commonParams.Windows:
|
||||
templateID = giteaWindowsSystemTemplate.ID
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
|
|
@ -582,64 +606,142 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// dropIndexIfExists drops an index if it exists
|
||||
func (s *sqlDatabase) dropIndexIfExists(model interface{}, indexName string) {
|
||||
if s.conn.Migrator().HasIndex(model, indexName) {
|
||||
if err := s.conn.Migrator().DropIndex(model, indexName); err != nil {
|
||||
slog.With(slog.Any("error", err)).
|
||||
Error(fmt.Sprintf("failed to drop index %s", indexName))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// migratePoolNullIDs updates pools to set null IDs instead of zero UUIDs
|
||||
func (s *sqlDatabase) migratePoolNullIDs() error {
|
||||
if !s.conn.Migrator().HasTable(&Pool{}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
zeroUUID := "00000000-0000-0000-0000-000000000000"
|
||||
updates := []struct {
|
||||
column string
|
||||
query string
|
||||
}{
|
||||
{"repo_id", fmt.Sprintf("update pools set repo_id=NULL where repo_id='%s'", zeroUUID)},
|
||||
{"org_id", fmt.Sprintf("update pools set org_id=NULL where org_id='%s'", zeroUUID)},
|
||||
{"enterprise_id", fmt.Sprintf("update pools set enterprise_id=NULL where enterprise_id='%s'", zeroUUID)},
|
||||
}
|
||||
|
||||
for _, update := range updates {
|
||||
if err := s.conn.Exec(update.query).Error; err != nil {
|
||||
return fmt.Errorf("error updating pools %s: %w", update.column, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateGithubEndpointType adds and initializes endpoint_type column
|
||||
func (s *sqlDatabase) migrateGithubEndpointType() error {
|
||||
if !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.conn.Migrator().HasColumn(&GithubEndpoint{}, "endpoint_type") {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := s.conn.Migrator().AutoMigrate(&GithubEndpoint{}); err != nil {
|
||||
return fmt.Errorf("error migrating github endpoints: %w", err)
|
||||
}
|
||||
|
||||
if err := s.conn.Exec("update github_endpoints set endpoint_type = 'github' where endpoint_type is null").Error; err != nil {
|
||||
return fmt.Errorf("error updating github endpoints: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateControllerInfo updates controller info with new fields
|
||||
func (s *sqlDatabase) migrateControllerInfo(hasMinAgeField, hasAgentURL bool) error {
|
||||
if hasMinAgeField && hasAgentURL {
|
||||
return nil
|
||||
}
|
||||
|
||||
var controller ControllerInfo
|
||||
if err := s.conn.First(&controller).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("error fetching controller info: %w", err)
|
||||
}
|
||||
|
||||
if !hasMinAgeField {
|
||||
controller.MinimumJobAgeBackoff = 30
|
||||
}
|
||||
|
||||
if controller.GARMAgentReleasesURL == "" {
|
||||
controller.GARMAgentReleasesURL = appdefaults.GARMAgentDefaultReleasesURL
|
||||
}
|
||||
|
||||
if !hasAgentURL && controller.WebhookBaseURL != "" {
|
||||
matchWebhooksPath := regexp.MustCompile(`/webhooks(/)?$`)
|
||||
controller.AgentURL = matchWebhooksPath.ReplaceAllLiteralString(controller.WebhookBaseURL, `/agent`)
|
||||
}
|
||||
|
||||
if err := s.conn.Save(&controller).Error; err != nil {
|
||||
return fmt.Errorf("error updating controller info: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// preMigrationChecks performs checks before running migrations
|
||||
func (s *sqlDatabase) preMigrationChecks() (needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL bool) {
|
||||
// Check if credentials need migration
|
||||
needsCredentialMigration = !s.conn.Migrator().HasTable(&GithubCredentials{}) ||
|
||||
!s.conn.Migrator().HasTable(&GithubEndpoint{})
|
||||
|
||||
// Check if templates need migration
|
||||
migrateTemplates = !s.conn.Migrator().HasTable(&Template{})
|
||||
|
||||
// Check for controller info fields
|
||||
if s.conn.Migrator().HasTable(&ControllerInfo{}) {
|
||||
hasMinAgeField = s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff")
|
||||
hasAgentURL = s.conn.Migrator().HasColumn(&ControllerInfo{}, "agent_url")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *sqlDatabase) migrateDB() error {
|
||||
if s.conn.Migrator().HasIndex(&Organization{}, "idx_organizations_name") {
|
||||
if err := s.conn.Migrator().DropIndex(&Organization{}, "idx_organizations_name"); err != nil {
|
||||
slog.With(slog.Any("error", err)).Error("failed to drop index idx_organizations_name")
|
||||
}
|
||||
}
|
||||
|
||||
if s.conn.Migrator().HasIndex(&Repository{}, "idx_owner") {
|
||||
if err := s.conn.Migrator().DropIndex(&Repository{}, "idx_owner"); err != nil {
|
||||
slog.With(slog.Any("error", err)).Error("failed to drop index idx_owner")
|
||||
}
|
||||
}
|
||||
// Drop obsolete indexes
|
||||
s.dropIndexIfExists(&Organization{}, "idx_organizations_name")
|
||||
s.dropIndexIfExists(&Repository{}, "idx_owner")
|
||||
|
||||
// Run cascade migration
|
||||
if err := s.cascadeMigration(); err != nil {
|
||||
return fmt.Errorf("error running cascade migration: %w", err)
|
||||
}
|
||||
|
||||
if s.conn.Migrator().HasTable(&Pool{}) {
|
||||
if err := s.conn.Exec("update pools set repo_id=NULL where repo_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
|
||||
return fmt.Errorf("error updating pools %w", err)
|
||||
}
|
||||
|
||||
if err := s.conn.Exec("update pools set org_id=NULL where org_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
|
||||
return fmt.Errorf("error updating pools: %w", err)
|
||||
}
|
||||
|
||||
if err := s.conn.Exec("update pools set enterprise_id=NULL where enterprise_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
|
||||
return fmt.Errorf("error updating pools: %w", err)
|
||||
}
|
||||
// Migrate pool null IDs
|
||||
if err := s.migratePoolNullIDs(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate workflows
|
||||
if err := s.migrateWorkflow(); err != nil {
|
||||
return fmt.Errorf("error migrating workflows: %w", err)
|
||||
}
|
||||
|
||||
if s.conn.Migrator().HasTable(&GithubEndpoint{}) {
|
||||
if !s.conn.Migrator().HasColumn(&GithubEndpoint{}, "endpoint_type") {
|
||||
if err := s.conn.Migrator().AutoMigrate(&GithubEndpoint{}); err != nil {
|
||||
return fmt.Errorf("error migrating github endpoints: %w", err)
|
||||
}
|
||||
if err := s.conn.Exec("update github_endpoints set endpoint_type = 'github' where endpoint_type is null").Error; err != nil {
|
||||
return fmt.Errorf("error updating github endpoints: %w", err)
|
||||
}
|
||||
}
|
||||
// Migrate GitHub endpoint type
|
||||
if err := s.migrateGithubEndpointType(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var needsCredentialMigration bool
|
||||
if !s.conn.Migrator().HasTable(&GithubCredentials{}) || !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
|
||||
needsCredentialMigration = true
|
||||
}
|
||||
|
||||
var hasMinAgeField bool
|
||||
if s.conn.Migrator().HasTable(&ControllerInfo{}) && s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff") {
|
||||
hasMinAgeField = true
|
||||
}
|
||||
|
||||
migrateTemplates := !s.conn.Migrator().HasTable(&Template{})
|
||||
// Check if we need to migrate credentials and templates
|
||||
needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL := s.preMigrationChecks()
|
||||
|
||||
// Run main schema migration
|
||||
s.conn.Exec("PRAGMA foreign_keys = OFF")
|
||||
if err := s.conn.AutoMigrate(
|
||||
&User{},
|
||||
|
|
@ -672,30 +774,24 @@ func (s *sqlDatabase) migrateDB() error {
|
|||
|
||||
s.conn.Exec("PRAGMA foreign_keys = ON")
|
||||
|
||||
if !hasMinAgeField {
|
||||
var controller ControllerInfo
|
||||
if err := s.conn.First(&controller).Error; err != nil {
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return fmt.Errorf("error updating controller info: %w", err)
|
||||
}
|
||||
} else {
|
||||
controller.MinimumJobAgeBackoff = 30
|
||||
if err := s.conn.Save(&controller).Error; err != nil {
|
||||
return fmt.Errorf("error updating controller info: %w", err)
|
||||
}
|
||||
}
|
||||
// Migrate controller info if needed
|
||||
if err := s.migrateControllerInfo(hasMinAgeField, hasAgentURL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure github endpoint exists
|
||||
if err := s.ensureGithubEndpoint(); err != nil {
|
||||
return fmt.Errorf("error ensuring github endpoint: %w", err)
|
||||
}
|
||||
|
||||
// Migrate credentials if needed
|
||||
if needsCredentialMigration {
|
||||
if err := s.migrateCredentialsToDB(); err != nil {
|
||||
return fmt.Errorf("error migrating credentials: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure templates exist
|
||||
if err := s.ensureTemplates(migrateTemplates); err != nil {
|
||||
return fmt.Errorf("failed to create default templates: %w", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue