Allow configuration of job backoff interval

GARM has a backoff interval when consuming queued jobs. This backoff
is intended to allow any potential idle runners to pick up a job before
GARM attempts to spin up a new one. This change allows users to set a
custom backoff interval or disable it altogether by setting it to 0.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>

database/sql/controller.go

@@ -38,6 +38,7 @@ func dbControllerToCommonController(dbInfo ControllerInfo) (params.ControllerInf
 		WebhookURL:           dbInfo.WebhookBaseURL,
 		ControllerWebhookURL: url,
 		CallbackURL:          dbInfo.CallbackURL,
+		MinimumJobAgeBackoff: dbInfo.MinimumJobAgeBackoff,
 	}, nil
 }
 
@@ -70,7 +71,8 @@ func (s *sqlDatabase) InitController() (params.ControllerInfo, error) {
 	}
 
 	newInfo := ControllerInfo{
-		ControllerID: newID,
+		ControllerID:         newID,
+		MinimumJobAgeBackoff: 30,
 	}
 
 	q := s.conn.Save(&newInfo)
@@ -115,6 +117,10 @@ func (s *sqlDatabase) UpdateController(info params.UpdateControllerParams) (para
 			dbInfo.WebhookBaseURL = *info.WebhookURL
 		}
 
+		if info.MinimumJobAgeBackoff != nil {
+			dbInfo.MinimumJobAgeBackoff = *info.MinimumJobAgeBackoff
+		}
+
 		q = tx.Save(&dbInfo)
 		if q.Error != nil {
 			return errors.Wrap(q.Error, "saving controller info")

database/sql/models.go

@@ -211,6 +211,12 @@ type ControllerInfo struct {
 	CallbackURL    string
 	MetadataURL    string
 	WebhookBaseURL string
+	// MinimumJobAgeBackoff is the minimum time that a job must be in the queue
+	// before GARM will attempt to allocate a runner to service it. This backoff
+	// is useful if you have idle runners in various pools that could potentially
+	// pick up the job. GARM would allow this amount of time for runners to react
+	// before spinning up a new one and potentially having to scale down later.
+	MinimumJobAgeBackoff uint
 }
 
 type WorkflowJob struct {

database/sql/sql.go

@@ -407,6 +407,12 @@ func (s *sqlDatabase) migrateDB() error {
 	if !s.conn.Migrator().HasTable(&GithubCredentials{}) || !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
 		needsCredentialMigration = true
 	}
+
+	var hasMinAgeField bool
+	if s.conn.Migrator().HasTable(&ControllerInfo{}) && s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff") {
+		hasMinAgeField = true
+	}
+
 	s.conn.Exec("PRAGMA foreign_keys = OFF")
 	if err := s.conn.AutoMigrate(
 		&User{},
@@ -427,6 +433,20 @@ func (s *sqlDatabase) migrateDB() error {
 	}
 	s.conn.Exec("PRAGMA foreign_keys = ON")
 
+	if !hasMinAgeField {
+		var controller ControllerInfo
+		if err := s.conn.First(&controller).Error; err != nil {
+			if !errors.Is(err, gorm.ErrRecordNotFound) {
+				return errors.Wrap(err, "updating controller info")
+			}
+		} else {
+			controller.MinimumJobAgeBackoff = 30
+			if err := s.conn.Save(&controller).Error; err != nil {
+				return errors.Wrap(err, "updating controller info")
+			}
+		}
+	}
+
 	if err := s.ensureGithubEndpoint(); err != nil {
 		return errors.Wrap(err, "ensuring github endpoint")
 	}