Retry failed runners

* retry adding runners for up to 5 times if they fail.
  * various fixes

database/sql/instances.go

@@ -175,6 +175,11 @@ func (s *sqlDatabase) UpdateInstance(ctx context.Context, instanceID string, par
 	if string(param.Status) != "" {
 		instance.Status = param.Status
 	}
+	if param.CreateAttempt != 0 {
+		instance.CreateAttempt = param.CreateAttempt
+	}
+
+	instance.ProviderFault = param.ProviderFault
 
 	q := s.conn.Save(&instance)
 	if q.Error != nil {

database/sql/models.go

@@ -114,16 +114,18 @@ type InstanceStatusUpdate struct {
 type Instance struct {
 	Base
 
-	ProviderID   *string `gorm:"uniqueIndex"`
-	Name         string  `gorm:"uniqueIndex"`
-	OSType       config.OSType
-	OSArch       config.OSArch
-	OSName       string
-	OSVersion    string
-	Addresses    []Address `gorm:"foreignKey:InstanceID"`
-	Status       common.InstanceStatus
-	RunnerStatus common.RunnerStatus
-	CallbackURL  string
+	ProviderID    *string `gorm:"uniqueIndex"`
+	Name          string  `gorm:"uniqueIndex"`
+	OSType        config.OSType
+	OSArch        config.OSArch
+	OSName        string
+	OSVersion     string
+	Addresses     []Address `gorm:"foreignKey:InstanceID"`
+	Status        common.InstanceStatus
+	RunnerStatus  common.RunnerStatus
+	CallbackURL   string
+	ProviderFault []byte `gorm:"type:longblob"`
+	CreateAttempt int
 
 	PoolID uuid.UUID
 	Pool   Pool `gorm:"foreignKey:PoolID"`

database/sql/util.go

@@ -41,6 +41,11 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) params.Instance {
 		PoolID:         instance.PoolID.String(),
 		CallbackURL:    instance.CallbackURL,
 		StatusMessages: []params.StatusMessage{},
+		CreateAttempt:  instance.CreateAttempt,
+	}
+
+	if len(instance.ProviderFault) > 0 {
+		ret.ProviderFault = instance.ProviderFault
 	}
 
 	for _, addr := range instance.Addresses {