4e3ad41c0b
The variable `runningIdleCount` would get incremented for instances on every pool, instead of only for the pool we are interested in. This change fixes this. Also, adjust the logging message when error occurs in this timeout exceeded scenario. Signed-off-by: Ionut Balutoiu <ibalutoiu@cloudbasesolutions.com>
112 lines
3.1 KiB
Go
112 lines
3.1 KiB
Go
package e2e
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
commonParams "github.com/cloudbase/garm-provider-common/params"
|
|
"github.com/cloudbase/garm/params"
|
|
)
|
|
|
|
func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) {
|
|
var timeWaited time.Duration = 0
|
|
var instance *params.Instance
|
|
|
|
log.Printf("Waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
|
|
for timeWaited < timeout {
|
|
instance, err := getInstance(cli, authToken, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
log.Printf("Instance %s status %s and runner status %s", name, instance.Status, instance.RunnerStatus)
|
|
if instance.Status == status && instance.RunnerStatus == runnerStatus {
|
|
return instance, nil
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
timeWaited += 5 * time.Second
|
|
}
|
|
|
|
if err := printJsonResponse(*instance); err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
|
|
}
|
|
|
|
func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
|
|
var timeWaited time.Duration = 0
|
|
var instance *params.Instance
|
|
|
|
log.Printf("Waiting for instance %s to be removed", name)
|
|
for timeWaited < timeout {
|
|
instances, err := listInstances(cli, authToken)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
instance = nil
|
|
for _, i := range instances {
|
|
if i.Name == name {
|
|
instance = &i
|
|
break
|
|
}
|
|
}
|
|
if instance == nil {
|
|
// The instance is not found in the list. We can safely assume
|
|
// that it is removed
|
|
return nil
|
|
}
|
|
|
|
time.Sleep(5 * time.Second)
|
|
timeWaited += 5 * time.Second
|
|
}
|
|
|
|
if err := printJsonResponse(*instance); err != nil {
|
|
return err
|
|
}
|
|
return fmt.Errorf("instance %s was not removed within the timeout", name)
|
|
}
|
|
|
|
func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
|
|
var timeWaited time.Duration = 0
|
|
var instances params.Instances
|
|
var poolInstances params.Instances
|
|
var err error
|
|
|
|
pool, err := getPool(cli, authToken, poolID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
|
|
for timeWaited < timeout {
|
|
instances, err = listInstances(cli, authToken)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
poolInstances = make(params.Instances, 0)
|
|
runningIdleCount := 0
|
|
for _, instance := range instances {
|
|
if instance.PoolID != poolID {
|
|
continue
|
|
}
|
|
// current instance belongs to the pool we are waiting for
|
|
poolInstances = append(poolInstances, instance)
|
|
if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
|
|
runningIdleCount++
|
|
}
|
|
}
|
|
|
|
log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
|
|
if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
|
|
return nil
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
timeWaited += 5 * time.Second
|
|
}
|
|
|
|
_ = dumpPoolInstancesDetails(pool.ID)
|
|
|
|
return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
|
|
}
|