DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Adding garm test suite

Browse source
This commit is contained in:
Fabian Fulga 2024-06-09 23:06:01 +03:00
parent 4c7c9b0e1e
commit 9d4c0a953c
25 changed files with 1672 additions and 1737 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/go-tests.yml vendored
View file

@ -31,7 +31,7 @@ jobs:
go-version: 'stable'
- uses: actions/checkout@v3
- name: make lint
run: make golangci-lint && GOLANGCI_LINT_EXTRA_ARGS="--timeout=8m --build-tags testing" make lint
run: make golangci-lint && GOLANGCI_LINT_EXTRA_ARGS="--timeout=8m --build-tags=testing,integration" make lint
- name: Verify go vendor, go modules and gofmt
run: |
sudo apt-get install -y jq

4
Makefile
View file

@ -59,7 +59,7 @@ release: build-static create-release-files ## Create a release
##@ Lint / Verify
.PHONY: lint
lint: golangci-lint $(GOLANGCI_LINT) ## Run linting.
$(GOLANGCI_LINT) run -v --build-tags testing $(GOLANGCI_LINT_EXTRA_ARGS)
$(GOLANGCI_LINT) run -v --build-tags=testing,integration $(GOLANGCI_LINT_EXTRA_ARGS)
.PHONY: lint-fix
lint-fix: golangci-lint $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported by the linte
@ -84,7 +84,7 @@ integration: build ## Run integration tests
}
trap cleanup EXIT
@./test/integration/scripts/setup-garm.sh
@$(GO) run ./test/integration/main.go
@$(GO) test -v ./test/integration/. -timeout=30m -tags=integration
##@ Development

22
test/integration/e2e/client_utils.go → test/integration/client_utils.go
View file

@ -1,4 +1,4 @@
package e2e
package integration
import (
"github.com/go-openapi/runtime"
@ -67,16 +67,6 @@ func deleteGithubCredentials(apiCli *client.GarmAPI, apiAuthToken runtime.Client
apiAuthToken)
}
func getGithubCredential(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, credentialsID int64) (*params.GithubCredentials, error) {
getCredentialsResponse, err := apiCli.Credentials.GetCredentials(
clientCredentials.NewGetCredentialsParams().WithID(credentialsID),
apiAuthToken)
if err != nil {
return nil, err
}
return &getCredentialsResponse.Payload, nil
}
func updateGithubCredentials(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, credentialsID int64, credentialsParams params.UpdateGithubCredentialsParams) (*params.GithubCredentials, error) {
updateCredentialsResponse, err := apiCli.Credentials.UpdateCredentials(
clientCredentials.NewUpdateCredentialsParams().WithID(credentialsID).WithBody(credentialsParams),
@ -501,16 +491,6 @@ func updatePool(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrite
return &updatePoolResponse.Payload, nil
}
func listPoolInstances(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) (params.Instances, error) {
listPoolInstancesResponse, err := apiCli.Instances.ListPoolInstances(
clientInstances.NewListPoolInstancesParams().WithPoolID(poolID),
apiAuthToken)
if err != nil {
return nil, err
}
return listPoolInstancesResponse.Payload, nil
}
func deletePool(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) error {
return apiCli.Pools.DeletePool(
clientPools.NewDeletePoolParams().WithPoolID(poolID),

233
test/integration/credentials_test.go Normal file
View file

@ -0,0 +1,233 @@
//go:build integration
// +build integration
package integration
import (
"github.com/cloudbase/garm/params"
)
const (
defaultEndpointName string = "github.com"
dummyCredentialsName string = "dummy"
)
func (suite *GarmSuite) TestGithubCredentialsErrorOnDuplicateCredentialsName() {
t := suite.T()
t.Log("Testing error on duplicate credentials name")
creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName)
suite.NoError(err)
t.Cleanup(func() {
suite.DeleteGithubCredential(int64(creds.ID))
})
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with duplicate name")
}
func (suite *GarmSuite) TestGithubCredentialsFailsToDeleteWhenInUse() {
t := suite.T()
t.Log("Testing error when deleting credentials in use")
creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName)
suite.NoError(err)
orgName := "dummy-owner"
repoName := "dummy-repo"
createParams := params.CreateRepoParams{
Owner: orgName,
Name: repoName,
CredentialsName: creds.Name,
WebhookSecret: "superSecret@123BlaBla",
}
t.Logf("Create repository with owner_name: %s, repo_name: %s", orgName, repoName)
repo, err := createRepo(suite.cli, suite.authToken, createParams)
suite.NoError(err)
t.Cleanup(func() {
deleteRepo(suite.cli, suite.authToken, repo.ID)
deleteGithubCredentials(suite.cli, suite.authToken, int64(creds.ID))
})
err = deleteGithubCredentials(suite.cli, suite.authToken, int64(creds.ID))
suite.Error(err, "expected error when deleting credentials in use")
}
func (suite *GarmSuite) TestGithubCredentialsFailsOnInvalidAuthType() {
t := suite.T()
t.Log("Testing error on invalid auth type")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthType("invalid"),
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with invalid auth type")
expectAPIStatusCode(err, 400)
}
func (suite *GarmSuite) TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect() {
t := suite.T()
t.Log("Testing error when auth type params are incorrect")
privateKeyBytes, err := getTestFileContents("certs/srv-key.pem")
suite.NoError(err)
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
App: params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: privateKeyBytes,
},
}
_, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with invalid auth type params")
expectAPIStatusCode(err, 400)
}
func (suite *GarmSuite) TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing() {
t := suite.T()
t.Log("Testing error when auth type params are missing")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypeApp,
}
_, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with missing auth type params")
expectAPIStatusCode(err, 400)
}
func (suite *GarmSuite) TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied() {
t := suite.T()
t.Log("Testing error when both PAT and App are supplied")
creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName)
suite.NoError(err)
t.Cleanup(func() {
suite.DeleteGithubCredential(int64(creds.ID))
})
privateKeyBytes, err := getTestFileContents("certs/srv-key.pem")
suite.NoError(err)
updateCredsParams := params.UpdateGithubCredentialsParams{
PAT: &params.GithubPAT{
OAuth2Token: "dummy",
},
App: &params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: privateKeyBytes,
},
}
_, err = updateGithubCredentials(suite.cli, suite.authToken, int64(creds.ID), updateCredsParams)
suite.Error(err, "expected error when updating credentials with both PAT and App")
expectAPIStatusCode(err, 400)
}
func (suite *GarmSuite) TestGithubCredentialsFailWhenAppKeyIsInvalid() {
t := suite.T()
t.Log("Testing error when app key is invalid")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypeApp,
App: params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: []byte("invalid"),
},
}
_, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with invalid app key")
expectAPIStatusCode(err, 400)
}
func (suite *GarmSuite) TestGithubCredentialsFailWhenEndpointDoesntExist() {
t := suite.T()
t.Log("Testing error when endpoint doesn't exist")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: "iDontExist.example.com",
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err := createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with invalid endpoint")
expectAPIStatusCode(err, 404)
}
func (suite *GarmSuite) TestGithubCredentialsFailsOnDuplicateName() {
t := suite.T()
t.Log("Testing error on duplicate credentials name")
creds, err := suite.createDummyCredentials(dummyCredentialsName, defaultEndpointName)
suite.NoError(err)
t.Cleanup(func() {
suite.DeleteGithubCredential(int64(creds.ID))
})
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err = createGithubCredentials(suite.cli, suite.authToken, createCredsParams)
suite.Error(err, "expected error when creating credentials with duplicate name")
expectAPIStatusCode(err, 409)
}
func (suite *GarmSuite) createDummyCredentials(name, endpointName string) (*params.GithubCredentials, error) {
createCredsParams := params.CreateGithubCredentialsParams{
Name: name,
Endpoint: endpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
return suite.CreateGithubCredentials(createCredsParams)
}
func (suite *GarmSuite) CreateGithubCredentials(credentialsParams params.CreateGithubCredentialsParams) (*params.GithubCredentials, error) {
t := suite.T()
t.Log("Create GitHub credentials")
credentials, err := createGithubCredentials(suite.cli, suite.authToken, credentialsParams)
if err != nil {
return nil, err
}
return credentials, nil
}
func (suite *GarmSuite) DeleteGithubCredential(id int64) error {
t := suite.T()
t.Log("Delete GitHub credential")
if err := deleteGithubCredentials(suite.cli, suite.authToken, id); err != nil {
return err
}
return nil
}

61
test/integration/e2e/client.go
View file

@ -1,61 +0,0 @@
package e2e
import (
"log/slog"
"net/url"
"github.com/go-openapi/runtime"
openapiRuntimeClient "github.com/go-openapi/runtime/client"
"github.com/cloudbase/garm/client"
"github.com/cloudbase/garm/params"
)
var (
cli *client.GarmAPI
authToken runtime.ClientAuthInfoWriter
)
func InitClient(baseURL string) {
garmURL, err := url.Parse(baseURL)
if err != nil {
panic(err)
}
apiPath, err := url.JoinPath(garmURL.Path, client.DefaultBasePath)
if err != nil {
panic(err)
}
transportCfg := client.DefaultTransportConfig().
WithHost(garmURL.Host).
WithBasePath(apiPath).
WithSchemes([]string{garmURL.Scheme})
cli = client.NewHTTPClientWithConfig(nil, transportCfg)
}
func FirstRun(adminUsername, adminPassword, adminFullName, adminEmail string) *params.User {
slog.Info("First run")
newUser := params.NewUserParams{
Username: adminUsername,
Password: adminPassword,
FullName: adminFullName,
Email: adminEmail,
}
user, err := firstRun(cli, newUser)
if err != nil {
panic(err)
}
return &user
}
func Login(username, password string) {
slog.Info("Login")
loginParams := params.PasswordLoginParams{
Username: username,
Password: password,
}
token, err := login(cli, loginParams)
if err != nil {
panic(err)
}
authToken = openapiRuntimeClient.BearerToken(token)
}

206
test/integration/e2e/credentials.go
View file

@ -1,206 +0,0 @@
package e2e
import (
"fmt"
"log/slog"
"github.com/cloudbase/garm/params"
)
func EnsureTestCredentials(name string, oauthToken string, endpointName string) {
slog.Info("Ensuring test credentials exist")
createCredsParams := params.CreateGithubCredentialsParams{
Name: name,
Endpoint: endpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: oauthToken,
},
}
CreateGithubCredentials(createCredsParams)
createCredsParams.Name = fmt.Sprintf("%s-clone", name)
CreateGithubCredentials(createCredsParams)
}
func createDummyCredentials(name, endpointName string) *params.GithubCredentials {
createCredsParams := params.CreateGithubCredentialsParams{
Name: name,
Endpoint: endpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
return CreateGithubCredentials(createCredsParams)
}
func TestGithubCredentialsErrorOnDuplicateCredentialsName() {
slog.Info("Testing error on duplicate credentials name")
creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName)
defer DeleteGithubCredential(int64(creds.ID))
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
if _, err := createGithubCredentials(cli, authToken, createCredsParams); err == nil {
panic("expected error when creating credentials with duplicate name")
}
}
func TestGithubCredentialsFailsToDeleteWhenInUse() {
slog.Info("Testing error when deleting credentials in use")
creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName)
repo := CreateRepo("dummy-owner", "dummy-repo", creds.Name, "superSecret@123BlaBla")
defer func() {
deleteRepo(cli, authToken, repo.ID)
deleteGithubCredentials(cli, authToken, int64(creds.ID))
}()
if err := deleteGithubCredentials(cli, authToken, int64(creds.ID)); err == nil {
panic("expected error when deleting credentials in use")
}
}
func TestGithubCredentialsFailsOnInvalidAuthType() {
slog.Info("Testing error on invalid auth type")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthType("invalid"),
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with invalid auth type")
}
expectAPIStatusCode(err, 400)
}
func TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect() {
slog.Info("Testing error when auth type params are incorrect")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
App: params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: getTestFileContents("certs/srv-key.pem"),
},
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with invalid auth type params")
}
expectAPIStatusCode(err, 400)
}
func TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing() {
slog.Info("Testing error when auth type params are missing")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypeApp,
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with missing auth type params")
}
expectAPIStatusCode(err, 400)
}
func TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied() {
slog.Info("Testing error when both PAT and App are supplied")
creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName)
defer DeleteGithubCredential(int64(creds.ID))
updateCredsParams := params.UpdateGithubCredentialsParams{
PAT: &params.GithubPAT{
OAuth2Token: "dummy",
},
App: &params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: getTestFileContents("certs/srv-key.pem"),
},
}
_, err := updateGithubCredentials(cli, authToken, int64(creds.ID), updateCredsParams)
if err == nil {
panic("expected error when updating credentials with both PAT and App")
}
expectAPIStatusCode(err, 400)
}
func TestGithubCredentialsFailWhenAppKeyIsInvalid() {
slog.Info("Testing error when app key is invalid")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypeApp,
App: params.GithubApp{
AppID: 123,
InstallationID: 456,
PrivateKeyBytes: []byte("invalid"),
},
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with invalid app key")
}
expectAPIStatusCode(err, 400)
}
func TestGithubCredentialsFailWhenEndpointDoesntExist() {
slog.Info("Testing error when endpoint doesn't exist")
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: "iDontExist.example.com",
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with invalid endpoint")
}
expectAPIStatusCode(err, 404)
}
func TestGithubCredentialsFailsOnDuplicateName() {
slog.Info("Testing error on duplicate credentials name")
creds := createDummyCredentials(dummyCredentialsName, defaultEndpointName)
defer DeleteGithubCredential(int64(creds.ID))
createCredsParams := params.CreateGithubCredentialsParams{
Name: dummyCredentialsName,
Endpoint: defaultEndpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: "dummy",
},
}
_, err := createGithubCredentials(cli, authToken, createCredsParams)
if err == nil {
panic("expected error when creating credentials with duplicate name")
}
expectAPIStatusCode(err, 409)
}

206
test/integration/e2e/e2e.go
View file

@ -1,206 +0,0 @@
package e2e
import (
"fmt"
"log/slog"
"os"
"time"
"github.com/cloudbase/garm/params"
)
func ListCredentials() params.Credentials {
slog.Info("List credentials")
credentials, err := listCredentials(cli, authToken)
if err != nil {
panic(err)
}
return credentials
}
func CreateGithubCredentials(credentialsParams params.CreateGithubCredentialsParams) *params.GithubCredentials {
slog.Info("Create GitHub credentials")
credentials, err := createGithubCredentials(cli, authToken, credentialsParams)
if err != nil {
panic(err)
}
return credentials
}
func GetGithubCredential(id int64) *params.GithubCredentials {
slog.Info("Get GitHub credential")
credentials, err := getGithubCredential(cli, authToken, id)
if err != nil {
panic(err)
}
return credentials
}
func DeleteGithubCredential(id int64) {
slog.Info("Delete GitHub credential")
if err := deleteGithubCredentials(cli, authToken, id); err != nil {
panic(err)
}
}
func CreateGithubEndpoint(endpointParams params.CreateGithubEndpointParams) *params.GithubEndpoint {
slog.Info("Create GitHub endpoint")
endpoint, err := createGithubEndpoint(cli, authToken, endpointParams)
if err != nil {
panic(err)
}
return endpoint
}
func ListGithubEndpoints() params.GithubEndpoints {
slog.Info("List GitHub endpoints")
endpoints, err := listGithubEndpoints(cli, authToken)
if err != nil {
panic(err)
}
return endpoints
}
func GetGithubEndpoint(name string) *params.GithubEndpoint {
slog.Info("Get GitHub endpoint")
endpoint, err := getGithubEndpoint(cli, authToken, name)
if err != nil {
panic(err)
}
return endpoint
}
func DeleteGithubEndpoint(name string) {
slog.Info("Delete GitHub endpoint")
if err := deleteGithubEndpoint(cli, authToken, name); err != nil {
panic(err)
}
}
func UpdateGithubEndpoint(name string, updateParams params.UpdateGithubEndpointParams) *params.GithubEndpoint {
slog.Info("Update GitHub endpoint")
updated, err := updateGithubEndpoint(cli, authToken, name, updateParams)
if err != nil {
panic(err)
}
return updated
}
func ListProviders() params.Providers {
slog.Info("List providers")
providers, err := listProviders(cli, authToken)
if err != nil {
panic(err)
}
return providers
}
func GetMetricsToken() {
slog.Info("Get metrics token")
_, err := getMetricsToken(cli, authToken)
if err != nil {
panic(err)
}
}
func GetControllerInfo() *params.ControllerInfo {
slog.Info("Get controller info")
controllerInfo, err := getControllerInfo(cli, authToken)
if err != nil {
panic(err)
}
if err := appendCtrlInfoToGitHubEnv(&controllerInfo); err != nil {
panic(err)
}
if err := printJSONResponse(controllerInfo); err != nil {
panic(err)
}
return &controllerInfo
}
func GracefulCleanup() {
slog.Info("Graceful cleanup")
// disable all the pools
pools, err := listPools(cli, authToken)
if err != nil {
panic(err)
}
enabled := false
poolParams := params.UpdatePoolParams{Enabled: &enabled}
for _, pool := range pools {
if _, err := updatePool(cli, authToken, pool.ID, poolParams); err != nil {
panic(err)
}
slog.Info("Pool disabled", "pool_id", pool.ID, "stage", "graceful_cleanup")
}
// delete all the instances
for _, pool := range pools {
poolInstances, err := listPoolInstances(cli, authToken, pool.ID)
if err != nil {
panic(err)
}
for _, instance := range poolInstances {
if err := deleteInstance(cli, authToken, instance.Name, false, false); err != nil {
panic(err)
}
slog.Info("Instance deletion initiated", "instance", instance.Name, "stage", "graceful_cleanup")
}
}
// wait for all instances to be deleted
for _, pool := range pools {
if err := waitPoolNoInstances(pool.ID, 3*time.Minute); err != nil {
panic(err)
}
}
// delete all the pools
for _, pool := range pools {
if err := deletePool(cli, authToken, pool.ID); err != nil {
panic(err)
}
slog.Info("Pool deleted", "pool_id", pool.ID, "stage", "graceful_cleanup")
}
// delete all the repositories
repos, err := listRepos(cli, authToken)
if err != nil {
panic(err)
}
for _, repo := range repos {
if err := deleteRepo(cli, authToken, repo.ID); err != nil {
panic(err)
}
slog.Info("Repo deleted", "repo_id", repo.ID, "stage", "graceful_cleanup")
}
// delete all the organizations
orgs, err := listOrgs(cli, authToken)
if err != nil {
panic(err)
}
for _, org := range orgs {
if err := deleteOrg(cli, authToken, org.ID); err != nil {
panic(err)
}
slog.Info("Org deleted", "org_id", org.ID, "stage", "graceful_cleanup")
}
}
func appendCtrlInfoToGitHubEnv(controllerInfo *params.ControllerInfo) error {
envFile, found := os.LookupEnv("GITHUB_ENV")
if !found {
slog.Info("GITHUB_ENV not set, skipping appending controller info")
return nil
}
file, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
if err != nil {
return err
}
defer file.Close()
if _, err := file.WriteString(fmt.Sprintf("export GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil {
return err
}
return nil
}

247
test/integration/e2e/endpoints.go
View file

@ -1,247 +0,0 @@
package e2e
import (
"log/slog"
"os"
"path/filepath"
"github.com/cloudbase/garm/params"
)
const (
defaultEndpointName string = "github.com"
dummyCredentialsName string = "dummy"
)
func MustDefaultGithubEndpoint() {
ep := GetGithubEndpoint("github.com")
if ep == nil {
panic("Default GitHub endpoint not found")
}
if ep.Name != "github.com" {
panic("Default GitHub endpoint name mismatch")
}
}
func checkEndpointParamsAreEqual(a, b params.GithubEndpoint) {
if a.Name != b.Name {
panic("Endpoint name mismatch")
}
if a.Description != b.Description {
panic("Endpoint description mismatch")
}
if a.BaseURL != b.BaseURL {
panic("Endpoint base URL mismatch")
}
if a.APIBaseURL != b.APIBaseURL {
panic("Endpoint API base URL mismatch")
}
if a.UploadBaseURL != b.UploadBaseURL {
panic("Endpoint upload base URL mismatch")
}
if string(a.CACertBundle) != string(b.CACertBundle) {
panic("Endpoint CA cert bundle mismatch")
}
}
func getTestFileContents(relPath string) []byte {
baseDir := os.Getenv("GARM_CHECKOUT_DIR")
if baseDir == "" {
panic("GARM_CHECKOUT_DIR not set")
}
contents, err := os.ReadFile(filepath.Join(baseDir, "testdata", relPath))
if err != nil {
panic(err)
}
return contents
}
func TestGithubEndpointOperations() {
slog.Info("Testing endpoint operations")
MustDefaultGithubEndpoint()
caBundle := getTestFileContents("certs/srv-pub.pem")
endpointParams := params.CreateGithubEndpointParams{
Name: "test-endpoint",
Description: "Test endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
CACertBundle: caBundle,
}
endpoint := CreateGithubEndpoint(endpointParams)
if endpoint.Name != endpointParams.Name {
panic("Endpoint name mismatch")
}
if endpoint.Description != endpointParams.Description {
panic("Endpoint description mismatch")
}
if endpoint.BaseURL != endpointParams.BaseURL {
panic("Endpoint base URL mismatch")
}
if endpoint.APIBaseURL != endpointParams.APIBaseURL {
panic("Endpoint API base URL mismatch")
}
if endpoint.UploadBaseURL != endpointParams.UploadBaseURL {
panic("Endpoint upload base URL mismatch")
}
if string(endpoint.CACertBundle) != string(caBundle) {
panic("Endpoint CA cert bundle mismatch")
}
endpoint2 := GetGithubEndpoint(endpointParams.Name)
if endpoint == nil || endpoint2 == nil {
panic("endpoint is nil")
}
checkEndpointParamsAreEqual(*endpoint, *endpoint2)
endpoints := ListGithubEndpoints()
var found bool
for _, ep := range endpoints {
if ep.Name == endpointParams.Name {
checkEndpointParamsAreEqual(*endpoint, ep)
found = true
break
}
}
if !found {
panic("Endpoint not found in list")
}
DeleteGithubEndpoint(endpoint.Name)
}
func TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint() {
slog.Info("Testing error when deleting default github.com endpoint")
if err := deleteGithubEndpoint(cli, authToken, "github.com"); err == nil {
panic("expected error when attempting to delete the default github.com endpoint")
}
}
func TestGithubEndpointFailsOnInvalidCABundle() {
slog.Info("Testing endpoint creation with invalid CA cert bundle")
badCABundle := getTestFileContents("certs/srv-key.pem")
endpointParams := params.CreateGithubEndpointParams{
Name: "dummy",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
CACertBundle: badCABundle,
}
if _, err := createGithubEndpoint(cli, authToken, endpointParams); err == nil {
panic("expected error when creating endpoint with invalid CA cert bundle")
}
}
func TestGithubEndpointDeletionFailsWhenCredentialsExist() {
slog.Info("Testing endpoint deletion when credentials exist")
endpointParams := params.CreateGithubEndpointParams{
Name: "dummy",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
endpoint := CreateGithubEndpoint(endpointParams)
creds := createDummyCredentials("test-creds", endpoint.Name)
if err := deleteGithubEndpoint(cli, authToken, endpoint.Name); err == nil {
panic("expected error when deleting endpoint with credentials")
}
DeleteGithubCredential(int64(creds.ID))
DeleteGithubEndpoint(endpoint.Name)
}
func TestGithubEndpointFailsOnDuplicateName() {
slog.Info("Testing endpoint creation with duplicate name")
endpointParams := params.CreateGithubEndpointParams{
Name: "github.com",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
if _, err := createGithubEndpoint(cli, authToken, endpointParams); err == nil {
panic("expected error when creating endpoint with duplicate name")
}
}
func TestGithubEndpointUpdateEndpoint() {
slog.Info("Testing endpoint update")
endpoint := createDummyEndpoint("dummy")
defer DeleteGithubEndpoint(endpoint.Name)
newDescription := "Updated description"
newBaseURL := "https://ghes2.example.com"
newAPIBaseURL := "https://api.ghes2.example.com/"
newUploadBaseURL := "https://uploads.ghes2.example.com/"
newCABundle := getTestFileContents("certs/srv-pub.pem")
updateParams := params.UpdateGithubEndpointParams{
Description: &newDescription,
BaseURL: &newBaseURL,
APIBaseURL: &newAPIBaseURL,
UploadBaseURL: &newUploadBaseURL,
CACertBundle: newCABundle,
}
updated, err := updateGithubEndpoint(cli, authToken, endpoint.Name, updateParams)
if err != nil {
panic(err)
}
if updated.Name != endpoint.Name {
panic("Endpoint name mismatch")
}
if updated.Description != newDescription {
panic("Endpoint description mismatch")
}
if updated.BaseURL != newBaseURL {
panic("Endpoint base URL mismatch")
}
if updated.APIBaseURL != newAPIBaseURL {
panic("Endpoint API base URL mismatch")
}
if updated.UploadBaseURL != newUploadBaseURL {
panic("Endpoint upload base URL mismatch")
}
if string(updated.CACertBundle) != string(newCABundle) {
panic("Endpoint CA cert bundle mismatch")
}
}
func createDummyEndpoint(name string) *params.GithubEndpoint {
endpointParams := params.CreateGithubEndpointParams{
Name: name,
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
return CreateGithubEndpoint(endpointParams)
}

203
test/integration/e2e/github_client_utils.go
View file

@ -1,203 +0,0 @@
package e2e
import (
"context"
"fmt"
"log/slog"
"github.com/google/go-github/v57/github"
"golang.org/x/oauth2"
)
func TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, labelName string) {
slog.Info("Trigger workflow", "label", labelName)
client := getGithubClient(ghToken)
eventReq := github.CreateWorkflowDispatchEventRequest{
Ref: "main",
Inputs: map[string]interface{}{
"sleep_time": "50",
"runner_label": labelName,
},
}
if _, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), orgName, repoName, workflowFileName, eventReq); err != nil {
panic(err)
}
}
func GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error {
slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName)
client := getGithubClient(ghToken)
ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil)
if err != nil {
return err
}
// Remove organization runners
controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID)
for _, orgRunner := range ghOrgRunners.Runners {
for _, label := range orgRunner.Labels {
if label.GetName() == controllerLabel {
if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil {
// We don't fail if we can't remove a single runner. This
// is a best effort to try and remove all the orphan runners.
slog.With(slog.Any("error", err)).Info("Failed to remove organization runner", "org_runner", orgRunner.GetName())
break
}
slog.Info("Removed organization runner", "org_runner", orgRunner.GetName())
break
}
}
}
return nil
}
func GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID string) error {
slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName, "repo_name", repoName)
client := getGithubClient(ghToken)
ghRepoRunners, _, err := client.Actions.ListRunners(context.Background(), orgName, repoName, nil)
if err != nil {
return err
}
// Remove repository runners
controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID)
for _, repoRunner := range ghRepoRunners.Runners {
for _, label := range repoRunner.Labels {
if label.GetName() == controllerLabel {
if _, err := client.Actions.RemoveRunner(context.Background(), orgName, repoName, repoRunner.GetID()); err != nil {
// We don't fail if we can't remove a single runner. This
// is a best effort to try and remove all the orphan runners.
slog.With(slog.Any("error", err)).Error("Failed to remove repository runner", "runner_name", repoRunner.GetName())
break
}
slog.Info("Removed repository runner", "runner_name", repoRunner.GetName())
break
}
}
}
return nil
}
func ValidateOrgWebhookInstalled(ghToken, url, orgName string) {
hook, err := getGhOrgWebhook(url, ghToken, orgName)
if err != nil {
panic(err)
}
if hook == nil {
panic(fmt.Errorf("github webhook with url %s, for org %s was not properly installed", url, orgName))
}
}
func ValidateOrgWebhookUninstalled(ghToken, url, orgName string) {
hook, err := getGhOrgWebhook(url, ghToken, orgName)
if err != nil {
panic(err)
}
if hook != nil {
panic(fmt.Errorf("github webhook with url %s, for org %s was not properly uninstalled", url, orgName))
}
}
func ValidateRepoWebhookInstalled(ghToken, url, orgName, repoName string) {
hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName)
if err != nil {
panic(err)
}
if hook == nil {
panic(fmt.Errorf("github webhook with url %s, for repo %s/%s was not properly installed", url, orgName, repoName))
}
}
func ValidateRepoWebhookUninstalled(ghToken, url, orgName, repoName string) {
hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName)
if err != nil {
panic(err)
}
if hook != nil {
panic(fmt.Errorf("github webhook with url %s, for repo %s/%s was not properly uninstalled", url, orgName, repoName))
}
}
func GhOrgWebhookCleanup(ghToken, webhookURL, orgName string) error {
slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName)
hook, err := getGhOrgWebhook(webhookURL, ghToken, orgName)
if err != nil {
return err
}
// Remove organization webhook
if hook != nil {
client := getGithubClient(ghToken)
if _, err := client.Organizations.DeleteHook(context.Background(), orgName, hook.GetID()); err != nil {
return err
}
slog.Info("Github webhook removed", "webhook_url", webhookURL, "org_name", orgName)
}
return nil
}
func GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName string) error {
slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName)
hook, err := getGhRepoWebhook(webhookURL, ghToken, orgName, repoName)
if err != nil {
return err
}
// Remove repository webhook
if hook != nil {
client := getGithubClient(ghToken)
if _, err := client.Repositories.DeleteHook(context.Background(), orgName, repoName, hook.GetID()); err != nil {
return err
}
slog.Info("Github webhook with", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName)
}
return nil
}
func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghOrgHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghRepoHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func getGithubClient(oauthToken string) *github.Client {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken})
tc := oauth2.NewClient(context.Background(), ts)
return github.NewClient(tc)
}

119
test/integration/e2e/instances.go
View file

@ -1,119 +0,0 @@
package e2e
import (
"fmt"
"log/slog"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) {
var timeWaited time.Duration // default is 0
var instance *params.Instance
var err error
slog.Info("Waiting for instance to reach desired status", "instance", name, "desired_status", status, "desired_runner_status", runnerStatus)
for timeWaited < timeout {
instance, err = getInstance(cli, authToken, name)
if err != nil {
return nil, err
}
slog.Info("Instance status", "instance_name", name, "status", instance.Status, "runner_status", instance.RunnerStatus)
if instance.Status == status && instance.RunnerStatus == runnerStatus {
return instance, nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*instance); err != nil {
return nil, err
}
return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
}
func DeleteInstance(name string, forceRemove, bypassGHUnauthorized bool) {
slog.Info("Delete instance", "instance_name", name, "force_remove", forceRemove)
if err := deleteInstance(cli, authToken, name, forceRemove, bypassGHUnauthorized); err != nil {
slog.Error("Failed to delete instance", "instance_name", name, "error", err)
panic(err)
}
slog.Info("Instance deletion initiated", "instance_name", name)
}
func WaitInstanceToBeRemoved(name string, timeout time.Duration) error {
var timeWaited time.Duration // default is 0
var instance *params.Instance
slog.Info("Waiting for instance to be removed", "instance_name", name)
for timeWaited < timeout {
instances, err := listInstances(cli, authToken)
if err != nil {
return err
}
instance = nil
for k, v := range instances {
if v.Name == name {
instance = &instances[k]
break
}
}
if instance == nil {
// The instance is not found in the list. We can safely assume
// that it is removed
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*instance); err != nil {
return err
}
return fmt.Errorf("instance %s was not removed within the timeout", name)
}
func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error {
var timeWaited time.Duration // default is 0
pool, err := getPool(cli, authToken, poolID)
if err != nil {
return err
}
slog.Info("Waiting for pool instances to reach desired status", "pool_id", poolID, "desired_status", status, "desired_runner_status", runnerStatus)
for timeWaited < timeout {
poolInstances, err := listPoolInstances(cli, authToken, poolID)
if err != nil {
return err
}
instancesCount := 0
for _, instance := range poolInstances {
if instance.Status == status && instance.RunnerStatus == runnerStatus {
instancesCount++
}
}
slog.Info(
"Pool instance reached status",
"pool_id", poolID,
"status", status,
"runner_status", runnerStatus,
"desired_instance_count", instancesCount,
"pool_instance_count", len(poolInstances))
if int(pool.MinIdleRunners) == instancesCount {
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
_ = dumpPoolInstancesDetails(pool.ID)
return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus)
}

123
test/integration/e2e/jobs.go
View file

@ -1,123 +0,0 @@
package e2e
import (
"fmt"
"log/slog"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func ValidateJobLifecycle(label string) {
slog.Info("Validate GARM job lifecycle", "label", label)
// wait for job list to be updated
job, err := waitLabelledJob(label, 4*time.Minute)
if err != nil {
panic(err)
}
// check expected job status
job, err = waitJobStatus(job.ID, params.JobStatusQueued, 4*time.Minute)
if err != nil {
panic(err)
}
job, err = waitJobStatus(job.ID, params.JobStatusInProgress, 4*time.Minute)
if err != nil {
panic(err)
}
// check expected instance status
instance, err := waitInstanceStatus(job.RunnerName, commonParams.InstanceRunning, params.RunnerActive, 5*time.Minute)
if err != nil {
panic(err)
}
// wait for job to be completed
_, err = waitJobStatus(job.ID, params.JobStatusCompleted, 4*time.Minute)
if err != nil {
panic(err)
}
// wait for instance to be removed
err = WaitInstanceToBeRemoved(instance.Name, 5*time.Minute)
if err != nil {
panic(err)
}
// wait for GARM to rebuild the pool running idle instances
err = WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute)
if err != nil {
panic(err)
}
}
func waitLabelledJob(label string, timeout time.Duration) (*params.Job, error) {
var timeWaited time.Duration // default is 0
var jobs params.Jobs
var err error
slog.Info("Waiting for job", "label", label)
for timeWaited < timeout {
jobs, err = listJobs(cli, authToken)
if err != nil {
return nil, err
}
for _, job := range jobs {
for _, jobLabel := range job.Labels {
if jobLabel == label {
return &job, err
}
}
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(jobs); err != nil {
return nil, err
}
return nil, fmt.Errorf("failed to wait job with label %s", label)
}
func waitJobStatus(id int64, status params.JobStatus, timeout time.Duration) (*params.Job, error) {
var timeWaited time.Duration // default is 0
var job *params.Job
slog.Info("Waiting for job to reach status", "job_id", id, "status", status)
for timeWaited < timeout {
jobs, err := listJobs(cli, authToken)
if err != nil {
return nil, err
}
job = nil
for k, v := range jobs {
if v.ID == id {
job = &jobs[k]
break
}
}
if job == nil {
if status == params.JobStatusCompleted {
// The job is not found in the list. We can safely assume
// that it is completed
return nil, nil
}
// if the job is not found, and expected status is not "completed",
// we need to error out.
return nil, fmt.Errorf("job %d not found, expected to be found in status %s", id, status)
} else if job.Status == string(status) {
return job, nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*job); err != nil {
return nil, err
}
return nil, fmt.Errorf("timeout waiting for job %d to reach status %s", id, status)
}

140
test/integration/e2e/organizations.go
View file

@ -1,140 +0,0 @@
package e2e
import (
"log/slog"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func CreateOrg(orgName, credentialsName, orgWebhookSecret string) *params.Organization {
slog.Info("Create org", "org_name", orgName)
orgParams := params.CreateOrgParams{
Name: orgName,
CredentialsName: credentialsName,
WebhookSecret: orgWebhookSecret,
}
org, err := createOrg(cli, authToken, orgParams)
if err != nil {
panic(err)
}
return org
}
func UpdateOrg(id, credentialsName string) *params.Organization {
slog.Info("Update org", "org_id", id)
updateParams := params.UpdateEntityParams{
CredentialsName: credentialsName,
}
org, err := updateOrg(cli, authToken, id, updateParams)
if err != nil {
panic(err)
}
return org
}
func InstallOrgWebhook(id string) *params.HookInfo {
slog.Info("Install org webhook", "org_id", id)
webhookParams := params.InstallWebhookParams{
WebhookEndpointType: params.WebhookEndpointDirect,
}
_, err := installOrgWebhook(cli, authToken, id, webhookParams)
if err != nil {
panic(err)
}
webhookInfo, err := getOrgWebhook(cli, authToken, id)
if err != nil {
panic(err)
}
return webhookInfo
}
func UninstallOrgWebhook(id string) {
slog.Info("Uninstall org webhook", "org_id", id)
if err := uninstallOrgWebhook(cli, authToken, id); err != nil {
panic(err)
}
}
func CreateOrgPool(orgID string, poolParams params.CreatePoolParams) *params.Pool {
slog.Info("Create org pool", "org_id", orgID)
pool, err := createOrgPool(cli, authToken, orgID, poolParams)
if err != nil {
panic(err)
}
return pool
}
func GetOrgPool(orgID, orgPoolID string) *params.Pool {
slog.Info("Get org pool", "org_id", orgID, "pool_id", orgPoolID)
pool, err := getOrgPool(cli, authToken, orgID, orgPoolID)
if err != nil {
panic(err)
}
return pool
}
func UpdateOrgPool(orgID, orgPoolID string, maxRunners, minIdleRunners uint) *params.Pool {
slog.Info("Update org pool", "org_id", orgID, "pool_id", orgPoolID)
poolParams := params.UpdatePoolParams{
MinIdleRunners: &minIdleRunners,
MaxRunners: &maxRunners,
}
pool, err := updateOrgPool(cli, authToken, orgID, orgPoolID, poolParams)
if err != nil {
panic(err)
}
return pool
}
func DeleteOrgPool(orgID, orgPoolID string) {
slog.Info("Delete org pool", "org_id", orgID, "pool_id", orgPoolID)
if err := deleteOrgPool(cli, authToken, orgID, orgPoolID); err != nil {
panic(err)
}
}
func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
orgPools, err := listOrgPools(cli, authToken, orgID)
if err != nil {
panic(err)
}
for _, pool := range orgPools {
err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
if err != nil {
_ = dumpOrgInstancesDetails(orgID)
panic(err)
}
}
}
func dumpOrgInstancesDetails(orgID string) error {
// print org details
slog.Info("Dumping org details", "org_id", orgID)
org, err := getOrg(cli, authToken, orgID)
if err != nil {
return err
}
if err := printJSONResponse(org); err != nil {
return err
}
// print org instances details
slog.Info("Dumping org instances details", "org_id", orgID)
instances, err := listOrgInstances(cli, authToken, orgID)
if err != nil {
return err
}
for _, instance := range instances {
instance, err := getInstance(cli, authToken, instance.Name)
if err != nil {
return err
}
slog.Info("Instance info", "instance_name", instance.Name)
if err := printJSONResponse(instance); err != nil {
return err
}
}
return nil
}

54
test/integration/e2e/pools.go
View file

@ -1,54 +0,0 @@
package e2e
import (
"fmt"
"log/slog"
"time"
"github.com/cloudbase/garm/params"
)
func waitPoolNoInstances(id string, timeout time.Duration) error {
var timeWaited time.Duration // default is 0
var pool *params.Pool
var err error
slog.Info("Wait until pool has no instances", "pool_id", id)
for timeWaited < timeout {
pool, err = getPool(cli, authToken, id)
if err != nil {
return err
}
slog.Info("Current pool instances", "instance_count", len(pool.Instances))
if len(pool.Instances) == 0 {
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
_ = dumpPoolInstancesDetails(pool.ID)
return fmt.Errorf("failed to wait for pool %s to have no instances", pool.ID)
}
func dumpPoolInstancesDetails(poolID string) error {
pool, err := getPool(cli, authToken, poolID)
if err != nil {
return err
}
if err := printJSONResponse(pool); err != nil {
return err
}
for _, instance := range pool.Instances {
instanceDetails, err := getInstance(cli, authToken, instance.Name)
if err != nil {
return err
}
slog.Info("Instance details", "instance_name", instance.Name)
if err := printJSONResponse(instanceDetails); err != nil {
return err
}
}
return nil
}

152
test/integration/e2e/repositories.go
View file

@ -1,152 +0,0 @@
package e2e
import (
"log/slog"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func CreateRepo(orgName, repoName, credentialsName, repoWebhookSecret string) *params.Repository {
slog.Info("Create repository", "owner_name", orgName, "repo_name", repoName)
createParams := params.CreateRepoParams{
Owner: orgName,
Name: repoName,
CredentialsName: credentialsName,
WebhookSecret: repoWebhookSecret,
}
repo, err := createRepo(cli, authToken, createParams)
if err != nil {
panic(err)
}
return repo
}
func UpdateRepo(id, credentialsName string) *params.Repository {
slog.Info("Update repo", "repo_id", id)
updateParams := params.UpdateEntityParams{
CredentialsName: credentialsName,
}
repo, err := updateRepo(cli, authToken, id, updateParams)
if err != nil {
panic(err)
}
return repo
}
func InstallRepoWebhook(id string) *params.HookInfo {
slog.Info("Install repo webhook", "repo_id", id)
webhookParams := params.InstallWebhookParams{
WebhookEndpointType: params.WebhookEndpointDirect,
}
_, err := installRepoWebhook(cli, authToken, id, webhookParams)
if err != nil {
slog.Error("Failed to install repo webhook", "error", err)
panic(err)
}
webhookInfo, err := getRepoWebhook(cli, authToken, id)
if err != nil {
panic(err)
}
return webhookInfo
}
func UninstallRepoWebhook(id string) {
slog.Info("Uninstall repo webhook", "repo_id", id)
if err := uninstallRepoWebhook(cli, authToken, id); err != nil {
panic(err)
}
}
func CreateRepoPool(repoID string, poolParams params.CreatePoolParams) *params.Pool {
slog.Info("Create repo pool", "repo_id", repoID, "pool_params", poolParams)
pool, err := createRepoPool(cli, authToken, repoID, poolParams)
if err != nil {
slog.Error("Failed to create repo pool", "error", err)
panic(err)
}
return pool
}
func GetRepoPool(repoID, repoPoolID string) *params.Pool {
slog.Info("Get repo pool", "repo_id", repoID, "pool_id", repoPoolID)
pool, err := getRepoPool(cli, authToken, repoID, repoPoolID)
if err != nil {
panic(err)
}
return pool
}
func UpdateRepoPool(repoID, repoPoolID string, maxRunners, minIdleRunners uint) *params.Pool {
slog.Info("Update repo pool", "repo_id", repoID, "pool_id", repoPoolID)
poolParams := params.UpdatePoolParams{
MinIdleRunners: &minIdleRunners,
MaxRunners: &maxRunners,
}
pool, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams)
if err != nil {
panic(err)
}
return pool
}
func DeleteRepoPool(repoID, repoPoolID string) {
slog.Info("Delete repo pool", "repo_id", repoID, "pool_id", repoPoolID)
if err := deleteRepoPool(cli, authToken, repoID, repoPoolID); err != nil {
panic(err)
}
}
func DisableRepoPool(repoID, repoPoolID string) {
slog.Info("Disable repo pool", "repo_id", repoID, "pool_id", repoPoolID)
enabled := false
poolParams := params.UpdatePoolParams{Enabled: &enabled}
if _, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams); err != nil {
panic(err)
}
}
func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
repoPools, err := listRepoPools(cli, authToken, repoID)
if err != nil {
panic(err)
}
for _, pool := range repoPools {
err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
if err != nil {
_ = dumpRepoInstancesDetails(repoID)
panic(err)
}
}
}
func dumpRepoInstancesDetails(repoID string) error {
// print repo details
slog.Info("Dumping repo details", "repo_id", repoID)
repo, err := getRepo(cli, authToken, repoID)
if err != nil {
return err
}
if err := printJSONResponse(repo); err != nil {
return err
}
// print repo instances details
slog.Info("Dumping repo instances details", "repo_id", repoID)
instances, err := listRepoInstances(cli, authToken, repoID)
if err != nil {
return err
}
for _, instance := range instances {
instance, err := getInstance(cli, authToken, instance.Name)
if err != nil {
return err
}
slog.Info("Instance info", "instance_name", instance.Name)
if err := printJSONResponse(instance); err != nil {
return err
}
}
return nil
}

48
test/integration/endpoints.go Normal file
View file

@ -0,0 +1,48 @@
package integration
import (
"fmt"
"os"
"path/filepath"
"github.com/cloudbase/garm/params"
)
func checkEndpointParamsAreEqual(a, b params.GithubEndpoint) error {
if a.Name != b.Name {
return fmt.Errorf("endpoint name mismatch")
}
if a.Description != b.Description {
return fmt.Errorf("endpoint description mismatch")
}
if a.BaseURL != b.BaseURL {
return fmt.Errorf("endpoint base URL mismatch")
}
if a.APIBaseURL != b.APIBaseURL {
return fmt.Errorf("endpoint API base URL mismatch")
}
if a.UploadBaseURL != b.UploadBaseURL {
return fmt.Errorf("endpoint upload base URL mismatch")
}
if string(a.CACertBundle) != string(b.CACertBundle) {
return fmt.Errorf("endpoint CA cert bundle mismatch")
}
return nil
}
func getTestFileContents(relPath string) ([]byte, error) {
baseDir := os.Getenv("GARM_CHECKOUT_DIR")
if baseDir == "" {
return nil, fmt.Errorf("ariable GARM_CHECKOUT_DIR not set")
}
contents, err := os.ReadFile(filepath.Join(baseDir, "testdata", relPath))
if err != nil {
return nil, err
}
return contents, nil
}

212
test/integration/endpoints_test.go Normal file
View file

@ -0,0 +1,212 @@
//go:build integration
// +build integration
package integration
import (
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) TestGithubEndpointOperations() {
t := suite.T()
t.Log("Testing endpoint operations")
suite.MustDefaultGithubEndpoint()
caBundle, err := getTestFileContents("certs/srv-pub.pem")
suite.NoError(err)
endpointParams := params.CreateGithubEndpointParams{
Name: "test-endpoint",
Description: "Test endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
CACertBundle: caBundle,
}
endpoint, err := suite.CreateGithubEndpoint(endpointParams)
suite.NoError(err)
suite.Equal(endpoint.Name, endpointParams.Name, "Endpoint name mismatch")
suite.Equal(endpoint.Description, endpointParams.Description, "Endpoint description mismatch")
suite.Equal(endpoint.BaseURL, endpointParams.BaseURL, "Endpoint base URL mismatch")
suite.Equal(endpoint.APIBaseURL, endpointParams.APIBaseURL, "Endpoint API base URL mismatch")
suite.Equal(endpoint.UploadBaseURL, endpointParams.UploadBaseURL, "Endpoint upload base URL mismatch")
suite.Equal(string(endpoint.CACertBundle), string(caBundle), "Endpoint CA cert bundle mismatch")
endpoint2 := suite.GetGithubEndpoint(endpointParams.Name)
suite.NotNil(endpoint, "endpoint is nil")
suite.NotNil(endpoint2, "endpoint2 is nil")
err = checkEndpointParamsAreEqual(*endpoint, *endpoint2)
suite.NoError(err, "endpoint params are not equal")
endpoints := suite.ListGithubEndpoints()
suite.NoError(err, "error listing github endpoints")
var found bool
for _, ep := range endpoints {
if ep.Name == endpointParams.Name {
checkEndpointParamsAreEqual(*endpoint, ep)
found = true
break
}
}
suite.Equal(found, true, "endpoint not found in list")
err = suite.DeleteGithubEndpoint(endpoint.Name)
suite.NoError(err, "error deleting github endpoint")
}
func (suite *GarmSuite) TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint() {
t := suite.T()
t.Log("Testing error when deleting default github.com endpoint")
err := deleteGithubEndpoint(suite.cli, suite.authToken, "github.com")
suite.Error(err, "expected error when attempting to delete the default github.com endpoint")
}
func (suite *GarmSuite) TestGithubEndpointFailsOnInvalidCABundle() {
t := suite.T()
t.Log("Testing endpoint creation with invalid CA cert bundle")
badCABundle, err := getTestFileContents("certs/srv-key.pem")
suite.NoError(err, "error reading CA cert bundle")
endpointParams := params.CreateGithubEndpointParams{
Name: "dummy",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
CACertBundle: badCABundle,
}
_, err = createGithubEndpoint(suite.cli, suite.authToken, endpointParams)
suite.Error(err, "expected error when creating endpoint with invalid CA cert bundle")
}
func (suite *GarmSuite) TestGithubEndpointDeletionFailsWhenCredentialsExist() {
t := suite.T()
t.Log("Testing endpoint deletion when credentials exist")
endpointParams := params.CreateGithubEndpointParams{
Name: "dummy",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
endpoint, err := suite.CreateGithubEndpoint(endpointParams)
suite.NoError(err, "error creating github endpoint")
creds, err := suite.createDummyCredentials("test-creds", endpoint.Name)
suite.NoError(err, "error creating dummy credentials")
err = deleteGithubEndpoint(suite.cli, suite.authToken, endpoint.Name)
suite.Error(err, "expected error when deleting endpoint with credentials")
err = suite.DeleteGithubCredential(int64(creds.ID))
suite.NoError(err, "error deleting credentials")
err = suite.DeleteGithubEndpoint(endpoint.Name)
suite.NoError(err, "error deleting endpoint")
}
func (suite *GarmSuite) TestGithubEndpointFailsOnDuplicateName() {
t := suite.T()
t.Log("Testing endpoint creation with duplicate name")
endpointParams := params.CreateGithubEndpointParams{
Name: "github.com",
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
_, err := createGithubEndpoint(suite.cli, suite.authToken, endpointParams)
suite.Error(err, "expected error when creating endpoint with duplicate name")
}
func (suite *GarmSuite) TestGithubEndpointUpdateEndpoint() {
t := suite.T()
t.Log("Testing endpoint update")
endpoint, err := suite.createDummyEndpoint("dummy")
suite.NoError(err, "error creating dummy endpoint")
t.Cleanup(func() {
suite.DeleteGithubEndpoint(endpoint.Name)
})
newDescription := "Updated description"
newBaseURL := "https://ghes2.example.com"
newAPIBaseURL := "https://api.ghes2.example.com/"
newUploadBaseURL := "https://uploads.ghes2.example.com/"
newCABundle, err := getTestFileContents("certs/srv-pub.pem")
suite.NoError(err, "error reading CA cert bundle")
updateParams := params.UpdateGithubEndpointParams{
Description: &newDescription,
BaseURL: &newBaseURL,
APIBaseURL: &newAPIBaseURL,
UploadBaseURL: &newUploadBaseURL,
CACertBundle: newCABundle,
}
updated, err := updateGithubEndpoint(suite.cli, suite.authToken, endpoint.Name, updateParams)
suite.NoError(err, "error updating github endpoint")
suite.Equal(updated.Name, endpoint.Name, "Endpoint name mismatch")
suite.Equal(updated.Description, newDescription, "Endpoint description mismatch")
suite.Equal(updated.BaseURL, newBaseURL, "Endpoint base URL mismatch")
suite.Equal(updated.APIBaseURL, newAPIBaseURL, "Endpoint API base URL mismatch")
suite.Equal(updated.UploadBaseURL, newUploadBaseURL, "Endpoint upload base URL mismatch")
suite.Equal(string(updated.CACertBundle), string(newCABundle), "Endpoint CA cert bundle mismatch")
}
func (suite *GarmSuite) MustDefaultGithubEndpoint() {
ep := suite.GetGithubEndpoint("github.com")
suite.NotNil(ep, "default GitHub endpoint not found")
suite.Equal(ep.Name, "github.com", "default GitHub endpoint name mismatch")
}
func (suite *GarmSuite) GetGithubEndpoint(name string) *params.GithubEndpoint {
t := suite.T()
t.Log("Get GitHub endpoint")
endpoint, err := getGithubEndpoint(suite.cli, suite.authToken, name)
suite.NoError(err, "error getting GitHub endpoint")
return endpoint
}
func (suite *GarmSuite) CreateGithubEndpoint(params params.CreateGithubEndpointParams) (*params.GithubEndpoint, error) {
t := suite.T()
t.Log("Create GitHub endpoint")
endpoint, err := createGithubEndpoint(suite.cli, suite.authToken, params)
suite.NoError(err, "error creating GitHub endpoint")
return endpoint, nil
}
func (suite *GarmSuite) DeleteGithubEndpoint(name string) error {
t := suite.T()
t.Log("Delete GitHub endpoint")
err := deleteGithubEndpoint(suite.cli, suite.authToken, name)
suite.NoError(err, "error deleting GitHub endpoint")
return nil
}
func (suite *GarmSuite) ListGithubEndpoints() params.GithubEndpoints {
t := suite.T()
t.Log("List GitHub endpoints")
endpoints, err := listGithubEndpoints(suite.cli, suite.authToken)
suite.NoError(err, "error listing GitHub endpoints")
return endpoints
}
func (suite *GarmSuite) createDummyEndpoint(name string) (*params.GithubEndpoint, error) {
endpointParams := params.CreateGithubEndpointParams{
Name: name,
Description: "Dummy endpoint",
BaseURL: "https://ghes.example.com",
APIBaseURL: "https://api.ghes.example.com/",
UploadBaseURL: "https://uploads.ghes.example.com/",
}
return suite.CreateGithubEndpoint(endpointParams)
}

171
test/integration/external_provider_test.go Normal file
View file

@ -0,0 +1,171 @@
//go:build integration
// +build integration
package integration
import (
"fmt"
"time"
"github.com/go-openapi/runtime"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/client"
clientInstances "github.com/cloudbase/garm/client/instances"
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) TestExternalProvider() {
t := suite.T()
t.Log("Testing external provider")
repoPoolParams2 := params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "test_external",
Tags: []string{"repo-runner-2"},
Enabled: true,
}
repoPool2 := suite.CreateRepoPool(suite.repo.ID, repoPoolParams2)
newParams := suite.UpdateRepoPool(suite.repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
t.Logf("Updated repo pool with pool_id %s with new_params %+v", repoPool2.ID, newParams)
err := suite.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute)
suite.NoError(err, "error waiting for pool instances to be running")
repoPool2 = suite.GetRepoPool(suite.repo.ID, repoPool2.ID)
suite.DisableRepoPool(suite.repo.ID, repoPool2.ID)
suite.DeleteInstance(repoPool2.Instances[0].Name, false, false)
err = suite.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute)
suite.NoError(err, "error waiting for pool instances to be pending delete")
suite.DeleteInstance(repoPool2.Instances[0].Name, true, false) // delete instance with forceRemove
err = suite.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute)
suite.NoError(err, "error waiting for instance to be removed")
suite.DeleteRepoPool(suite.repo.ID, repoPool2.ID)
}
func (suite *GarmSuite) WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error {
t := suite.T()
var timeWaited time.Duration // default is 0
pool, err := getPool(suite.cli, suite.authToken, poolID)
if err != nil {
return err
}
t.Logf("Waiting for pool instances with pool_id %s to reach desired status %v and desired_runner_status %v", poolID, status, runnerStatus)
for timeWaited < timeout {
poolInstances, err := listPoolInstances(suite.cli, suite.authToken, poolID)
if err != nil {
return err
}
instancesCount := 0
for _, instance := range poolInstances {
if instance.Status == status && instance.RunnerStatus == runnerStatus {
instancesCount++
}
}
t.Logf(
"Pool instance with pool_id %s reached status %v and runner_status %v, desired_instance_count %d, pool_instance_count %d",
poolID, status, runnerStatus, instancesCount,
len(poolInstances))
if int(pool.MinIdleRunners) == instancesCount {
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
err = suite.dumpPoolInstancesDetails(pool.ID)
suite.NoError(err, "error dumping pool instances details")
return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus)
}
func (suite *GarmSuite) dumpPoolInstancesDetails(poolID string) error {
t := suite.T()
pool, err := getPool(suite.cli, suite.authToken, poolID)
if err != nil {
return err
}
if err := printJSONResponse(pool); err != nil {
return err
}
for _, instance := range pool.Instances {
instanceDetails, err := getInstance(suite.cli, suite.authToken, instance.Name)
if err != nil {
return err
}
t.Logf("Instance details: instance_name %s", instance.Name)
if err := printJSONResponse(instanceDetails); err != nil {
return err
}
}
return nil
}
func (suite *GarmSuite) DisableRepoPool(repoID, repoPoolID string) {
t := suite.T()
t.Logf("Disable repo pool with repo_id %s and pool_id %s", repoID, repoPoolID)
enabled := false
poolParams := params.UpdatePoolParams{Enabled: &enabled}
_, err := updateRepoPool(suite.cli, suite.authToken, repoID, repoPoolID, poolParams)
suite.NoError(err, "error disabling repository pool")
}
func (suite *GarmSuite) DeleteInstance(name string, forceRemove, bypassGHUnauthorized bool) {
t := suite.T()
t.Logf("Delete instance %s with force_remove %t", name, forceRemove)
err := deleteInstance(suite.cli, suite.authToken, name, forceRemove, bypassGHUnauthorized)
suite.NoError(err, "error deleting instance", name)
t.Logf("Instance deletion initiated for instance %s", name)
}
func (suite *GarmSuite) WaitInstanceToBeRemoved(name string, timeout time.Duration) error {
t := suite.T()
var timeWaited time.Duration // default is 0
var instance *params.Instance
t.Logf("Waiting for instance %s to be removed", name)
for timeWaited < timeout {
instances, err := listInstances(suite.cli, suite.authToken)
if err != nil {
return err
}
instance = nil
for k, v := range instances {
if v.Name == name {
instance = &instances[k]
break
}
}
if instance == nil {
// The instance is not found in the list. We can safely assume
// that it is removed
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*instance); err != nil {
return err
}
return fmt.Errorf("instance %s was not removed within the timeout", name)
}
func listPoolInstances(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, poolID string) (params.Instances, error) {
listPoolInstancesResponse, err := apiCli.Instances.ListPoolInstances(
clientInstances.NewListPoolInstancesParams().WithPoolID(poolID),
apiAuthToken)
if err != nil {
return nil, err
}
return listPoolInstancesResponse.Payload, nil
}

149
test/integration/gh_cleanup/main.go
View file

@ -1,11 +1,13 @@
package main
import (
"context"
"fmt"
"log/slog"
"os"
"github.com/cloudbase/garm/test/integration/e2e"
"github.com/google/go-github/v57/github"
"golang.org/x/oauth2"
)
var (
@ -18,8 +20,8 @@ var (
func main() {
controllerID, ctrlIDFound := os.LookupEnv("GARM_CONTROLLER_ID")
if ctrlIDFound {
_ = e2e.GhOrgRunnersCleanup(ghToken, orgName, controllerID)
_ = e2e.GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID)
_ = GhOrgRunnersCleanup(ghToken, orgName, controllerID)
_ = GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID)
} else {
slog.Warn("Env variable GARM_CONTROLLER_ID is not set, skipping GitHub runners cleanup")
}
@ -27,9 +29,146 @@ func main() {
baseURL, baseURLFound := os.LookupEnv("GARM_BASE_URL")
if ctrlIDFound && baseURLFound {
webhookURL := fmt.Sprintf("%s/webhooks/%s", baseURL, controllerID)
_ = e2e.GhOrgWebhookCleanup(ghToken, webhookURL, orgName)
_ = e2e.GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName)
_ = GhOrgWebhookCleanup(ghToken, webhookURL, orgName)
_ = GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName)
} else {
slog.Warn("Env variables GARM_CONTROLLER_ID & GARM_BASE_URL are not set, skipping webhooks cleanup")
}
}
func GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error {
slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName)
client := getGithubClient(ghToken)
ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil)
if err != nil {
return err
}
// Remove organization runners
controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID)
for _, orgRunner := range ghOrgRunners.Runners {
for _, label := range orgRunner.Labels {
if label.GetName() == controllerLabel {
if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil {
// We don't fail if we can't remove a single runner. This
// is a best effort to try and remove all the orphan runners.
slog.With(slog.Any("error", err)).Info("Failed to remove organization runner", "org_runner", orgRunner.GetName())
break
}
slog.Info("Removed organization runner", "org_runner", orgRunner.GetName())
break
}
}
}
return nil
}
func GhRepoRunnersCleanup(ghToken, orgName, repoName, controllerID string) error {
slog.Info("Cleanup Github runners", "controller_id", controllerID, "org_name", orgName, "repo_name", repoName)
client := getGithubClient(ghToken)
ghRepoRunners, _, err := client.Actions.ListRunners(context.Background(), orgName, repoName, nil)
if err != nil {
return err
}
// Remove repository runners
controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID)
for _, repoRunner := range ghRepoRunners.Runners {
for _, label := range repoRunner.Labels {
if label.GetName() == controllerLabel {
if _, err := client.Actions.RemoveRunner(context.Background(), orgName, repoName, repoRunner.GetID()); err != nil {
// We don't fail if we can't remove a single runner. This
// is a best effort to try and remove all the orphan runners.
slog.With(slog.Any("error", err)).Error("Failed to remove repository runner", "runner_name", repoRunner.GetName())
break
}
slog.Info("Removed repository runner", "runner_name", repoRunner.GetName())
break
}
}
}
return nil
}
func GhOrgWebhookCleanup(ghToken, webhookURL, orgName string) error {
slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName)
hook, err := getGhOrgWebhook(webhookURL, ghToken, orgName)
if err != nil {
return err
}
// Remove organization webhook
if hook != nil {
client := getGithubClient(ghToken)
if _, err := client.Organizations.DeleteHook(context.Background(), orgName, hook.GetID()); err != nil {
return err
}
slog.Info("Github webhook removed", "webhook_url", webhookURL, "org_name", orgName)
}
return nil
}
func GhRepoWebhookCleanup(ghToken, webhookURL, orgName, repoName string) error {
slog.Info("Cleanup Github webhook", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName)
hook, err := getGhRepoWebhook(webhookURL, ghToken, orgName, repoName)
if err != nil {
return err
}
// Remove repository webhook
if hook != nil {
client := getGithubClient(ghToken)
if _, err := client.Repositories.DeleteHook(context.Background(), orgName, repoName, hook.GetID()); err != nil {
return err
}
slog.Info("Github webhook with", "webhook_url", webhookURL, "org_name", orgName, "repo_name", repoName)
}
return nil
}
func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghOrgHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghRepoHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func getGithubClient(oauthToken string) *github.Client {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken})
tc := oauth2.NewClient(context.Background(), ts)
return github.NewClient(tc)
}

168
test/integration/jobs_test.go Normal file
View file

@ -0,0 +1,168 @@
//go:build integration
// +build integration
package integration
import (
"context"
"fmt"
"time"
"github.com/google/go-github/v57/github"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) TestWorkflowJobs() {
suite.TriggerWorkflow(suite.ghToken, orgName, repoName, workflowFileName, "org-runner")
suite.ValidateJobLifecycle("org-runner")
suite.TriggerWorkflow(suite.ghToken, orgName, repoName, workflowFileName, "repo-runner")
suite.ValidateJobLifecycle("repo-runner")
}
func (suite *GarmSuite) TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, labelName string) {
t := suite.T()
t.Logf("Trigger workflow with label %s", labelName)
client := getGithubClient(ghToken)
eventReq := github.CreateWorkflowDispatchEventRequest{
Ref: "main",
Inputs: map[string]interface{}{
"sleep_time": "50",
"runner_label": labelName,
},
}
_, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), orgName, repoName, workflowFileName, eventReq)
suite.NoError(err, "error triggering workflow")
}
func (suite *GarmSuite) ValidateJobLifecycle(label string) {
t := suite.T()
t.Logf("Validate GARM job lifecycle with label %s", label)
// wait for job list to be updated
job, err := suite.waitLabelledJob(label, 4*time.Minute)
suite.NoError(err, "error waiting for job to be created")
// check expected job status
job, err = suite.waitJobStatus(job.ID, params.JobStatusQueued, 4*time.Minute)
suite.NoError(err, "error waiting for job to be queued")
job, err = suite.waitJobStatus(job.ID, params.JobStatusInProgress, 4*time.Minute)
suite.NoError(err, "error waiting for job to be in progress")
// check expected instance status
instance, err := suite.waitInstanceStatus(job.RunnerName, commonParams.InstanceRunning, params.RunnerActive, 5*time.Minute)
suite.NoError(err, "error waiting for instance to be running")
// wait for job to be completed
_, err = suite.waitJobStatus(job.ID, params.JobStatusCompleted, 4*time.Minute)
suite.NoError(err, "error waiting for job to be completed")
// wait for instance to be removed
err = suite.WaitInstanceToBeRemoved(instance.Name, 5*time.Minute)
suite.NoError(err, "error waiting for instance to be removed")
// wait for GARM to rebuild the pool running idle instances
err = suite.WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute)
suite.NoError(err, "error waiting for pool instances to be running idle")
}
func (suite *GarmSuite) waitLabelledJob(label string, timeout time.Duration) (*params.Job, error) {
t := suite.T()
var timeWaited time.Duration // default is 0
var jobs params.Jobs
var err error
t.Logf("Waiting for job with label %s", label)
for timeWaited < timeout {
jobs, err = listJobs(suite.cli, suite.authToken)
if err != nil {
return nil, err
}
for _, job := range jobs {
for _, jobLabel := range job.Labels {
if jobLabel == label {
return &job, err
}
}
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(jobs); err != nil {
return nil, err
}
return nil, fmt.Errorf("failed to wait job with label %s", label)
}
func (suite *GarmSuite) waitJobStatus(id int64, status params.JobStatus, timeout time.Duration) (*params.Job, error) {
t := suite.T()
var timeWaited time.Duration // default is 0
var job *params.Job
t.Logf("Waiting for job %d to reach status %v", id, status)
for timeWaited < timeout {
jobs, err := listJobs(suite.cli, suite.authToken)
if err != nil {
return nil, err
}
job = nil
for k, v := range jobs {
if v.ID == id {
job = &jobs[k]
break
}
}
if job == nil {
if status == params.JobStatusCompleted {
// The job is not found in the list. We can safely assume
// that it is completed
return nil, nil
}
// if the job is not found, and expected status is not "completed",
// we need to error out.
return nil, fmt.Errorf("job %d not found, expected to be found in status %s", id, status)
} else if job.Status == string(status) {
return job, nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*job); err != nil {
return nil, err
}
return nil, fmt.Errorf("timeout waiting for job %d to reach status %s", id, status)
}
func (suite *GarmSuite) waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) {
t := suite.T()
var timeWaited time.Duration // default is 0
var instance *params.Instance
var err error
t.Logf("Waiting for instance %s to reach desired status %v and desired runner status %v", name, status, runnerStatus)
for timeWaited < timeout {
instance, err = getInstance(suite.cli, suite.authToken, name)
if err != nil {
return nil, err
}
t.Logf("Instance %s has status %v and runner status %v", name, instance.Status, instance.RunnerStatus)
if instance.Status == status && instance.RunnerStatus == runnerStatus {
return instance, nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJSONResponse(*instance); err != nil {
return nil, err
}
return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
}

72
test/integration/list_info_test.go Normal file
View file

@ -0,0 +1,72 @@
//go:build integration
// +build integration
package integration
import (
"fmt"
"os"
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) TestGetControllerInfo() {
controllerInfo := suite.GetControllerInfo()
suite.NotEmpty(controllerInfo.ControllerID, "controller ID is empty")
}
func (suite *GarmSuite) GetMetricsToken() {
t := suite.T()
t.Log("Get metrics token")
metricsToken, err := getMetricsToken(suite.cli, suite.authToken)
suite.NoError(err, "error getting metrics token")
suite.NotEmpty(metricsToken, "metrics token is empty")
}
func (suite *GarmSuite) GetControllerInfo() *params.ControllerInfo {
t := suite.T()
t.Log("Get controller info")
controllerInfo, err := getControllerInfo(suite.cli, suite.authToken)
suite.NoError(err, "error getting controller info")
err = suite.appendCtrlInfoToGitHubEnv(&controllerInfo)
suite.NoError(err, "error appending controller info to GitHub env")
err = printJSONResponse(controllerInfo)
suite.NoError(err, "error printing controller info")
return &controllerInfo
}
func (suite *GarmSuite) TestListCredentials() {
t := suite.T()
t.Log("List credentials")
credentials, err := listCredentials(suite.cli, suite.authToken)
suite.NoError(err, "error listing credentials")
suite.NotEmpty(credentials, "credentials list is empty")
}
func (suite *GarmSuite) TestListProviders() {
t := suite.T()
t.Log("List providers")
providers, err := listProviders(suite.cli, suite.authToken)
suite.NoError(err, "error listing providers")
suite.NotEmpty(providers, "providers list is empty")
}
func (suite *GarmSuite) appendCtrlInfoToGitHubEnv(controllerInfo *params.ControllerInfo) error {
t := suite.T()
envFile, found := os.LookupEnv("GITHUB_ENV")
if !found {
t.Log("GITHUB_ENV not set, skipping appending controller info")
return nil
}
file, err := os.OpenFile(envFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
if err != nil {
return err
}
t.Cleanup(func() {
file.Close()
})
if _, err := file.WriteString(fmt.Sprintf("export GARM_CONTROLLER_ID=%s\n", controllerInfo.ControllerID)); err != nil {
return err
}
return nil
}

191
test/integration/main.go
View file

@ -1,191 +0,0 @@
package main
import (
"fmt"
"log/slog"
"os"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/test/integration/e2e"
)
var (
adminPassword = os.Getenv("GARM_PASSWORD")
adminUsername = os.Getenv("GARM_ADMIN_USERNAME")
adminFullName = "GARM Admin"
adminEmail = "admin@example.com"
baseURL = os.Getenv("GARM_BASE_URL")
credentialsName = os.Getenv("CREDENTIALS_NAME")
repoName = os.Getenv("REPO_NAME")
repoWebhookSecret = os.Getenv("REPO_WEBHOOK_SECRET")
repoPoolParams = params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "lxd_local",
Tags: []string{"repo-runner"},
Enabled: true,
}
repoPoolParams2 = params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "test_external",
Tags: []string{"repo-runner-2"},
Enabled: true,
}
orgName = os.Getenv("ORG_NAME")
orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
orgPoolParams = params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "lxd_local",
Tags: []string{"org-runner"},
Enabled: true,
}
ghToken = os.Getenv("GH_TOKEN")
workflowFileName = os.Getenv("WORKFLOW_FILE_NAME")
)
func main() {
/////////////
// Cleanup //
/////////////
defer e2e.GracefulCleanup()
///////////////
// garm init //
///////////////
e2e.InitClient(baseURL)
e2e.FirstRun(adminUsername, adminPassword, adminFullName, adminEmail)
e2e.Login(adminUsername, adminPassword)
// Test endpoint operations
e2e.TestGithubEndpointOperations()
e2e.TestGithubEndpointFailsOnInvalidCABundle()
e2e.TestGithubEndpointDeletionFailsWhenCredentialsExist()
e2e.TestGithubEndpointFailsOnDuplicateName()
e2e.TestGithubEndpointMustFailToDeleteDefaultGithubEndpoint()
// Create test credentials
e2e.EnsureTestCredentials(credentialsName, ghToken, "github.com")
e2e.TestGithubCredentialsErrorOnDuplicateCredentialsName()
e2e.TestGithubCredentialsFailsToDeleteWhenInUse()
e2e.TestGithubCredentialsFailsOnInvalidAuthType()
e2e.TestGithubCredentialsFailsWhenAuthTypeParamsAreIncorrect()
e2e.TestGithubCredentialsFailsWhenAuthTypeParamsAreMissing()
e2e.TestGithubCredentialsUpdateFailsWhenBothPATAndAppAreSupplied()
e2e.TestGithubCredentialsFailWhenAppKeyIsInvalid()
e2e.TestGithubCredentialsFailWhenEndpointDoesntExist()
e2e.TestGithubCredentialsFailsOnDuplicateName()
// //////////////////
// controller info //
// //////////////////
e2e.GetControllerInfo()
// ////////////////////////////
// credentials and providers //
// ////////////////////////////
e2e.ListCredentials()
e2e.ListProviders()
////////////////////
/// metrics token //
////////////////////
e2e.GetMetricsToken()
//////////////////
// repositories //
//////////////////
repo := e2e.CreateRepo(orgName, repoName, credentialsName, repoWebhookSecret)
repo = e2e.UpdateRepo(repo.ID, fmt.Sprintf("%s-clone", credentialsName))
hookRepoInfo := e2e.InstallRepoWebhook(repo.ID)
e2e.ValidateRepoWebhookInstalled(ghToken, hookRepoInfo.URL, orgName, repoName)
e2e.UninstallRepoWebhook(repo.ID)
e2e.ValidateRepoWebhookUninstalled(ghToken, hookRepoInfo.URL, orgName, repoName)
_ = e2e.InstallRepoWebhook(repo.ID)
e2e.ValidateRepoWebhookInstalled(ghToken, hookRepoInfo.URL, orgName, repoName)
repoPool := e2e.CreateRepoPool(repo.ID, repoPoolParams)
repoPool = e2e.GetRepoPool(repo.ID, repoPool.ID)
e2e.DeleteRepoPool(repo.ID, repoPool.ID)
repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams)
_ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
/////////////////////////////
// Test external provider ///
/////////////////////////////
slog.Info("Testing external provider")
repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
newParams := e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
slog.Info("Updated repo pool", "new_params", newParams)
err := e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute)
if err != nil {
slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running", "pool_id", repoPool2.ID, "provider_name", repoPoolParams2.ProviderName)
}
repoPool2 = e2e.GetRepoPool(repo.ID, repoPool2.ID)
e2e.DisableRepoPool(repo.ID, repoPool2.ID)
e2e.DeleteInstance(repoPool2.Instances[0].Name, false, false)
err = e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute)
if err != nil {
slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be running")
}
e2e.DeleteInstance(repoPool2.Instances[0].Name, true, false) // delete instance with forceRemove
err = e2e.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute)
if err != nil {
slog.With(slog.Any("error", err)).Error("Failed to wait for instance to be removed")
}
e2e.DeleteRepoPool(repo.ID, repoPool2.ID)
///////////////////
// organizations //
///////////////////
org := e2e.CreateOrg(orgName, credentialsName, orgWebhookSecret)
org = e2e.UpdateOrg(org.ID, fmt.Sprintf("%s-clone", credentialsName))
orgHookInfo := e2e.InstallOrgWebhook(org.ID)
e2e.ValidateOrgWebhookInstalled(ghToken, orgHookInfo.URL, orgName)
e2e.UninstallOrgWebhook(org.ID)
e2e.ValidateOrgWebhookUninstalled(ghToken, orgHookInfo.URL, orgName)
_ = e2e.InstallOrgWebhook(org.ID)
e2e.ValidateOrgWebhookInstalled(ghToken, orgHookInfo.URL, orgName)
orgPool := e2e.CreateOrgPool(org.ID, orgPoolParams)
orgPool = e2e.GetOrgPool(org.ID, orgPool.ID)
e2e.DeleteOrgPool(org.ID, orgPool.ID)
orgPool = e2e.CreateOrgPool(org.ID, orgPoolParams)
_ = e2e.UpdateOrgPool(org.ID, orgPool.ID, orgPoolParams.MaxRunners, 1)
///////////////
// instances //
///////////////
e2e.WaitRepoRunningIdleInstances(repo.ID, 6*time.Minute)
e2e.WaitOrgRunningIdleInstances(org.ID, 6*time.Minute)
//////////
// jobs //
//////////
e2e.TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, "org-runner")
e2e.ValidateJobLifecycle("org-runner")
e2e.TriggerWorkflow(ghToken, orgName, repoName, workflowFileName, "repo-runner")
e2e.ValidateJobLifecycle("repo-runner")
}

192
test/integration/organizations_test.go Normal file
View file

@ -0,0 +1,192 @@
//go:build integration
// +build integration
package integration
import (
"context"
"fmt"
"time"
"github.com/google/go-github/v57/github"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) TestOrganizations() {
organization := suite.CreateOrg(orgName, suite.credentialsName, orgWebhookSecret)
org := suite.UpdateOrg(organization.ID, fmt.Sprintf("%s-clone", suite.credentialsName))
suite.NotEqual(organization, org, "organization not updated")
orgHookInfo := suite.InstallOrgWebhook(org.ID)
suite.ValidateOrgWebhookInstalled(suite.ghToken, orgHookInfo.URL, orgName)
suite.UninstallOrgWebhook(org.ID)
suite.ValidateOrgWebhookUninstalled(suite.ghToken, orgHookInfo.URL, orgName)
_ = suite.InstallOrgWebhook(org.ID)
suite.ValidateOrgWebhookInstalled(suite.ghToken, orgHookInfo.URL, orgName)
orgPoolParams := params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "lxd_local",
Tags: []string{"org-runner"},
Enabled: true,
}
orgPool := suite.CreateOrgPool(org.ID, orgPoolParams)
orgPoolGot := suite.GetOrgPool(org.ID, orgPool.ID)
suite.Equal(orgPool, orgPoolGot, "organization pool mismatch")
suite.DeleteOrgPool(org.ID, orgPool.ID)
orgPool = suite.CreateOrgPool(org.ID, orgPoolParams)
orgPoolUpdated := suite.UpdateOrgPool(org.ID, orgPool.ID, orgPoolParams.MaxRunners, 1)
suite.NotEqual(orgPool, orgPoolUpdated, "organization pool not updated")
suite.WaitOrgRunningIdleInstances(org.ID, 6*time.Minute)
}
func (suite *GarmSuite) CreateOrg(orgName, credentialsName, orgWebhookSecret string) *params.Organization {
t := suite.T()
t.Logf("Create org with org_name %s", orgName)
orgParams := params.CreateOrgParams{
Name: orgName,
CredentialsName: credentialsName,
WebhookSecret: orgWebhookSecret,
}
org, err := createOrg(suite.cli, suite.authToken, orgParams)
suite.NoError(err, "error creating organization")
return org
}
func (suite *GarmSuite) UpdateOrg(id, credentialsName string) *params.Organization {
t := suite.T()
t.Logf("Update org with org_id %s", id)
updateParams := params.UpdateEntityParams{
CredentialsName: credentialsName,
}
org, err := updateOrg(suite.cli, suite.authToken, id, updateParams)
suite.NoError(err, "error updating organization")
return org
}
func (suite *GarmSuite) InstallOrgWebhook(id string) *params.HookInfo {
t := suite.T()
t.Logf("Install org webhook with org_id %s", id)
webhookParams := params.InstallWebhookParams{
WebhookEndpointType: params.WebhookEndpointDirect,
}
_, err := installOrgWebhook(suite.cli, suite.authToken, id, webhookParams)
suite.NoError(err, "error installing organization webhook")
webhookInfo, err := getOrgWebhook(suite.cli, suite.authToken, id)
suite.NoError(err, "error getting organization webhook")
return webhookInfo
}
func (suite *GarmSuite) ValidateOrgWebhookInstalled(ghToken, url, orgName string) {
hook, err := getGhOrgWebhook(url, ghToken, orgName)
suite.NoError(err, "error getting github webhook")
suite.NotNil(hook, "github webhook with url %s, for org %s was not properly installed", url, orgName)
}
func getGhOrgWebhook(url, ghToken, orgName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghOrgHooks, _, err := client.Organizations.ListHooks(context.Background(), orgName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghOrgHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func (suite *GarmSuite) UninstallOrgWebhook(id string) {
t := suite.T()
t.Logf("Uninstall org webhook with org_id %s", id)
err := uninstallOrgWebhook(suite.cli, suite.authToken, id)
suite.NoError(err, "error uninstalling organization webhook")
}
func (suite *GarmSuite) ValidateOrgWebhookUninstalled(ghToken, url, orgName string) {
hook, err := getGhOrgWebhook(url, ghToken, orgName)
suite.NoError(err, "error getting github webhook")
suite.Nil(hook, "github webhook with url %s, for org %s was not properly uninstalled", url, orgName)
}
func (suite *GarmSuite) CreateOrgPool(orgID string, poolParams params.CreatePoolParams) *params.Pool {
t := suite.T()
t.Logf("Create org pool with org_id %s", orgID)
pool, err := createOrgPool(suite.cli, suite.authToken, orgID, poolParams)
suite.NoError(err, "error creating organization pool")
return pool
}
func (suite *GarmSuite) GetOrgPool(orgID, orgPoolID string) *params.Pool {
t := suite.T()
t.Logf("Get org pool with org_id %s and pool_id %s", orgID, orgPoolID)
pool, err := getOrgPool(suite.cli, suite.authToken, orgID, orgPoolID)
suite.NoError(err, "error getting organization pool")
return pool
}
func (suite *GarmSuite) DeleteOrgPool(orgID, orgPoolID string) {
t := suite.T()
t.Logf("Delete org pool with org_id %s and pool_id %s", orgID, orgPoolID)
err := deleteOrgPool(suite.cli, suite.authToken, orgID, orgPoolID)
suite.NoError(err, "error deleting organization pool")
}
func (suite *GarmSuite) UpdateOrgPool(orgID, orgPoolID string, maxRunners, minIdleRunners uint) *params.Pool {
t := suite.T()
t.Logf("Update org pool with org_id %s and pool_id %s", orgID, orgPoolID)
poolParams := params.UpdatePoolParams{
MinIdleRunners: &minIdleRunners,
MaxRunners: &maxRunners,
}
pool, err := updateOrgPool(suite.cli, suite.authToken, orgID, orgPoolID, poolParams)
suite.NoError(err, "error updating organization pool")
return pool
}
func (suite *GarmSuite) WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
t := suite.T()
orgPools, err := listOrgPools(suite.cli, suite.authToken, orgID)
suite.NoError(err, "error listing organization pools")
for _, pool := range orgPools {
err := suite.WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
if err != nil {
suite.dumpOrgInstancesDetails(orgID)
t.Errorf("timeout waiting for organization %s instances to reach status: %s and runner status: %s", orgID, commonParams.InstanceRunning, params.RunnerIdle)
}
}
}
func (suite *GarmSuite) dumpOrgInstancesDetails(orgID string) {
t := suite.T()
// print org details
t.Logf("Dumping org details with org_id %s", orgID)
org, err := getOrg(suite.cli, suite.authToken, orgID)
suite.NoError(err, "error getting organization")
err = printJSONResponse(org)
suite.NoError(err, "error printing organization")
// print org instances details
t.Logf("Dumping org instances details for org %s", orgID)
instances, err := listOrgInstances(suite.cli, suite.authToken, orgID)
suite.NoError(err, "error listing organization instances")
for _, instance := range instances {
instance, err := getInstance(suite.cli, suite.authToken, instance.Name)
suite.NoError(err, "error getting instance")
t.Logf("Instance info for instace %s", instance.Name)
err = printJSONResponse(instance)
suite.NoError(err, "error printing instance")
}
}

208
test/integration/repositories_test.go Normal file
View file

@ -0,0 +1,208 @@
//go:build integration
// +build integration
package integration
import (
"context"
"fmt"
"time"
"github.com/google/go-github/v57/github"
"golang.org/x/oauth2"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func (suite *GarmSuite) EnsureTestCredentials(name string, oauthToken string, endpointName string) {
t := suite.T()
t.Log("Ensuring test credentials exist")
createCredsParams := params.CreateGithubCredentialsParams{
Name: name,
Endpoint: endpointName,
Description: "GARM test credentials",
AuthType: params.GithubAuthTypePAT,
PAT: params.GithubPAT{
OAuth2Token: oauthToken,
},
}
suite.CreateGithubCredentials(createCredsParams)
createCredsParams.Name = fmt.Sprintf("%s-clone", name)
suite.CreateGithubCredentials(createCredsParams)
}
func (suite *GarmSuite) TestRepositories() {
t := suite.T()
t.Logf("Update repo with repo_id %s", suite.repo.ID)
updateParams := params.UpdateEntityParams{
CredentialsName: fmt.Sprintf("%s-clone", suite.credentialsName),
}
repo, err := updateRepo(suite.cli, suite.authToken, suite.repo.ID, updateParams)
suite.NoError(err, "error updating repository")
suite.Equal(fmt.Sprintf("%s-clone", suite.credentialsName), repo.CredentialsName, "credentials name mismatch")
suite.repo = repo
hookRepoInfo := suite.InstallRepoWebhook(suite.repo.ID)
suite.ValidateRepoWebhookInstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName)
suite.UninstallRepoWebhook(suite.repo.ID)
suite.ValidateRepoWebhookUninstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName)
suite.InstallRepoWebhook(suite.repo.ID)
suite.ValidateRepoWebhookInstalled(suite.ghToken, hookRepoInfo.URL, orgName, repoName)
repoPoolParams := params.CreatePoolParams{
MaxRunners: 2,
MinIdleRunners: 0,
Flavor: "default",
Image: "ubuntu:22.04",
OSType: commonParams.Linux,
OSArch: commonParams.Amd64,
ProviderName: "lxd_local",
Tags: []string{"repo-runner"},
Enabled: true,
}
repoPool := suite.CreateRepoPool(suite.repo.ID, repoPoolParams)
suite.Equal(repoPool.MaxRunners, repoPoolParams.MaxRunners, "max runners mismatch")
suite.Equal(repoPool.MinIdleRunners, repoPoolParams.MinIdleRunners, "min idle runners mismatch")
repoPoolGet := suite.GetRepoPool(suite.repo.ID, repoPool.ID)
suite.Equal(*repoPool, *repoPoolGet, "pool get mismatch")
suite.DeleteRepoPool(suite.repo.ID, repoPool.ID)
repoPool = suite.CreateRepoPool(suite.repo.ID, repoPoolParams)
updatedRepoPool := suite.UpdateRepoPool(suite.repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
suite.NotEqual(updatedRepoPool.MinIdleRunners, repoPool.MinIdleRunners, "min idle runners mismatch")
suite.WaitRepoRunningIdleInstances(suite.repo.ID, 6*time.Minute)
}
func (suite *GarmSuite) InstallRepoWebhook(id string) *params.HookInfo {
t := suite.T()
t.Logf("Install repo webhook with repo_id %s", id)
webhookParams := params.InstallWebhookParams{
WebhookEndpointType: params.WebhookEndpointDirect,
}
_, err := installRepoWebhook(suite.cli, suite.authToken, id, webhookParams)
suite.NoError(err, "error installing repository webhook")
webhookInfo, err := getRepoWebhook(suite.cli, suite.authToken, id)
suite.NoError(err, "error getting repository webhook")
return webhookInfo
}
func (suite *GarmSuite) ValidateRepoWebhookInstalled(ghToken, url, orgName, repoName string) {
hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName)
suite.NoError(err, "error getting github webhook")
suite.NotNil(hook, "github webhook with url %s, for repo %s/%s was not properly installed", url, orgName, repoName)
}
func getGhRepoWebhook(url, ghToken, orgName, repoName string) (*github.Hook, error) {
client := getGithubClient(ghToken)
ghRepoHooks, _, err := client.Repositories.ListHooks(context.Background(), orgName, repoName, nil)
if err != nil {
return nil, err
}
for _, hook := range ghRepoHooks {
hookURL, ok := hook.Config["url"].(string)
if ok && hookURL == url {
return hook, nil
}
}
return nil, nil
}
func getGithubClient(oauthToken string) *github.Client {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: oauthToken})
tc := oauth2.NewClient(context.Background(), ts)
return github.NewClient(tc)
}
func (suite *GarmSuite) UninstallRepoWebhook(id string) {
t := suite.T()
t.Logf("Uninstall repo webhook with repo_id %s", id)
err := uninstallRepoWebhook(suite.cli, suite.authToken, id)
suite.NoError(err, "error uninstalling repository webhook")
}
func (suite *GarmSuite) ValidateRepoWebhookUninstalled(ghToken, url, orgName, repoName string) {
hook, err := getGhRepoWebhook(url, ghToken, orgName, repoName)
suite.NoError(err, "error getting github webhook")
suite.Nil(hook, "github webhook with url %s, for repo %s/%s was not properly uninstalled", url, orgName, repoName)
}
func (suite *GarmSuite) CreateRepoPool(repoID string, poolParams params.CreatePoolParams) *params.Pool {
t := suite.T()
t.Logf("Create repo pool with repo_id %s and pool_params %+v", repoID, poolParams)
pool, err := createRepoPool(suite.cli, suite.authToken, repoID, poolParams)
suite.NoError(err, "error creating repository pool")
return pool
}
func (suite *GarmSuite) GetRepoPool(repoID, repoPoolID string) *params.Pool {
t := suite.T()
t.Logf("Get repo pool repo_id %s and pool_id %s", repoID, repoPoolID)
pool, err := getRepoPool(suite.cli, suite.authToken, repoID, repoPoolID)
suite.NoError(err, "error getting repository pool")
return pool
}
func (suite *GarmSuite) DeleteRepoPool(repoID, repoPoolID string) {
t := suite.T()
t.Logf("Delete repo pool with repo_id %s and pool_id %s", repoID, repoPoolID)
err := deleteRepoPool(suite.cli, suite.authToken, repoID, repoPoolID)
suite.NoError(err, "error deleting repository pool")
}
func (suite *GarmSuite) UpdateRepoPool(repoID, repoPoolID string, maxRunners, minIdleRunners uint) *params.Pool {
t := suite.T()
t.Logf("Update repo pool with repo_id %s and pool_id %s", repoID, repoPoolID)
poolParams := params.UpdatePoolParams{
MinIdleRunners: &minIdleRunners,
MaxRunners: &maxRunners,
}
pool, err := updateRepoPool(suite.cli, suite.authToken, repoID, repoPoolID, poolParams)
suite.NoError(err, "error updating repository pool")
return pool
}
func (suite *GarmSuite) WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
t := suite.T()
repoPools, err := listRepoPools(suite.cli, suite.authToken, repoID)
suite.NoError(err, "error listing repo pools")
for _, pool := range repoPools {
err := suite.WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
if err != nil {
suite.dumpRepoInstancesDetails(repoID)
t.Errorf("error waiting for pool instances to be running idle: %v", err)
}
}
}
func (suite *GarmSuite) dumpRepoInstancesDetails(repoID string) {
t := suite.T()
// print repo details
t.Logf("Dumping repo details for repo %s", repoID)
repo, err := getRepo(suite.cli, suite.authToken, repoID)
suite.NoError(err, "error getting repo")
err = printJSONResponse(repo)
suite.NoError(err, "error printing repo")
// print repo instances details
t.Logf("Dumping repo instances details for repo %s", repoID)
instances, err := listRepoInstances(suite.cli, suite.authToken, repoID)
suite.NoError(err, "error listing repo instances")
for _, instance := range instances {
instance, err := getInstance(suite.cli, suite.authToken, instance.Name)
suite.NoError(err, "error getting instance")
t.Logf("Instance info for instance %s", instance.Name)
err = printJSONResponse(instance)
suite.NoError(err, "error printing instance")
}
}

212
test/integration/suite_test.go Normal file
View file

@ -0,0 +1,212 @@
//go:build integration
// +build integration
package integration
import (
"context"
"fmt"
"net/url"
"os"
"testing"
"time"
"github.com/go-openapi/runtime"
openapiRuntimeClient "github.com/go-openapi/runtime/client"
"github.com/stretchr/testify/suite"
"github.com/cloudbase/garm/client"
"github.com/cloudbase/garm/params"
)
var (
orgName string
repoName string
orgWebhookSecret string
workflowFileName string
)
type GarmSuite struct {
suite.Suite
cli *client.GarmAPI
authToken runtime.ClientAuthInfoWriter
ghToken string
credentialsName string
repo *params.Repository
}
func (suite *GarmSuite) SetupSuite() {
t := suite.T()
suite.ghToken = os.Getenv("GH_TOKEN")
orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
workflowFileName = os.Getenv("WORKFLOW_FILE_NAME")
baseURL := os.Getenv("GARM_BASE_URL")
adminPassword := os.Getenv("GARM_PASSWORD")
adminUsername := os.Getenv("GARM_ADMIN_USERNAME")
adminFullName := "GARM Admin"
adminEmail := "admin@example.com"
garmURL, err := url.Parse(baseURL)
suite.NoError(err, "error parsing GARM_BASE_URL")
apiPath, err := url.JoinPath(garmURL.Path, client.DefaultBasePath)
suite.NoError(err, "error joining path")
transportCfg := client.DefaultTransportConfig().
WithHost(garmURL.Host).
WithBasePath(apiPath).
WithSchemes([]string{garmURL.Scheme})
suite.cli = client.NewHTTPClientWithConfig(nil, transportCfg)
t.Log("First run")
newUser := params.NewUserParams{
Username: adminUsername,
Password: adminPassword,
FullName: adminFullName,
Email: adminEmail,
}
_, err = firstRun(suite.cli, newUser)
suite.NoError(err, "error at first run")
t.Log("Login")
loginParams := params.PasswordLoginParams{
Username: adminUsername,
Password: adminPassword,
}
token, err := login(suite.cli, loginParams)
suite.NoError(err, "error at login")
suite.authToken = openapiRuntimeClient.BearerToken(token)
t.Log("Log in successful")
suite.credentialsName = os.Getenv("CREDENTIALS_NAME")
suite.EnsureTestCredentials(suite.credentialsName, suite.ghToken, "github.com")
t.Log("Create repository")
orgName = os.Getenv("ORG_NAME")
repoName = os.Getenv("REPO_NAME")
repoWebhookSecret := os.Getenv("REPO_WEBHOOK_SECRET")
createParams := params.CreateRepoParams{
Owner: orgName,
Name: repoName,
CredentialsName: suite.credentialsName,
WebhookSecret: repoWebhookSecret,
}
suite.repo, err = createRepo(suite.cli, suite.authToken, createParams)
suite.NoError(err, "error creating repository")
suite.Equal(orgName, suite.repo.Owner, "owner name mismatch")
suite.Equal(repoName, suite.repo.Name, "repo name mismatch")
suite.Equal(suite.credentialsName, suite.repo.CredentialsName, "credentials name mismatch")
}
func (suite *GarmSuite) TearDownSuite() {
t := suite.T()
t.Log("Graceful cleanup")
// disable all the pools
pools, err := listPools(suite.cli, suite.authToken)
suite.NoError(err, "error listing pools")
enabled := false
poolParams := params.UpdatePoolParams{Enabled: &enabled}
for _, pool := range pools {
_, err := updatePool(suite.cli, suite.authToken, pool.ID, poolParams)
suite.NoError(err, "error disabling pool")
t.Logf("Pool %s disabled during stage graceful_cleanup", pool.ID)
}
// delete all the instances
for _, pool := range pools {
poolInstances, err := listPoolInstances(suite.cli, suite.authToken, pool.ID)
suite.NoError(err, "error listing pool instances")
for _, instance := range poolInstances {
err := deleteInstance(suite.cli, suite.authToken, instance.Name, false, false)
suite.NoError(err, "error deleting instance")
t.Logf("Instance deletion initiated for instace %s during stage graceful_cleanup", instance.Name)
}
}
// wait for all instances to be deleted
for _, pool := range pools {
err := suite.waitPoolNoInstances(pool.ID, 3*time.Minute)
suite.NoError(err, "error waiting for pool to have no instances")
}
// delete all the pools
for _, pool := range pools {
err := deletePool(suite.cli, suite.authToken, pool.ID)
suite.NoError(err, "error deleting pool")
t.Logf("Pool %s deleted during stage graceful_cleanup", pool.ID)
}
// delete all the repositories
repos, err := listRepos(suite.cli, suite.authToken)
suite.NoError(err, "error listing repositories")
for _, repo := range repos {
err := deleteRepo(suite.cli, suite.authToken, repo.ID)
suite.NoError(err, "error deleting repository")
t.Logf("Repo %s deleted during stage graceful_cleanup", repo.ID)
}
// delete all the organizations
orgs, err := listOrgs(suite.cli, suite.authToken)
suite.NoError(err, "error listing organizations")
for _, org := range orgs {
err := deleteOrg(suite.cli, suite.authToken, org.ID)
suite.NoError(err, "error deleting organization")
t.Logf("Org %s deleted during stage graceful_cleanup", org.ID)
}
}
func TestGarmTestSuite(t *testing.T) {
suite.Run(t, new(GarmSuite))
}
func (suite *GarmSuite) waitPoolNoInstances(id string, timeout time.Duration) error {
t := suite.T()
var timeWaited time.Duration // default is 0
var pool *params.Pool
var err error
t.Logf("Wait until pool with id %s has no instances", id)
for timeWaited < timeout {
pool, err = getPool(suite.cli, suite.authToken, id)
suite.NoError(err, "error getting pool")
t.Logf("Current pool has %d instances", len(pool.Instances))
if len(pool.Instances) == 0 {
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
err = suite.dumpPoolInstancesDetails(pool.ID)
suite.NoError(err, "error dumping pool instances details")
return fmt.Errorf("failed to wait for pool %s to have no instances", pool.ID)
}
func (suite *GarmSuite) GhOrgRunnersCleanup(ghToken, orgName, controllerID string) error {
t := suite.T()
t.Logf("Cleanup Github runners for controller %s and org %s", controllerID, orgName)
client := getGithubClient(ghToken)
ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil)
if err != nil {
return err
}
// Remove organization runners
controllerLabel := fmt.Sprintf("runner-controller-id:%s", controllerID)
for _, orgRunner := range ghOrgRunners.Runners {
for _, label := range orgRunner.Labels {
if label.GetName() == controllerLabel {
if _, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID()); err != nil {
// We don't fail if we can't remove a single runner. This
// is a best effort to try and remove all the orphan runners.
t.Logf("Failed to remove organization runner %s: %v", orgRunner.GetName(), err)
break
}
t.Logf("Removed organization runner %s", orgRunner.GetName())
break
}
}
}
return nil
}

14
test/integration/e2e/utils.go → test/integration/utils.go
View file

@ -1,8 +1,8 @@
package e2e
package integration
import (
"encoding/json"
"log"
"fmt"
"log/slog"
)
@ -19,15 +19,17 @@ type apiCodeGetter interface {
IsCode(code int) bool
}
func expectAPIStatusCode(err error, expectedCode int) {
func expectAPIStatusCode(err error, expectedCode int) error {
if err == nil {
panic("expected error")
return fmt.Errorf("expected error, got nil")
}
apiErr, ok := err.(apiCodeGetter)
if !ok {
log.Fatalf("expected API error, got %v (%T)", err, err)
return fmt.Errorf("expected API error, got %v (%T)", err, err)
}
if !apiErr.IsCode(expectedCode) {
log.Fatalf("expected status code %d: %v", expectedCode, err)
return fmt.Errorf("expected status code %d: %v", expectedCode, err)
}
return nil
}