Daniel Sy
f92a4a7c06
Some checks failed
Go Tests / go-tests (push) Failing after 1m2s
Changes the runner infrastructure from single Pods to Deployments for improved reliability and management: - Adds restart policy configuration - Improves container and volume naming conventions - Maintains single replica to ensure one runner instance - Sets up proper label selectors for deployment management
519 lines
14 KiB
Go
519 lines
14 KiB
Go
// Copyright 2023 Cloudbase Solutions SRL
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
// not use this file except in compliance with the License. You may obtain
|
|
// a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
// License for the specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package provider
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"edp.buildth.ing/DevFW-CICD/edge-connect-client/sdk/edgeconnect"
|
|
"edp.buildth.ing/DevFW-CICD/garm-provider-edge-connect/config"
|
|
"edp.buildth.ing/DevFW-CICD/garm-provider-edge-connect/internal/spec"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
execution "github.com/cloudbase/garm-provider-common/execution/v0.1.0"
|
|
"github.com/cloudbase/garm-provider-common/params"
|
|
"k8s.io/utils/ptr"
|
|
)
|
|
|
|
var Version = "v0.0.0-unknown"
|
|
|
|
type GitHubScopeDetails struct {
|
|
BaseURL string
|
|
Repo string
|
|
Org string
|
|
Enterprise string
|
|
}
|
|
|
|
func NewEdgeConnectProvider(configPath, controllerID string) (execution.ExternalProvider, string, error) {
|
|
conf, err := config.NewConfig(configPath)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("error loading config: %w", err)
|
|
}
|
|
|
|
creds, err := config.NewCredentials(conf.CredentialsFile)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("error loading config: %w", err)
|
|
}
|
|
|
|
client := edgeconnect.NewClientWithCredentials(
|
|
conf.EdgeConnectURL,
|
|
creds.Username,
|
|
creds.Password,
|
|
edgeconnect.WithHTTPClient(&http.Client{Timeout: 30 * time.Second}),
|
|
)
|
|
|
|
return &edgeConnectProvider{
|
|
client: client,
|
|
controllerID: controllerID,
|
|
cfg: conf,
|
|
}, conf.LogFile, nil
|
|
}
|
|
|
|
type edgeConnectProvider struct {
|
|
client *edgeconnect.Client
|
|
controllerID string
|
|
cfg *config.Config
|
|
}
|
|
|
|
// CreateInstance creates a new compute instance in the provider.
|
|
func (a *edgeConnectProvider) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.ProviderInstance, error) {
|
|
log.Printf("Executing CreateInstance with %v\n", bootstrapParams)
|
|
log.Printf("Executing CreateInstance with RepoURL %v\n", bootstrapParams.RepoURL)
|
|
|
|
instancename := fmt.Sprintf("garm-%v-%v-%v", a.controllerID[:8], bootstrapParams.PoolID[:8], strings.ToLower(bootstrapParams.Name))
|
|
|
|
gitHubScopeDetails, err := spec.ExtractGitHubScopeDetails(bootstrapParams.RepoURL)
|
|
if err != nil {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
|
|
envs := spec.GetRunnerEnvs(gitHubScopeDetails, bootstrapParams)
|
|
|
|
deployment := appsv1.Deployment{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "Deployment",
|
|
APIVersion: "apps/v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: instancename,
|
|
Labels: map[string]string{"app": instancename},
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: ptr.To(int32(1)),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": instancename},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"app": instancename},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyNever,
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "runner",
|
|
Image: "edp.buildth.ing/devfw-cicd/garm-act-runner:1",
|
|
ImagePullPolicy: "Always",
|
|
Env: envs,
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "runner-dir",
|
|
MountPath: "/runner",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "runner-dir",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
manifest, err := json.Marshal(deployment)
|
|
if err != nil {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
|
|
_, err = a.client.ShowApp(ctx, edgeconnect.AppKey{
|
|
Organization: a.cfg.Organization,
|
|
Name: instancename,
|
|
Version: "0.0.1",
|
|
}, a.cfg.Region)
|
|
if err != nil && !errors.Is(err, edgeconnect.ErrResourceNotFound) {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
if errors.Is(err, edgeconnect.ErrResourceNotFound) {
|
|
err = a.client.CreateApp(ctx, &edgeconnect.NewAppInput{
|
|
Region: a.cfg.Region,
|
|
App: edgeconnect.App{
|
|
Key: edgeconnect.AppKey{
|
|
Organization: a.cfg.Organization,
|
|
Name: instancename,
|
|
Version: "0.0.1",
|
|
},
|
|
Deployment: "kubernetes",
|
|
ImageType: "Docker",
|
|
ImagePath: "edp.buildth.ing/devfw-cicd/garm-act-runner:1",
|
|
AllowServerless: true,
|
|
ServerlessConfig: struct{}{},
|
|
DefaultFlavor: edgeconnect.Flavor{
|
|
Name: "EU.small",
|
|
},
|
|
DeploymentGenerator: "kubernetes-basic",
|
|
DeploymentManifest: string(manifest),
|
|
RequiredOutboundConnections: []edgeconnect.SecurityRule{
|
|
edgeconnect.SecurityRule{
|
|
PortRangeMax: 65535,
|
|
PortRangeMin: 1,
|
|
Protocol: "TCP",
|
|
RemoteCIDR: "0.0.0.0/0",
|
|
},
|
|
edgeconnect.SecurityRule{
|
|
PortRangeMax: 65535,
|
|
PortRangeMin: 1,
|
|
Protocol: "UDP",
|
|
RemoteCIDR: "0.0.0.0/0",
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
}
|
|
|
|
_, err = a.client.ShowAppInstance(ctx, edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
Name: instancename,
|
|
CloudletKey: edgeconnect.CloudletKey(a.cfg.CloudletKey),
|
|
}, a.cfg.Region)
|
|
if err != nil && !errors.Is(err, edgeconnect.ErrResourceNotFound) {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
if errors.Is(err, edgeconnect.ErrResourceNotFound) {
|
|
err = a.client.CreateAppInstance(ctx, &edgeconnect.NewAppInstanceInput{
|
|
Region: a.cfg.Region,
|
|
AppInst: edgeconnect.AppInstance{
|
|
Key: edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
Name: instancename,
|
|
CloudletKey: edgeconnect.CloudletKey(a.cfg.CloudletKey),
|
|
},
|
|
AppKey: edgeconnect.AppKey{
|
|
Organization: a.cfg.Organization,
|
|
Name: instancename,
|
|
Version: "0.0.1",
|
|
},
|
|
Flavor: edgeconnect.Flavor{
|
|
Name: "EU.small",
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
}
|
|
|
|
instance := params.ProviderInstance{
|
|
ProviderID: instancename,
|
|
Name: instancename,
|
|
OSType: params.Linux,
|
|
OSArch: params.Amd64,
|
|
OSName: "lala",
|
|
OSVersion: "lalatest",
|
|
Status: params.InstanceRunning,
|
|
}
|
|
|
|
return instance, nil
|
|
}
|
|
|
|
func GetRunnerEnvs(gitHubScope GitHubScopeDetails, bootstrapParams params.BootstrapInstance) []corev1.EnvVar {
|
|
return []corev1.EnvVar{
|
|
{
|
|
Name: "RUNNER_ORG",
|
|
Value: gitHubScope.Org,
|
|
},
|
|
{
|
|
Name: "RUNNER_REPO",
|
|
Value: gitHubScope.Repo,
|
|
},
|
|
{
|
|
Name: "RUNNER_ENTERPRISE",
|
|
Value: gitHubScope.Enterprise,
|
|
},
|
|
{
|
|
Name: "RUNNER_GROUP",
|
|
Value: bootstrapParams.GitHubRunnerGroup,
|
|
},
|
|
{
|
|
Name: "RUNNER_NAME",
|
|
Value: bootstrapParams.Name,
|
|
},
|
|
{
|
|
Name: "RUNNER_LABELS",
|
|
Value: strings.Join(bootstrapParams.Labels, ","),
|
|
},
|
|
{
|
|
Name: "RUNNER_NO_DEFAULT_LABELS",
|
|
Value: "true",
|
|
},
|
|
{
|
|
Name: "DISABLE_RUNNER_UPDATE",
|
|
Value: "true",
|
|
},
|
|
{
|
|
Name: "RUNNER_WORKDIR",
|
|
Value: "/runner/_work/",
|
|
},
|
|
{
|
|
Name: "GITHUB_URL",
|
|
Value: gitHubScope.BaseURL,
|
|
},
|
|
{
|
|
Name: "RUNNER_EPHEMERAL",
|
|
Value: "true",
|
|
},
|
|
{
|
|
Name: "RUNNER_TOKEN",
|
|
Value: "dummy",
|
|
},
|
|
{
|
|
Name: "METADATA_URL",
|
|
Value: bootstrapParams.MetadataURL,
|
|
},
|
|
{
|
|
Name: "BEARER_TOKEN",
|
|
Value: bootstrapParams.InstanceToken,
|
|
},
|
|
{
|
|
Name: "CALLBACK_URL",
|
|
Value: bootstrapParams.CallbackURL,
|
|
},
|
|
{
|
|
Name: "JIT_CONFIG_ENABLED",
|
|
Value: fmt.Sprintf("%t", bootstrapParams.JitConfigEnabled),
|
|
},
|
|
}
|
|
}
|
|
|
|
func ExtractGitHubScopeDetails(gitRepoURL string) (GitHubScopeDetails, error) {
|
|
if gitRepoURL == "" {
|
|
return GitHubScopeDetails{}, fmt.Errorf("no gitRepoURL supplied")
|
|
}
|
|
u, err := url.Parse(gitRepoURL)
|
|
if err != nil {
|
|
return GitHubScopeDetails{}, fmt.Errorf("invalid URL: %w", err)
|
|
}
|
|
|
|
if u.Scheme == "" || u.Host == "" {
|
|
return GitHubScopeDetails{}, fmt.Errorf("invalid URL: %s", gitRepoURL)
|
|
}
|
|
|
|
pathParts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
|
|
|
scope := GitHubScopeDetails{
|
|
BaseURL: u.Scheme + "://" + u.Host,
|
|
}
|
|
|
|
switch {
|
|
case len(pathParts) == 1:
|
|
scope.Org = pathParts[0]
|
|
case len(pathParts) == 2 && pathParts[0] == "enterprises":
|
|
scope.Enterprise = pathParts[1]
|
|
case len(pathParts) == 2:
|
|
scope.Org = pathParts[0]
|
|
scope.Repo = pathParts[1]
|
|
default:
|
|
return GitHubScopeDetails{}, fmt.Errorf("URL does not match the expected patterns")
|
|
}
|
|
|
|
return scope, nil
|
|
}
|
|
|
|
// Delete instance will delete the instance in a provider.
|
|
func (a *edgeConnectProvider) DeleteInstance(ctx context.Context, instance string) error {
|
|
log.Printf("Executing DeleteInstance %s\n", instance)
|
|
|
|
appsinstances, err := a.client.ShowAppInstances(ctx, edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
}, a.cfg.Region)
|
|
if err != nil {
|
|
log.Printf("Error in method DeleteInstance() ShowAppInstances")
|
|
return err
|
|
}
|
|
|
|
log.Printf("got %d appinstances", len(appsinstances))
|
|
|
|
myappintances := filter(appsinstances, func(app edgeconnect.AppInstance) bool {
|
|
return strings.HasSuffix(app.Key.Name, strings.ToLower(instance))
|
|
})
|
|
|
|
log.Printf("filtered %d appinstances", len(myappintances))
|
|
|
|
for _, v := range myappintances {
|
|
err = a.client.DeleteAppInstance(ctx, v.Key, a.cfg.Region)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
apps, err := a.client.ShowApps(ctx, edgeconnect.AppKey{
|
|
Organization: a.cfg.Organization,
|
|
}, a.cfg.Region)
|
|
if err != nil {
|
|
log.Printf("Error in method DeleteInstance() ShowApps")
|
|
return err
|
|
}
|
|
|
|
myapps := filter(apps, func(app edgeconnect.App) bool {
|
|
return strings.HasSuffix(app.Key.Name, strings.ToLower(instance))
|
|
})
|
|
|
|
for _, v := range myapps {
|
|
err = a.client.DeleteApp(ctx, v.Key, a.cfg.Region)
|
|
if err != nil {
|
|
log.Printf("Error in method DeleteInstance() DeleteApp")
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetInstance will return details about one instance.
|
|
func (a *edgeConnectProvider) GetInstance(ctx context.Context, instance string) (params.ProviderInstance, error) {
|
|
log.Printf("Executing GetInstance %s\n", instance)
|
|
providerInstance := params.ProviderInstance{
|
|
ProviderID: a.controllerID,
|
|
Name: instance,
|
|
OSType: params.Linux,
|
|
OSArch: params.Amd64,
|
|
OSName: "lala",
|
|
OSVersion: "lalatest",
|
|
Status: params.InstanceStatusUnknown,
|
|
}
|
|
|
|
appinstances, err := a.client.ShowAppInstances(ctx, edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
}, a.cfg.Region)
|
|
if err != nil {
|
|
return params.ProviderInstance{}, err
|
|
}
|
|
|
|
myappintances := filter(appinstances, func(app edgeconnect.AppInstance) bool {
|
|
return strings.HasSuffix(app.Key.Name, strings.ToLower(instance))
|
|
})
|
|
|
|
if len(myappintances) == 0 {
|
|
return params.ProviderInstance{}, fmt.Errorf("AppInstance not found!")
|
|
}
|
|
|
|
appinst := myappintances[0]
|
|
|
|
if appinst.State == "Ready" {
|
|
providerInstance.Status = params.InstanceRunning
|
|
}
|
|
|
|
return providerInstance, nil
|
|
}
|
|
|
|
// ListInstances will list all instances for a provider.
|
|
func (a *edgeConnectProvider) ListInstances(ctx context.Context, poolID string) ([]params.ProviderInstance, error) {
|
|
log.Printf("Executing ListInstances for PoolID %s\n", poolID)
|
|
|
|
apps, err := a.client.ShowAppInstances(ctx, edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
}, a.cfg.Region)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
myappintances := filter(apps, func(app edgeconnect.AppInstance) bool {
|
|
return strings.HasPrefix(app.Key.Name, fmt.Sprintf("garm-%v-%v", a.controllerID[:8], poolID[:8]))
|
|
})
|
|
|
|
providerinstances := []params.ProviderInstance{}
|
|
|
|
for _, v := range myappintances {
|
|
providerInstance := params.ProviderInstance{
|
|
ProviderID: a.controllerID,
|
|
Name: v.Key.Name,
|
|
OSType: params.Linux,
|
|
OSArch: params.Amd64,
|
|
OSName: "lala",
|
|
OSVersion: "lalatest",
|
|
Status: params.InstanceStatusUnknown,
|
|
}
|
|
|
|
if v.State == "Ready" {
|
|
providerInstance.Status = params.InstanceRunning
|
|
}
|
|
|
|
providerinstances = append(providerinstances, providerInstance)
|
|
}
|
|
|
|
return providerinstances, nil
|
|
}
|
|
|
|
func filter[T any](s []T, predicate func(T) bool) []T {
|
|
result := make([]T, 0, len(s)) // Pre-allocate for efficiency
|
|
for _, v := range s {
|
|
if predicate(v) {
|
|
result = append(result, v)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// RemoveAllInstances will remove all instances created by this provider.
|
|
func (a *edgeConnectProvider) RemoveAllInstances(ctx context.Context) error {
|
|
log.Printf("Executing RemoveAllInstances\n")
|
|
|
|
apps, err := a.client.ShowAppInstances(ctx, edgeconnect.AppInstanceKey{
|
|
Organization: a.cfg.Organization,
|
|
}, a.cfg.Region)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
myappintances := filter(apps, func(app edgeconnect.AppInstance) bool {
|
|
return strings.HasPrefix(app.Key.Name, fmt.Sprintf("garm-%v", a.controllerID[:8]))
|
|
})
|
|
|
|
for _, v := range myappintances {
|
|
err = a.DeleteInstance(ctx, v.Key.Name)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Stop shuts down the instance.
|
|
func (a *edgeConnectProvider) Stop(ctx context.Context, instance string, force bool) error {
|
|
log.Printf("Executing Stop %s\n", instance)
|
|
panic(fmt.Sprintf("Stop() not implemented, called with instance: %s, force: %t", instance, force))
|
|
}
|
|
|
|
// Start boots up an instance.
|
|
func (a *edgeConnectProvider) Start(ctx context.Context, instance string) error {
|
|
log.Printf("Executing Start %s\n", instance)
|
|
panic(fmt.Sprintf("Start() not implemented, called with instance: %s", instance))
|
|
}
|
|
|
|
// GetVersion returns the version of the provider.
|
|
func (a *edgeConnectProvider) GetVersion(ctx context.Context) string {
|
|
return Version
|
|
}
|