DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

LXD provider implementation

Browse source 
finished implementation of LXD provider.
This commit is contained in:
Gabriel Adrian Samfira 2022-04-19 20:22:50 +00:00
parent d68b842375
commit bf0a5bf147
6 changed files with 276 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

112
cmd/runner-manager/main.go
View file

@ -7,16 +7,13 @@ import (
"log"
"os/signal"
"runner-manager/cloudconfig"
"runner-manager/config"
"runner-manager/params"
"runner-manager/runner"
"runner-manager/runner/providers/lxd"
"runner-manager/util"
"github.com/google/go-github/v43/github"
"golang.org/x/oauth2"
"gopkg.in/yaml.v3"
// "github.com/google/go-github/v43/github"
// "golang.org/x/oauth2"
// "gopkg.in/yaml.v3"
)
var (
@ -44,13 +41,13 @@ func main() {
log.Fatal(err)
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: cfg.Github.OAuth2Token},
)
// ts := oauth2.StaticTokenSource(
// &oauth2.Token{AccessToken: cfg.Github.OAuth2Token},
// )
tc := oauth2.NewClient(ctx, ts)
// tc := oauth2.NewClient(ctx, ts)
ghClient := github.NewClient(tc)
// ghClient := github.NewClient(tc)
// // list all repositories for the authenticated user
// repos, _, err := client.Repositories.List(ctx, "", nil)
@ -63,65 +60,68 @@ func main() {
}
log.SetOutput(logWriter)
runnerWorker, err := runner.NewRunner(ctx, cfg)
fmt.Println(runnerWorker)
cloudCfg := cloudconfig.NewDefaultCloudInitConfig()
cloudCfg.AddPackage("wget", "bmon", "wget")
cloudCfg.AddFile(nil, "/home/runner/hi.txt", "runner:runner", "0755")
asStr, err := cloudCfg.Serialize()
if err != nil {
log.Fatal(err)
}
fmt.Println(asStr)
runner, err := runner.NewRunner(ctx, cfg)
if err != nil {
log.Fatal(err)
}
fmt.Println(runner)
provider, err := lxd.NewProvider(ctx, &cfg.Providers[0], &cfg.Repositories[0].Pool)
controllerID := "026d374d-6a8a-4241-8ed9-a246fff6762f"
provider, err := lxd.NewProvider(ctx, &cfg.Providers[0], &cfg.Repositories[0].Pool, controllerID)
if err != nil {
log.Fatal(err)
}
fmt.Println(provider)
log.Print("Fetching tools")
tools, _, err := ghClient.Actions.ListRunnerApplicationDownloads(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name)
if err != nil {
if err := provider.RemoveAllInstances(ctx); err != nil {
log.Fatal(err)
}
toolsAsYaml, err := yaml.Marshal(tools)
if err != nil {
log.Fatal(err)
}
log.Printf("got tools:\n%s\n", string(toolsAsYaml))
// fmt.Println(provider)
log.Print("fetching runner token")
ghRunnerToken, _, err := ghClient.Actions.CreateRegistrationToken(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name)
if err != nil {
log.Fatal(err)
}
log.Printf("got token %v", ghRunnerToken)
// if err := provider.DeleteInstance(ctx, "runner-manager-2fbe5354-be28-4e00-95a8-11479912368d"); err != nil {
// log.Fatal(err)
// }
bootstrapArgs := params.BootstrapInstance{
Tools: tools,
RepoURL: cfg.Repositories[0].String(),
GithubRunnerAccessToken: *ghRunnerToken.Token,
RunnerType: cfg.Repositories[0].Pool.Runners[0].Name,
CallbackURL: "",
InstanceToken: "",
SSHKeys: []string{
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2oT7j/+elHY9U2ibgk2RYJgCvqIwewYKJTtHslTQFDWlHLeDam93BBOFlQJm9/wKX/qjC8d26qyzjeeeVf2EEAztp+jQfEq9OU+EtgQUi589jxtVmaWuYED8KVNbzLuP79SrBtEZD4xqgmnNotPhRshh3L6eYj4XzLWDUuOD6kzNdsJA2QOKeMOIFpBN6urKJHRHYD+oUPUX1w5QMv1W1Srlffl4m5uE+0eJYAMr02980PG4+jS4bzM170wYdWwUI0pSZsEDC8Fn7jef6QARU2CgHJYlaTem+KWSXislOUTaCpR0uhakP1ezebW20yuuc3bdRNgSlZi9B7zAPALGZpOshVqwF+KmLDi6XiFwG+NnwAFa6zaQfhOxhw/rF5Jk/wVjHIHkNNvYewycZPbKui0E3QrdVtR908N3VsPtLhMQ59BEMl3xlURSi0fiOU3UjnwmOkOoFDy/WT8qk//gFD93tUxlf4eKXDgNfME3zNz8nVi2uCPvG5NT/P/VWR8NMqW6tZcmWyswM/GgL6Y84JQ3ESZq/7WvAetdc1gVIDQJ2ejYbSHBcQpWvkocsiuMTCwiEvQ0sr+UE5jmecQvLPUyXOhuMhw43CwxnLk1ZSeYeCorxbskyqIXH71o8zhbPoPiEbwgB+i9WEoq02u7c8CmCmO8Y9aOnh8MzTKxIgQ==",
},
}
// instances, err := provider.ListInstances(ctx)
if err := provider.CreateInstance(ctx, bootstrapArgs); err != nil {
log.Fatal(err)
}
// asJs, err := json.MarshalIndent(instances, "", " ")
// fmt.Println(string(asJs), err)
// log.Print("Fetching tools")
// tools, _, err := ghClient.Actions.ListRunnerApplicationDownloads(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name)
// if err != nil {
// log.Fatal(err)
// }
// toolsAsYaml, err := yaml.Marshal(tools)
// if err != nil {
// log.Fatal(err)
// }
// log.Printf("got tools:\n%s\n", string(toolsAsYaml))
// log.Print("fetching runner token")
// ghRunnerToken, _, err := ghClient.Actions.CreateRegistrationToken(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name)
// if err != nil {
// log.Fatal(err)
// }
// log.Printf("got token %v", ghRunnerToken)
// bootstrapArgs := params.BootstrapInstance{
// Tools: tools,
// RepoURL: cfg.Repositories[0].String(),
// GithubRunnerAccessToken: *ghRunnerToken.Token,
// RunnerType: cfg.Repositories[0].Pool.Runners[0].Name,
// CallbackURL: "",
// InstanceToken: "",
// SSHKeys: []string{
// "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2oT7j/+elHY9U2ibgk2RYJgCvqIwewYKJTtHslTQFDWlHLeDam93BBOFlQJm9/wKX/qjC8d26qyzjeeeVf2EEAztp+jQfEq9OU+EtgQUi589jxtVmaWuYED8KVNbzLuP79SrBtEZD4xqgmnNotPhRshh3L6eYj4XzLWDUuOD6kzNdsJA2QOKeMOIFpBN6urKJHRHYD+oUPUX1w5QMv1W1Srlffl4m5uE+0eJYAMr02980PG4+jS4bzM170wYdWwUI0pSZsEDC8Fn7jef6QARU2CgHJYlaTem+KWSXislOUTaCpR0uhakP1ezebW20yuuc3bdRNgSlZi9B7zAPALGZpOshVqwF+KmLDi6XiFwG+NnwAFa6zaQfhOxhw/rF5Jk/wVjHIHkNNvYewycZPbKui0E3QrdVtR908N3VsPtLhMQ59BEMl3xlURSi0fiOU3UjnwmOkOoFDy/WT8qk//gFD93tUxlf4eKXDgNfME3zNz8nVi2uCPvG5NT/P/VWR8NMqW6tZcmWyswM/GgL6Y84JQ3ESZq/7WvAetdc1gVIDQJ2ejYbSHBcQpWvkocsiuMTCwiEvQ0sr+UE5jmecQvLPUyXOhuMhw43CwxnLk1ZSeYeCorxbskyqIXH71o8zhbPoPiEbwgB+i9WEoq02u7c8CmCmO8Y9aOnh8MzTKxIgQ==",
// },
// }
// instance, err := provider.CreateInstance(ctx, bootstrapArgs)
// if err != nil {
// log.Fatal(err)
// }
// fmt.Println(instance)
}

29
params/params.go
View file

@ -1,12 +1,9 @@
package params
import "github.com/google/go-github/v43/github"
import (
"runner-manager/config"
type OSType string
const (
Linux OSType = "linux"
Windows OSType = "windows"
"github.com/google/go-github/v43/github"
)
type Instance struct {
@ -21,14 +18,18 @@ type Instance struct {
Name string `json:"name,omitempty"`
// OSType is the operating system type. For now, only Linux and
// Windows are supported.
OSType OSType `json:"os_type,omitempty"`
OSType config.OSType `json:"os-type,omitempty"`
// OSName is the name of the OS. Eg: ubuntu, centos, etc.
OSName string `json:"os-name,omitempty"`
// OSVersion is the version of the operating system.
OSVersion string `json:"os_version,omitempty"`
OSVersion string `json:"os-version,omitempty"`
// OSArch is the operating system architecture.
OSArch string `json:"os_arch,omitempty"`
OSArch string `json:"os-arch,omitempty"`
// Addresses is a list of IP addresses the provider reports
// for this instance.
Addresses []string `json:"ip_addresses,omitempty"`
Addresses []string `json:"ip-addresses,omitempty"`
// Status is the status of the instance inside the provider (eg: running, stopped, etc)
Status string `json:"status"`
}
type BootstrapInstance struct {
@ -42,14 +43,14 @@ type BootstrapInstance struct {
// needs this to determine which flavor/image/settings it needs to use to create the
// instance. This is provider/runner specific. The config for the runner type is defined
// in the configuration file, as part of the pool definition.
RunnerType string `json:"runner_type"`
RunnerType string `json:"runner-type"`
// CallbackUrl is the URL where the instance can send a post, signaling
// progress or status.
CallbackURL string `json:"callback_url"`
CallbackURL string `json:"callback-url"`
// InstanceToken is the token that needs to be set by the instance in the headers
// in order to send updated back to the runner-manager via CallbackURL.
InstanceToken string `json:"instance_token"`
InstanceToken string `json:"instance-token"`
// SSHKeys are the ssh public keys we may want to inject inside the runners, if the
// provider supports it.
SSHKeys []string `json:"ssh_keys"`
SSHKeys []string `json:"ssh-keys"`
}

8
params/requests.go
View file

@ -1,7 +1,9 @@
package params
import "runner-manager/config"
type InstanceRequest struct {
Name string `json:"name"`
OSType OSType `json:"os_type"`
OSVersion string `json:"os_version"`
Name string `json:"name"`
OSType config.OSType `json:"os_type"`
OSVersion string `json:"os_version"`
}

10
runner/common/provider.go
View file

@ -7,17 +7,17 @@ import (
type Provider interface {
// CreateInstance creates a new compute instance in the provider.
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error)
// Delete instance will delete the instance in a provider.
DeleteInstance(ctx context.Context, instance string) error
// GetInstance will return details about one instance.
GetInstance(ctx context.Context, instance string) (params.Instance, error)
// ListInstances will list all instances for a provider.
ListInstances(ctx context.Context) error
ListInstances(ctx context.Context) ([]params.Instance, error)
// RemoveAllInstances will remove all instances created by this provider.
RemoveAllInstances(ctx context.Context) error
// Status returns the status of one instance.
Status(ctx context.Context, instance string) error
// Stop shuts down the instance.
Stop(ctx context.Context, instance string) error
Stop(ctx context.Context, instance string, force bool) error
// Start boots up an instance.
Start(ctx context.Context, instance string) error
}

157
runner/providers/lxd/lxd.go
View file

@ -3,7 +3,6 @@ package lxd
import (
"context"
"fmt"
"io/ioutil"
"strings"
"runner-manager/cloudconfig"
@ -23,6 +22,12 @@ import (
var _ common.Provider = &LXD{}
const (
// We look for this key in the config of the instances to determine if they are
// created by us or not.
controllerIDKeyName = "user.runner-controller-id"
)
var (
// lxdToGithubArchMap translates LXD architectures to Github tools architectures.
// TODO: move this in a separate package. This will most likely be used
@ -49,58 +54,7 @@ const (
DefaultProjectName = "runner-manager-project"
)
func getClientFromConfig(ctx context.Context, cfg *config.LXD) (cli lxd.InstanceServer, err error) {
if cfg.UnixSocket != "" {
return lxd.ConnectLXDUnixWithContext(ctx, cfg.UnixSocket, nil)
}
var srvCrtContents, tlsCAContents, clientCertContents, clientKeyContents []byte
if cfg.TLSServerCert != "" {
srvCrtContents, err = ioutil.ReadFile(cfg.TLSServerCert)
if err != nil {
return nil, errors.Wrap(err, "reading TLSServerCert")
}
}
if cfg.TLSCA != "" {
tlsCAContents, err = ioutil.ReadFile(cfg.TLSCA)
if err != nil {
return nil, errors.Wrap(err, "reading TLSCA")
}
}
if cfg.ClientCertificate != "" {
clientCertContents, err = ioutil.ReadFile(cfg.ClientCertificate)
if err != nil {
return nil, errors.Wrap(err, "reading ClientCertificate")
}
}
if cfg.ClientKey != "" {
clientKeyContents, err = ioutil.ReadFile(cfg.ClientKey)
if err != nil {
return nil, errors.Wrap(err, "reading ClientKey")
}
}
connectArgs := lxd.ConnectionArgs{
TLSServerCert: string(srvCrtContents),
TLSCA: string(tlsCAContents),
TLSClientCert: string(clientCertContents),
TLSClientKey: string(clientKeyContents),
}
return lxd.ConnectLXD(cfg.URL, &connectArgs)
}
func projectName(cfg config.LXD) string {
if cfg.ProjectName != "" {
return cfg.ProjectName
}
return DefaultProjectName
}
func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool) (common.Provider, error) {
func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool, controllerID string) (common.Provider, error) {
if err := cfg.Validate(); err != nil {
return nil, errors.Wrap(err, "validating provider config")
}
@ -128,10 +82,11 @@ func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool) (
cli = cli.UseProject(projectName(cfg.LXD))
provider := &LXD{
ctx: ctx,
cfg: cfg,
pool: pool,
cli: cli,
ctx: ctx,
cfg: cfg,
pool: pool,
cli: cli,
controllerID: controllerID,
imageManager: &image{
cli: cli,
remotes: cfg.LXD.ImageRemotes,
@ -153,6 +108,8 @@ type LXD struct {
cli lxd.InstanceServer
// imageManager downloads images from remotes
imageManager *image
// controllerID is the ID of this controller
controllerID string
}
func (l *LXD) getProfiles(runner config.Runner) ([]string, error) {
@ -314,6 +271,7 @@ func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance) (a
Config: map[string]string{
"user.user-data": cloudCfg,
"security.secureboot": l.secureBootEnabled(),
controllerIDKeyName: l.controllerID,
},
},
Source: api.InstanceSource{
@ -359,43 +317,108 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error {
}
// CreateInstance creates a new compute instance in the provider.
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error {
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
args, err := l.getCreateInstanceArgs(bootstrapParams)
if err != nil {
return errors.Wrap(err, "fetching create args")
return params.Instance{}, errors.Wrap(err, "fetching create args")
}
asJs, err := yaml.Marshal(args)
fmt.Println(string(asJs), err)
return l.launchInstance(args)
if err := l.launchInstance(args); err != nil {
return params.Instance{}, errors.Wrap(err, "creating instance")
}
return l.GetInstance(ctx, args.Name)
}
// GetInstance will return details about one instance.
func (l *LXD) GetInstance(ctx context.Context, instanceName string) (params.Instance, error) {
instance, _, err := l.cli.GetInstanceFull(instanceName)
if err != nil {
return params.Instance{}, errors.Wrap(err, "fetching instance")
}
return lxdInstanceToAPIInstance(instance), nil
}
// Delete instance will delete the instance in a provider.
func (l *LXD) DeleteInstance(ctx context.Context, instance string) error {
if err := l.setState(instance, "start", true); err != nil {
return errors.Wrap(err, "stopping instance")
}
op, err := l.cli.DeleteInstance(instance)
if err != nil {
return errors.Wrap(err, "removing instance")
}
err = op.Wait()
if err != nil {
return errors.Wrap(err, "waiting for instance deletion")
}
return nil
}
// ListInstances will list all instances for a provider.
func (l *LXD) ListInstances(ctx context.Context) error {
return nil
func (l *LXD) ListInstances(ctx context.Context) ([]params.Instance, error) {
instances, err := l.cli.GetInstancesFull(api.InstanceTypeAny)
if err != nil {
return []params.Instance{}, errors.Wrap(err, "fetching instances")
}
ret := []params.Instance{}
for _, instance := range instances {
if id, ok := instance.ExpandedConfig[controllerIDKeyName]; ok && id == l.controllerID {
ret = append(ret, lxdInstanceToAPIInstance(&instance))
}
}
return ret, nil
}
// RemoveAllInstances will remove all instances created by this provider.
func (l *LXD) RemoveAllInstances(ctx context.Context) error {
instances, err := l.ListInstances(ctx)
if err != nil {
return errors.Wrap(err, "fetching instance list")
}
for _, instance := range instances {
// TODO: remove in parallel
if err := l.DeleteInstance(ctx, instance.Name); err != nil {
return errors.Wrapf(err, "removing instance %s", instance.Name)
}
}
return nil
}
// Status returns the instance status.
func (l *LXD) Status(ctx context.Context, instance string) error {
func (l *LXD) setState(instance, state string, force bool) error {
reqState := api.InstanceStatePut{
Action: state,
Timeout: -1,
Force: force,
}
op, err := l.cli.UpdateInstanceState(instance, reqState, "")
if err != nil {
return errors.Wrapf(err, "setting state to %s", state)
}
err = op.Wait()
if err != nil {
return errors.Wrapf(err, "waiting for instance to transition to state %s", state)
}
return nil
}
// Stop shuts down the instance.
func (l *LXD) Stop(ctx context.Context, instance string) error {
return nil
func (l *LXD) Stop(ctx context.Context, instance string, force bool) error {
return l.setState(instance, "stop", force)
}
// Start boots up an instance.
func (l *LXD) Start(ctx context.Context, instance string) error {
return nil
return l.setState(instance, "start", false)
}

105
runner/providers/lxd/util.go Normal file
View file

@ -0,0 +1,105 @@
package lxd
import (
"context"
"io/ioutil"
"log"
"runner-manager/config"
"runner-manager/params"
"runner-manager/util"
"strings"
lxd "github.com/lxc/lxd/client"
"github.com/lxc/lxd/shared/api"
"github.com/pkg/errors"
)
func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
os, ok := instance.ExpandedConfig["image.os"]
if !ok {
log.Printf("failed to find OS in instance config")
}
osType, err := util.OSToOSType(os)
if err != nil {
log.Printf("failed to find OS type for OS %s", os)
}
osRelease, ok := instance.ExpandedConfig["image.release"]
if !ok {
log.Printf("failed to find OS release instance config")
}
state := instance.State
addresses := []string{}
if state.Network != nil {
for _, details := range state.Network {
for _, addr := range details.Addresses {
if addr.Scope != "global" {
continue
}
addresses = append(addresses, addr.Address)
}
}
}
return params.Instance{
OSArch: instance.Architecture,
ProviderID: instance.Name,
Name: instance.Name,
OSType: osType,
OSName: strings.ToLower(os),
OSVersion: osRelease,
Addresses: addresses,
Status: state.Status,
}
}
func getClientFromConfig(ctx context.Context, cfg *config.LXD) (cli lxd.InstanceServer, err error) {
if cfg.UnixSocket != "" {
return lxd.ConnectLXDUnixWithContext(ctx, cfg.UnixSocket, nil)
}
var srvCrtContents, tlsCAContents, clientCertContents, clientKeyContents []byte
if cfg.TLSServerCert != "" {
srvCrtContents, err = ioutil.ReadFile(cfg.TLSServerCert)
if err != nil {
return nil, errors.Wrap(err, "reading TLSServerCert")
}
}
if cfg.TLSCA != "" {
tlsCAContents, err = ioutil.ReadFile(cfg.TLSCA)
if err != nil {
return nil, errors.Wrap(err, "reading TLSCA")
}
}
if cfg.ClientCertificate != "" {
clientCertContents, err = ioutil.ReadFile(cfg.ClientCertificate)
if err != nil {
return nil, errors.Wrap(err, "reading ClientCertificate")
}
}
if cfg.ClientKey != "" {
clientKeyContents, err = ioutil.ReadFile(cfg.ClientKey)
if err != nil {
return nil, errors.Wrap(err, "reading ClientKey")
}
}
connectArgs := lxd.ConnectionArgs{
TLSServerCert: string(srvCrtContents),
TLSCA: string(tlsCAContents),
TLSClientCert: string(clientCertContents),
TLSClientKey: string(clientKeyContents),
}
return lxd.ConnectLXD(cfg.URL, &connectArgs)
}
func projectName(cfg config.LXD) string {
if cfg.ProjectName != "" {
return cfg.ProjectName
}
return DefaultProjectName
}