From bf0a5bf147b1ec024622d2b6b7bfef68ba0598a9 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 19 Apr 2022 20:22:50 +0000 Subject: [PATCH] LXD provider implementation finished implementation of LXD provider. --- cmd/runner-manager/main.go | 112 ++++++++++++------------- params/params.go | 29 +++---- params/requests.go | 8 +- runner/common/provider.go | 10 +-- runner/providers/lxd/lxd.go | 157 ++++++++++++++++++++--------------- runner/providers/lxd/util.go | 105 +++++++++++++++++++++++ 6 files changed, 276 insertions(+), 145 deletions(-) create mode 100644 runner/providers/lxd/util.go diff --git a/cmd/runner-manager/main.go b/cmd/runner-manager/main.go index d2f6d94b..f26b8e15 100644 --- a/cmd/runner-manager/main.go +++ b/cmd/runner-manager/main.go @@ -7,16 +7,13 @@ import ( "log" "os/signal" - "runner-manager/cloudconfig" "runner-manager/config" - "runner-manager/params" "runner-manager/runner" "runner-manager/runner/providers/lxd" "runner-manager/util" - - "github.com/google/go-github/v43/github" - "golang.org/x/oauth2" - "gopkg.in/yaml.v3" + // "github.com/google/go-github/v43/github" + // "golang.org/x/oauth2" + // "gopkg.in/yaml.v3" ) var ( @@ -44,13 +41,13 @@ func main() { log.Fatal(err) } - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: cfg.Github.OAuth2Token}, - ) + // ts := oauth2.StaticTokenSource( + // &oauth2.Token{AccessToken: cfg.Github.OAuth2Token}, + // ) - tc := oauth2.NewClient(ctx, ts) + // tc := oauth2.NewClient(ctx, ts) - ghClient := github.NewClient(tc) + // ghClient := github.NewClient(tc) // // list all repositories for the authenticated user // repos, _, err := client.Repositories.List(ctx, "", nil) @@ -63,65 +60,68 @@ func main() { } log.SetOutput(logWriter) - runnerWorker, err := runner.NewRunner(ctx, cfg) - - fmt.Println(runnerWorker) - - cloudCfg := cloudconfig.NewDefaultCloudInitConfig() - cloudCfg.AddPackage("wget", "bmon", "wget") - cloudCfg.AddFile(nil, "/home/runner/hi.txt", "runner:runner", "0755") - asStr, err := cloudCfg.Serialize() - if err != nil { - log.Fatal(err) - } - fmt.Println(asStr) - runner, err := runner.NewRunner(ctx, cfg) if err != nil { log.Fatal(err) } fmt.Println(runner) - - provider, err := lxd.NewProvider(ctx, &cfg.Providers[0], &cfg.Repositories[0].Pool) + controllerID := "026d374d-6a8a-4241-8ed9-a246fff6762f" + provider, err := lxd.NewProvider(ctx, &cfg.Providers[0], &cfg.Repositories[0].Pool, controllerID) if err != nil { log.Fatal(err) } - fmt.Println(provider) - - log.Print("Fetching tools") - tools, _, err := ghClient.Actions.ListRunnerApplicationDownloads(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) - if err != nil { + if err := provider.RemoveAllInstances(ctx); err != nil { log.Fatal(err) } - toolsAsYaml, err := yaml.Marshal(tools) - if err != nil { - log.Fatal(err) - } - log.Printf("got tools:\n%s\n", string(toolsAsYaml)) + // fmt.Println(provider) - log.Print("fetching runner token") - ghRunnerToken, _, err := ghClient.Actions.CreateRegistrationToken(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) - if err != nil { - log.Fatal(err) - } - log.Printf("got token %v", ghRunnerToken) + // if err := provider.DeleteInstance(ctx, "runner-manager-2fbe5354-be28-4e00-95a8-11479912368d"); err != nil { + // log.Fatal(err) + // } - bootstrapArgs := params.BootstrapInstance{ - Tools: tools, - RepoURL: cfg.Repositories[0].String(), - GithubRunnerAccessToken: *ghRunnerToken.Token, - RunnerType: cfg.Repositories[0].Pool.Runners[0].Name, - CallbackURL: "", - InstanceToken: "", - SSHKeys: []string{ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2oT7j/+elHY9U2ibgk2RYJgCvqIwewYKJTtHslTQFDWlHLeDam93BBOFlQJm9/wKX/qjC8d26qyzjeeeVf2EEAztp+jQfEq9OU+EtgQUi589jxtVmaWuYED8KVNbzLuP79SrBtEZD4xqgmnNotPhRshh3L6eYj4XzLWDUuOD6kzNdsJA2QOKeMOIFpBN6urKJHRHYD+oUPUX1w5QMv1W1Srlffl4m5uE+0eJYAMr02980PG4+jS4bzM170wYdWwUI0pSZsEDC8Fn7jef6QARU2CgHJYlaTem+KWSXislOUTaCpR0uhakP1ezebW20yuuc3bdRNgSlZi9B7zAPALGZpOshVqwF+KmLDi6XiFwG+NnwAFa6zaQfhOxhw/rF5Jk/wVjHIHkNNvYewycZPbKui0E3QrdVtR908N3VsPtLhMQ59BEMl3xlURSi0fiOU3UjnwmOkOoFDy/WT8qk//gFD93tUxlf4eKXDgNfME3zNz8nVi2uCPvG5NT/P/VWR8NMqW6tZcmWyswM/GgL6Y84JQ3ESZq/7WvAetdc1gVIDQJ2ejYbSHBcQpWvkocsiuMTCwiEvQ0sr+UE5jmecQvLPUyXOhuMhw43CwxnLk1ZSeYeCorxbskyqIXH71o8zhbPoPiEbwgB+i9WEoq02u7c8CmCmO8Y9aOnh8MzTKxIgQ==", - }, - } + // instances, err := provider.ListInstances(ctx) - if err := provider.CreateInstance(ctx, bootstrapArgs); err != nil { - log.Fatal(err) - } + // asJs, err := json.MarshalIndent(instances, "", " ") + // fmt.Println(string(asJs), err) + + // log.Print("Fetching tools") + // tools, _, err := ghClient.Actions.ListRunnerApplicationDownloads(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) + // if err != nil { + // log.Fatal(err) + // } + + // toolsAsYaml, err := yaml.Marshal(tools) + // if err != nil { + // log.Fatal(err) + // } + // log.Printf("got tools:\n%s\n", string(toolsAsYaml)) + + // log.Print("fetching runner token") + // ghRunnerToken, _, err := ghClient.Actions.CreateRegistrationToken(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) + // if err != nil { + // log.Fatal(err) + // } + // log.Printf("got token %v", ghRunnerToken) + + // bootstrapArgs := params.BootstrapInstance{ + // Tools: tools, + // RepoURL: cfg.Repositories[0].String(), + // GithubRunnerAccessToken: *ghRunnerToken.Token, + // RunnerType: cfg.Repositories[0].Pool.Runners[0].Name, + // CallbackURL: "", + // InstanceToken: "", + // SSHKeys: []string{ + // "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2oT7j/+elHY9U2ibgk2RYJgCvqIwewYKJTtHslTQFDWlHLeDam93BBOFlQJm9/wKX/qjC8d26qyzjeeeVf2EEAztp+jQfEq9OU+EtgQUi589jxtVmaWuYED8KVNbzLuP79SrBtEZD4xqgmnNotPhRshh3L6eYj4XzLWDUuOD6kzNdsJA2QOKeMOIFpBN6urKJHRHYD+oUPUX1w5QMv1W1Srlffl4m5uE+0eJYAMr02980PG4+jS4bzM170wYdWwUI0pSZsEDC8Fn7jef6QARU2CgHJYlaTem+KWSXislOUTaCpR0uhakP1ezebW20yuuc3bdRNgSlZi9B7zAPALGZpOshVqwF+KmLDi6XiFwG+NnwAFa6zaQfhOxhw/rF5Jk/wVjHIHkNNvYewycZPbKui0E3QrdVtR908N3VsPtLhMQ59BEMl3xlURSi0fiOU3UjnwmOkOoFDy/WT8qk//gFD93tUxlf4eKXDgNfME3zNz8nVi2uCPvG5NT/P/VWR8NMqW6tZcmWyswM/GgL6Y84JQ3ESZq/7WvAetdc1gVIDQJ2ejYbSHBcQpWvkocsiuMTCwiEvQ0sr+UE5jmecQvLPUyXOhuMhw43CwxnLk1ZSeYeCorxbskyqIXH71o8zhbPoPiEbwgB+i9WEoq02u7c8CmCmO8Y9aOnh8MzTKxIgQ==", + // }, + // } + + // instance, err := provider.CreateInstance(ctx, bootstrapArgs) + // if err != nil { + // log.Fatal(err) + // } + + // fmt.Println(instance) } diff --git a/params/params.go b/params/params.go index c1203508..ce77b9eb 100644 --- a/params/params.go +++ b/params/params.go @@ -1,12 +1,9 @@ package params -import "github.com/google/go-github/v43/github" +import ( + "runner-manager/config" -type OSType string - -const ( - Linux OSType = "linux" - Windows OSType = "windows" + "github.com/google/go-github/v43/github" ) type Instance struct { @@ -21,14 +18,18 @@ type Instance struct { Name string `json:"name,omitempty"` // OSType is the operating system type. For now, only Linux and // Windows are supported. - OSType OSType `json:"os_type,omitempty"` + OSType config.OSType `json:"os-type,omitempty"` + // OSName is the name of the OS. Eg: ubuntu, centos, etc. + OSName string `json:"os-name,omitempty"` // OSVersion is the version of the operating system. - OSVersion string `json:"os_version,omitempty"` + OSVersion string `json:"os-version,omitempty"` // OSArch is the operating system architecture. - OSArch string `json:"os_arch,omitempty"` + OSArch string `json:"os-arch,omitempty"` // Addresses is a list of IP addresses the provider reports // for this instance. - Addresses []string `json:"ip_addresses,omitempty"` + Addresses []string `json:"ip-addresses,omitempty"` + // Status is the status of the instance inside the provider (eg: running, stopped, etc) + Status string `json:"status"` } type BootstrapInstance struct { @@ -42,14 +43,14 @@ type BootstrapInstance struct { // needs this to determine which flavor/image/settings it needs to use to create the // instance. This is provider/runner specific. The config for the runner type is defined // in the configuration file, as part of the pool definition. - RunnerType string `json:"runner_type"` + RunnerType string `json:"runner-type"` // CallbackUrl is the URL where the instance can send a post, signaling // progress or status. - CallbackURL string `json:"callback_url"` + CallbackURL string `json:"callback-url"` // InstanceToken is the token that needs to be set by the instance in the headers // in order to send updated back to the runner-manager via CallbackURL. - InstanceToken string `json:"instance_token"` + InstanceToken string `json:"instance-token"` // SSHKeys are the ssh public keys we may want to inject inside the runners, if the // provider supports it. - SSHKeys []string `json:"ssh_keys"` + SSHKeys []string `json:"ssh-keys"` } diff --git a/params/requests.go b/params/requests.go index cb4e50d4..8e8bac92 100644 --- a/params/requests.go +++ b/params/requests.go @@ -1,7 +1,9 @@ package params +import "runner-manager/config" + type InstanceRequest struct { - Name string `json:"name"` - OSType OSType `json:"os_type"` - OSVersion string `json:"os_version"` + Name string `json:"name"` + OSType config.OSType `json:"os_type"` + OSVersion string `json:"os_version"` } diff --git a/runner/common/provider.go b/runner/common/provider.go index 53eb5ded..95e4cc7b 100644 --- a/runner/common/provider.go +++ b/runner/common/provider.go @@ -7,17 +7,17 @@ import ( type Provider interface { // CreateInstance creates a new compute instance in the provider. - CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error + CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) // Delete instance will delete the instance in a provider. DeleteInstance(ctx context.Context, instance string) error + // GetInstance will return details about one instance. + GetInstance(ctx context.Context, instance string) (params.Instance, error) // ListInstances will list all instances for a provider. - ListInstances(ctx context.Context) error + ListInstances(ctx context.Context) ([]params.Instance, error) // RemoveAllInstances will remove all instances created by this provider. RemoveAllInstances(ctx context.Context) error - // Status returns the status of one instance. - Status(ctx context.Context, instance string) error // Stop shuts down the instance. - Stop(ctx context.Context, instance string) error + Stop(ctx context.Context, instance string, force bool) error // Start boots up an instance. Start(ctx context.Context, instance string) error } diff --git a/runner/providers/lxd/lxd.go b/runner/providers/lxd/lxd.go index 6e2247dc..279c6a65 100644 --- a/runner/providers/lxd/lxd.go +++ b/runner/providers/lxd/lxd.go @@ -3,7 +3,6 @@ package lxd import ( "context" "fmt" - "io/ioutil" "strings" "runner-manager/cloudconfig" @@ -23,6 +22,12 @@ import ( var _ common.Provider = &LXD{} +const ( + // We look for this key in the config of the instances to determine if they are + // created by us or not. + controllerIDKeyName = "user.runner-controller-id" +) + var ( // lxdToGithubArchMap translates LXD architectures to Github tools architectures. // TODO: move this in a separate package. This will most likely be used @@ -49,58 +54,7 @@ const ( DefaultProjectName = "runner-manager-project" ) -func getClientFromConfig(ctx context.Context, cfg *config.LXD) (cli lxd.InstanceServer, err error) { - if cfg.UnixSocket != "" { - return lxd.ConnectLXDUnixWithContext(ctx, cfg.UnixSocket, nil) - } - - var srvCrtContents, tlsCAContents, clientCertContents, clientKeyContents []byte - - if cfg.TLSServerCert != "" { - srvCrtContents, err = ioutil.ReadFile(cfg.TLSServerCert) - if err != nil { - return nil, errors.Wrap(err, "reading TLSServerCert") - } - } - - if cfg.TLSCA != "" { - tlsCAContents, err = ioutil.ReadFile(cfg.TLSCA) - if err != nil { - return nil, errors.Wrap(err, "reading TLSCA") - } - } - - if cfg.ClientCertificate != "" { - clientCertContents, err = ioutil.ReadFile(cfg.ClientCertificate) - if err != nil { - return nil, errors.Wrap(err, "reading ClientCertificate") - } - } - - if cfg.ClientKey != "" { - clientKeyContents, err = ioutil.ReadFile(cfg.ClientKey) - if err != nil { - return nil, errors.Wrap(err, "reading ClientKey") - } - } - - connectArgs := lxd.ConnectionArgs{ - TLSServerCert: string(srvCrtContents), - TLSCA: string(tlsCAContents), - TLSClientCert: string(clientCertContents), - TLSClientKey: string(clientKeyContents), - } - return lxd.ConnectLXD(cfg.URL, &connectArgs) -} - -func projectName(cfg config.LXD) string { - if cfg.ProjectName != "" { - return cfg.ProjectName - } - return DefaultProjectName -} - -func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool) (common.Provider, error) { +func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool, controllerID string) (common.Provider, error) { if err := cfg.Validate(); err != nil { return nil, errors.Wrap(err, "validating provider config") } @@ -128,10 +82,11 @@ func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool) ( cli = cli.UseProject(projectName(cfg.LXD)) provider := &LXD{ - ctx: ctx, - cfg: cfg, - pool: pool, - cli: cli, + ctx: ctx, + cfg: cfg, + pool: pool, + cli: cli, + controllerID: controllerID, imageManager: &image{ cli: cli, remotes: cfg.LXD.ImageRemotes, @@ -153,6 +108,8 @@ type LXD struct { cli lxd.InstanceServer // imageManager downloads images from remotes imageManager *image + // controllerID is the ID of this controller + controllerID string } func (l *LXD) getProfiles(runner config.Runner) ([]string, error) { @@ -314,6 +271,7 @@ func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance) (a Config: map[string]string{ "user.user-data": cloudCfg, "security.secureboot": l.secureBootEnabled(), + controllerIDKeyName: l.controllerID, }, }, Source: api.InstanceSource{ @@ -359,43 +317,108 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error { } // CreateInstance creates a new compute instance in the provider. -func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error { +func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { args, err := l.getCreateInstanceArgs(bootstrapParams) if err != nil { - return errors.Wrap(err, "fetching create args") + return params.Instance{}, errors.Wrap(err, "fetching create args") } asJs, err := yaml.Marshal(args) fmt.Println(string(asJs), err) - return l.launchInstance(args) + if err := l.launchInstance(args); err != nil { + return params.Instance{}, errors.Wrap(err, "creating instance") + } + + return l.GetInstance(ctx, args.Name) +} + +// GetInstance will return details about one instance. +func (l *LXD) GetInstance(ctx context.Context, instanceName string) (params.Instance, error) { + instance, _, err := l.cli.GetInstanceFull(instanceName) + if err != nil { + return params.Instance{}, errors.Wrap(err, "fetching instance") + } + + return lxdInstanceToAPIInstance(instance), nil } // Delete instance will delete the instance in a provider. func (l *LXD) DeleteInstance(ctx context.Context, instance string) error { + if err := l.setState(instance, "start", true); err != nil { + return errors.Wrap(err, "stopping instance") + } + + op, err := l.cli.DeleteInstance(instance) + if err != nil { + return errors.Wrap(err, "removing instance") + } + + err = op.Wait() + if err != nil { + return errors.Wrap(err, "waiting for instance deletion") + } return nil } // ListInstances will list all instances for a provider. -func (l *LXD) ListInstances(ctx context.Context) error { - return nil +func (l *LXD) ListInstances(ctx context.Context) ([]params.Instance, error) { + instances, err := l.cli.GetInstancesFull(api.InstanceTypeAny) + if err != nil { + return []params.Instance{}, errors.Wrap(err, "fetching instances") + } + + ret := []params.Instance{} + + for _, instance := range instances { + if id, ok := instance.ExpandedConfig[controllerIDKeyName]; ok && id == l.controllerID { + ret = append(ret, lxdInstanceToAPIInstance(&instance)) + } + } + + return ret, nil } // RemoveAllInstances will remove all instances created by this provider. func (l *LXD) RemoveAllInstances(ctx context.Context) error { + instances, err := l.ListInstances(ctx) + if err != nil { + return errors.Wrap(err, "fetching instance list") + } + + for _, instance := range instances { + // TODO: remove in parallel + if err := l.DeleteInstance(ctx, instance.Name); err != nil { + return errors.Wrapf(err, "removing instance %s", instance.Name) + } + } + return nil } -// Status returns the instance status. -func (l *LXD) Status(ctx context.Context, instance string) error { +func (l *LXD) setState(instance, state string, force bool) error { + reqState := api.InstanceStatePut{ + Action: state, + Timeout: -1, + Force: force, + } + + op, err := l.cli.UpdateInstanceState(instance, reqState, "") + if err != nil { + return errors.Wrapf(err, "setting state to %s", state) + } + err = op.Wait() + if err != nil { + return errors.Wrapf(err, "waiting for instance to transition to state %s", state) + } return nil } // Stop shuts down the instance. -func (l *LXD) Stop(ctx context.Context, instance string) error { - return nil +func (l *LXD) Stop(ctx context.Context, instance string, force bool) error { + return l.setState(instance, "stop", force) } // Start boots up an instance. func (l *LXD) Start(ctx context.Context, instance string) error { - return nil + return l.setState(instance, "start", false) } diff --git a/runner/providers/lxd/util.go b/runner/providers/lxd/util.go new file mode 100644 index 00000000..537d44a5 --- /dev/null +++ b/runner/providers/lxd/util.go @@ -0,0 +1,105 @@ +package lxd + +import ( + "context" + "io/ioutil" + "log" + "runner-manager/config" + "runner-manager/params" + "runner-manager/util" + "strings" + + lxd "github.com/lxc/lxd/client" + "github.com/lxc/lxd/shared/api" + "github.com/pkg/errors" +) + +func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance { + os, ok := instance.ExpandedConfig["image.os"] + if !ok { + log.Printf("failed to find OS in instance config") + } + + osType, err := util.OSToOSType(os) + if err != nil { + log.Printf("failed to find OS type for OS %s", os) + } + osRelease, ok := instance.ExpandedConfig["image.release"] + if !ok { + log.Printf("failed to find OS release instance config") + } + + state := instance.State + addresses := []string{} + if state.Network != nil { + for _, details := range state.Network { + for _, addr := range details.Addresses { + if addr.Scope != "global" { + continue + } + addresses = append(addresses, addr.Address) + } + } + } + return params.Instance{ + OSArch: instance.Architecture, + ProviderID: instance.Name, + Name: instance.Name, + OSType: osType, + OSName: strings.ToLower(os), + OSVersion: osRelease, + Addresses: addresses, + Status: state.Status, + } +} + +func getClientFromConfig(ctx context.Context, cfg *config.LXD) (cli lxd.InstanceServer, err error) { + if cfg.UnixSocket != "" { + return lxd.ConnectLXDUnixWithContext(ctx, cfg.UnixSocket, nil) + } + + var srvCrtContents, tlsCAContents, clientCertContents, clientKeyContents []byte + + if cfg.TLSServerCert != "" { + srvCrtContents, err = ioutil.ReadFile(cfg.TLSServerCert) + if err != nil { + return nil, errors.Wrap(err, "reading TLSServerCert") + } + } + + if cfg.TLSCA != "" { + tlsCAContents, err = ioutil.ReadFile(cfg.TLSCA) + if err != nil { + return nil, errors.Wrap(err, "reading TLSCA") + } + } + + if cfg.ClientCertificate != "" { + clientCertContents, err = ioutil.ReadFile(cfg.ClientCertificate) + if err != nil { + return nil, errors.Wrap(err, "reading ClientCertificate") + } + } + + if cfg.ClientKey != "" { + clientKeyContents, err = ioutil.ReadFile(cfg.ClientKey) + if err != nil { + return nil, errors.Wrap(err, "reading ClientKey") + } + } + + connectArgs := lxd.ConnectionArgs{ + TLSServerCert: string(srvCrtContents), + TLSCA: string(tlsCAContents), + TLSClientCert: string(clientCertContents), + TLSClientKey: string(clientKeyContents), + } + return lxd.ConnectLXD(cfg.URL, &connectArgs) +} + +func projectName(cfg config.LXD) string { + if cfg.ProjectName != "" { + return cfg.ProjectName + } + return DefaultProjectName +}