From eb285421106b073614a9fa8fac8b63a5270d0327 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 18 Apr 2022 17:26:13 +0000 Subject: [PATCH] Added cloud config --- cloudconfig/cloudconfig.go | 142 +++++++++++++++++++++++++ cloudconfig/templates.go | 49 +++++++++ cmd/runner-manager/main.go | 76 ++++++++++++-- config/config.go | 28 ++++- go.mod | 6 +- go.sum | 9 +- params/params.go | 25 +++++ runner/common/provider.go | 5 +- runner/providers/lxd/cloud_config.go | 27 ----- runner/providers/lxd/lxd.go | 152 ++++++++++++++++++++++++--- runner/runner.go | 95 ++++++++++++++++- testdata/config.toml | 2 +- util/util.go | 70 +++++++++++- 13 files changed, 621 insertions(+), 65 deletions(-) create mode 100644 cloudconfig/cloudconfig.go create mode 100644 cloudconfig/templates.go delete mode 100644 runner/providers/lxd/cloud_config.go diff --git a/cloudconfig/cloudconfig.go b/cloudconfig/cloudconfig.go new file mode 100644 index 00000000..2f4aff26 --- /dev/null +++ b/cloudconfig/cloudconfig.go @@ -0,0 +1,142 @@ +package cloudconfig + +import ( + "encoding/base64" + "fmt" + "runner-manager/config" + "strings" + "sync" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +func NewDefaultCloudInitConfig() *CloudInit { + return &CloudInit{ + PackageUpgrade: true, + Packages: []string{ + "curl", + }, + SystemInfo: &SystemInfo{ + DefaultUser: DefaultUser{ + Name: config.DefaultUser, + Home: fmt.Sprintf("/home/%s", config.DefaultUser), + Shell: config.DefaultUserShell, + Groups: config.DefaultUserGroups, + Sudo: "ALL=(ALL) NOPASSWD:ALL", + }, + }, + } +} + +type DefaultUser struct { + Name string `yaml:"name"` + Home string `yaml:"home"` + Shell string `yaml:"shell"` + Groups []string `yaml:"groups,omitempty"` + Sudo string `yaml:"sudo"` +} + +type SystemInfo struct { + DefaultUser DefaultUser `yaml:"default_user"` +} + +type File struct { + Encoding string `yaml:"encoding"` + Content string `yaml:"content"` + Owner string `yaml:"owner"` + Path string `yaml:"path"` + Permissions string `yaml:"permissions"` +} + +type CloudInit struct { + mux sync.Mutex + + PackageUpgrade bool `yaml:"package_upgrade"` + Packages []string `yaml:"packages,omitempty"` + SSHAuthorizedKeys []string `yaml:"ssh_authorized_keys,omitempty"` + SystemInfo *SystemInfo `yaml:"system_info,omitempty"` + RunCmd []string `yaml:"runcmd,omitempty"` + WriteFiles []File `yaml:"write_files,omitempty"` +} + +func (c *CloudInit) AddSSHKey(keys ...string) { + c.mux.Lock() + defer c.mux.Unlock() + + // TODO(gabriel-samfira): Validate the SSH public key. + for _, key := range keys { + found := false + for _, val := range c.SSHAuthorizedKeys { + if val == key { + found = true + break + } + } + if !found { + c.SSHAuthorizedKeys = append(c.SSHAuthorizedKeys, key) + } + } +} + +func (c *CloudInit) AddPackage(pkgs ...string) { + c.mux.Lock() + defer c.mux.Unlock() + + for _, pkg := range pkgs { + found := false + for _, val := range c.Packages { + if val == pkg { + found = true + break + } + } + if !found { + c.Packages = append(c.Packages, pkg) + } + } +} + +func (c *CloudInit) AddRunCmd(cmd string) { + c.mux.Lock() + defer c.mux.Unlock() + + c.RunCmd = append(c.RunCmd, cmd) +} + +func (c *CloudInit) AddFile(contents []byte, path, owner, permissions string) { + c.mux.Lock() + defer c.mux.Unlock() + + for _, val := range c.WriteFiles { + if val.Path == path { + return + } + } + + file := File{ + Encoding: "b64", + Content: base64.StdEncoding.EncodeToString(contents), + Owner: owner, + Permissions: permissions, + Path: path, + } + c.WriteFiles = append(c.WriteFiles, file) +} + +func (c *CloudInit) Serialize() (string, error) { + c.mux.Lock() + defer c.mux.Unlock() + + ret := []string{ + "#cloud-config", + } + + asYaml, err := yaml.Marshal(c) + if err != nil { + return "", errors.Wrap(err, "marshaling to yaml") + } + + ret = append(ret, string(asYaml)) + return strings.Join(ret, "\n"), nil +} diff --git a/cloudconfig/templates.go b/cloudconfig/templates.go new file mode 100644 index 00000000..2c643fde --- /dev/null +++ b/cloudconfig/templates.go @@ -0,0 +1,49 @@ +package cloudconfig + +import ( + "bytes" + "text/template" + + "github.com/pkg/errors" +) + +var CloudConfigTemplate = ` +#!/bin/bash + +set -ex + +curl -o "/home/runner/{{ .FileName }}" "{{ .DownloadURL }}" +mkdir -p /home/runner/action-runner +tar xf "/home/runner/{{ .FileName }}" -C /home/runner/action-runner/ +chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R /home/{{ .RunnerUsername }}/action-runner/ +sudo /home/{{ .RunnerUsername }}/actions-runner/bin/installdependencies.sh +sudo -u {{ .RunnerUsername }} -- /home/{{ .RunnerUsername }}/actions-runner/config.sh --unattended --url "{{ .RepoURL }}" --token "{{ .GithubToken }}" --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral +/home/{{ .RunnerUsername }}/actions-runner/svc.sh install +/home/{{ .RunnerUsername }}/actions-runner/svc.sh start +` + +type InstallRunnerParams struct { + FileName string + DownloadURL string + RunnerUsername string + RunnerGroup string + RepoURL string + GithubToken string + RunnerName string + RunnerLabels string +} + +func InstallRunnerScript(params InstallRunnerParams) ([]byte, error) { + + t, err := template.New("").Parse(CloudConfigTemplate) + if err != nil { + return nil, errors.Wrap(err, "parsing template") + } + + var buf bytes.Buffer + if err := t.Execute(&buf, params); err != nil { + return nil, errors.Wrap(err, "rendering template") + } + + return buf.Bytes(), nil +} diff --git a/cmd/runner-manager/main.go b/cmd/runner-manager/main.go index 2e67ac37..0f30877c 100644 --- a/cmd/runner-manager/main.go +++ b/cmd/runner-manager/main.go @@ -7,9 +7,15 @@ import ( "log" "os/signal" + "runner-manager/cloudconfig" "runner-manager/config" + "runner-manager/params" "runner-manager/runner" + "runner-manager/runner/providers/lxd" "runner-manager/util" + + "github.com/google/go-github/v43/github" + "golang.org/x/oauth2" ) var ( @@ -37,6 +43,19 @@ func main() { log.Fatal(err) } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: cfg.Github.OAuth2Token}, + ) + + tc := oauth2.NewClient(ctx, ts) + + ghClient := github.NewClient(tc) + + // // list all repositories for the authenticated user + // repos, _, err := client.Repositories.List(ctx, "", nil) + + // fmt.Println(repos, err) + logWriter, err := util.GetLoggingWriter(cfg) if err != nil { log.Fatal(err) @@ -47,17 +66,56 @@ func main() { fmt.Println(runnerWorker) - // ctx := context.Background() - // ts := oauth2.StaticTokenSource( - // &oauth2.Token{AccessToken: token}, - // ) + cloudCfg := cloudconfig.NewDefaultCloudInitConfig() + cloudCfg.AddPackage("wget", "bmon", "wget") + cloudCfg.AddFile(nil, "/home/runner/hi.txt", "runner:runner", "0755") + asStr, err := cloudCfg.Serialize() + if err != nil { + log.Fatal(err) + } + fmt.Println(asStr) - // tc := oauth2.NewClient(ctx, ts) + runner, err := runner.NewRunner(ctx, cfg) + if err != nil { + log.Fatal(err) + } - // client := github.NewClient(tc) + fmt.Println(runner) - // // list all repositories for the authenticated user - // repos, _, err := client.Repositories.List(ctx, "", nil) + provider, err := lxd.NewProvider(ctx, &cfg.Providers[0], &cfg.Repositories[0].Pool) + if err != nil { + log.Fatal(err) + } - // fmt.Println(repos, err) + fmt.Println(provider) + + log.Print("Fetching tools") + tools, _, err := ghClient.Actions.ListRunnerApplicationDownloads(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) + if err != nil { + log.Fatal(err) + } + log.Printf("got tools: %v", tools) + + log.Print("fetching runner token") + ghRunnerToken, _, err := ghClient.Actions.CreateRegistrationToken(ctx, cfg.Repositories[0].Owner, cfg.Repositories[0].Name) + if err != nil { + log.Fatal(err) + } + log.Printf("got token %v", ghRunnerToken) + + bootstrapArgs := params.BootstrapInstance{ + Tools: tools, + RepoURL: cfg.Repositories[0].String(), + GithubRunnerAccessToken: *ghRunnerToken.Token, + RunnerType: cfg.Repositories[0].Pool.Runners[0].Name, + CallbackURL: "", + InstanceToken: "", + SSHKeys: []string{ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2oT7j/+elHY9U2ibgk2RYJgCvqIwewYKJTtHslTQFDWlHLeDam93BBOFlQJm9/wKX/qjC8d26qyzjeeeVf2EEAztp+jQfEq9OU+EtgQUi589jxtVmaWuYED8KVNbzLuP79SrBtEZD4xqgmnNotPhRshh3L6eYj4XzLWDUuOD6kzNdsJA2QOKeMOIFpBN6urKJHRHYD+oUPUX1w5QMv1W1Srlffl4m5uE+0eJYAMr02980PG4+jS4bzM170wYdWwUI0pSZsEDC8Fn7jef6QARU2CgHJYlaTem+KWSXislOUTaCpR0uhakP1ezebW20yuuc3bdRNgSlZi9B7zAPALGZpOshVqwF+KmLDi6XiFwG+NnwAFa6zaQfhOxhw/rF5Jk/wVjHIHkNNvYewycZPbKui0E3QrdVtR908N3VsPtLhMQ59BEMl3xlURSi0fiOU3UjnwmOkOoFDy/WT8qk//gFD93tUxlf4eKXDgNfME3zNz8nVi2uCPvG5NT/P/VWR8NMqW6tZcmWyswM/GgL6Y84JQ3ESZq/7WvAetdc1gVIDQJ2ejYbSHBcQpWvkocsiuMTCwiEvQ0sr+UE5jmecQvLPUyXOhuMhw43CwxnLk1ZSeYeCorxbskyqIXH71o8zhbPoPiEbwgB+i9WEoq02u7c8CmCmO8Y9aOnh8MzTKxIgQ==", + }, + } + + if err := provider.CreateInstance(ctx, bootstrapArgs); err != nil { + log.Fatal(err) + } } diff --git a/config/config.go b/config/config.go index 5513a400..5d477cfd 100644 --- a/config/config.go +++ b/config/config.go @@ -35,11 +35,28 @@ const ( // DefaultConfigFilePath is the default path on disk to the runner-manager // configuration file. DefaultConfigFilePath = "/etc/runner-manager/config.toml" + // DefaultConfigDir is the default path on disk to the config dir. The config + // file will probably be in the same folder, but it is not mandatory. + DefaultConfigDir = "/etc/runner-manager" + + // DefaultUser is the default username that should exist on the instances. + DefaultUser = "runner" + // DefaultUserShell is the shell for the default user. + DefaultUserShell = "/bin/bash" +) + +var ( + // DefaultUserGroups are the groups the default user will be part of. + DefaultUserGroups = []string{ + "sudo", "adm", "cdrom", "dialout", + "dip", "video", "plugdev", "netdev", + } ) const ( Windows OSType = "windows" Linux OSType = "linux" + Unknown OSType = "unknown" ) const ( @@ -57,10 +74,17 @@ func NewConfig(cfgFile string) (*Config, error) { if err := config.Validate(); err != nil { return nil, errors.Wrap(err, "validating config") } + if config.ConfigDir == "" { + config.ConfigDir = DefaultConfigDir + } return &config, nil } type Config struct { + // ConfigDir is the folder where the runner may save any aditional files + // or configurations it may need. Things like auto-generated SSH keys that + // may be used to access the runner instances. + ConfigDir string `toml:"config_dir" json:"config-dir"` APIServer APIServer `toml:"apiserver" json:"apiserver"` Database Database `toml:"database" json:"database"` Repositories []Repository `toml:"repository" json:"repository"` @@ -153,6 +177,8 @@ type LXD struct { TLSServerCert string `toml:"tls_server_certificate" json:"tls-server-certificate"` // TLSCA is the TLS CA certificate when running LXD in PKI mode. TLSCA string `toml:"tls_ca" json:"tls-ca"` + + // TODO: add simplestreams sources } func (l *LXD) Validate() error { @@ -308,7 +334,7 @@ type Repository struct { } func (r *Repository) String() string { - return fmt.Sprintf("%s/%s", r.Owner, r.Name) + return fmt.Sprintf("https://github.com/%s/%s", r.Owner, r.Name) } func (r *Repository) Validate() error { diff --git a/go.mod b/go.mod index aee9c3ca..d892f4cb 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,14 @@ go 1.18 require ( github.com/BurntSushi/toml v0.3.1 - github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v43 v43.0.0 github.com/lxc/lxd v0.0.0-20220415052741-1170f2806124 github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 + golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) require ( @@ -27,10 +29,10 @@ require ( github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/fastuuid v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 // indirect golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/errgo.v1 v1.0.1 // indirect gopkg.in/httprequest.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index d1e0e1e9..d29495f1 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ github.com/go-macaroon-bakery/macaroonpb v1.0.0/go.mod h1:UzrGOcbiwTXISFP2XDLDPj github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -44,8 +45,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v43 v43.0.0 h1:y+GL7LIsAIF2NZlJ46ZoC/D1W1ivZasT0lnWHMYPZ+U= github.com/google/go-github/v43 v43.0.0/go.mod h1:ZkTvvmCXBvsfPpTHXnH/d2hP9Y0cTbvN9kr5xqyXOIc= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -124,12 +123,14 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -149,6 +150,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -164,6 +166,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -209,5 +213,6 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/params/params.go b/params/params.go index 506c068a..c1203508 100644 --- a/params/params.go +++ b/params/params.go @@ -1,5 +1,7 @@ package params +import "github.com/google/go-github/v43/github" + type OSType string const ( @@ -28,3 +30,26 @@ type Instance struct { // for this instance. Addresses []string `json:"ip_addresses,omitempty"` } + +type BootstrapInstance struct { + Tools []*github.RunnerApplicationDownload `json:"tools"` + // RepoURL is the URL the github runner agent needs to configure itself. + RepoURL string `json:"repo_url"` + // GithubRunnerAccessToken is the token we fetch from github to allow the runner to + // register itself. + GithubRunnerAccessToken string `json:"github_runner_access_token"` + // RunnerType is the name of the defined runner type in a particular pool. The provider + // needs this to determine which flavor/image/settings it needs to use to create the + // instance. This is provider/runner specific. The config for the runner type is defined + // in the configuration file, as part of the pool definition. + RunnerType string `json:"runner_type"` + // CallbackUrl is the URL where the instance can send a post, signaling + // progress or status. + CallbackURL string `json:"callback_url"` + // InstanceToken is the token that needs to be set by the instance in the headers + // in order to send updated back to the runner-manager via CallbackURL. + InstanceToken string `json:"instance_token"` + // SSHKeys are the ssh public keys we may want to inject inside the runners, if the + // provider supports it. + SSHKeys []string `json:"ssh_keys"` +} diff --git a/runner/common/provider.go b/runner/common/provider.go index c2dc1a30..53eb5ded 100644 --- a/runner/common/provider.go +++ b/runner/common/provider.go @@ -2,13 +2,12 @@ package common import ( "context" - - "github.com/google/go-github/v43/github" + "runner-manager/params" ) type Provider interface { // CreateInstance creates a new compute instance in the provider. - CreateInstance(ctx context.Context, runnerType string, tools github.RunnerApplicationDownload) error + CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error // Delete instance will delete the instance in a provider. DeleteInstance(ctx context.Context, instance string) error // ListInstances will list all instances for a provider. diff --git a/runner/providers/lxd/cloud_config.go b/runner/providers/lxd/cloud_config.go deleted file mode 100644 index 1694bdcb..00000000 --- a/runner/providers/lxd/cloud_config.go +++ /dev/null @@ -1,27 +0,0 @@ -package lxd - -var cloudConfigTemplate = ` -#cloud-config -package_upgrade: true -packages: - - curl -ssh_authorized_keys: - {{ ssh_authorized_keys }} -system_info: - default_user: - name: runner - home: /home/runner - shell: /bin/bash - groups: [sudo, plugdev, dip, netdev] - sudo: ALL=(ALL) NOPASSWD:ALL - -runcmd: - - [ ls, -l, / ] - - [ sh, -xc, "echo $(date) ': hello world!'" ] - - [ sh, -c, echo "=========hello world=========" ] - - ls -l /root - # Note: Don't write files to /tmp from cloud-init use /run/somedir instead. - # Early boot environments can race systemd-tmpfiles-clean LP: #1707222. - - mkdir /run/mydir - - [ wget, "http://slashdot.org", -O, /run/mydir/index.html ] -` diff --git a/runner/providers/lxd/lxd.go b/runner/providers/lxd/lxd.go index 62f86be5..94dc8482 100644 --- a/runner/providers/lxd/lxd.go +++ b/runner/providers/lxd/lxd.go @@ -2,10 +2,14 @@ package lxd import ( "context" + "encoding/json" "fmt" + "strings" + "runner-manager/cloudconfig" "runner-manager/config" runnerErrors "runner-manager/errors" + "runner-manager/params" "runner-manager/runner/common" "runner-manager/util" @@ -18,6 +22,13 @@ import ( var _ common.Provider = &LXD{} +var ( + archMap map[string]string = map[string]string{ + "x86_64": "x64", + "amd64": "x64", + } +) + const ( DefaultProjectDescription = "This project was created automatically by runner-manager to be used for github ephemeral action runners." DefaultProjectName = "runner-manager-project" @@ -73,7 +84,7 @@ func NewProvider(ctx context.Context, cfg *config.Provider, pool *config.Pool) ( _, _, err = cli.GetProject(projectName(cfg.LXD)) if err != nil { - return nil, errors.Wrap(err, "fetching project name") + return nil, errors.Wrapf(err, "fetching project name: %s", projectName(cfg.LXD)) } cli = cli.UseProject(projectName(cfg.LXD)) @@ -99,7 +110,7 @@ type LXD struct { cli lxd.InstanceServer } -func (l *LXD) getProfiles(runnerType string) ([]string, error) { +func (l *LXD) getProfiles(runner config.Runner) ([]string, error) { ret := []string{} if l.cfg.LXD.IncludeDefaultProfile { ret = append(ret, "default") @@ -107,11 +118,6 @@ func (l *LXD) getProfiles(runnerType string) ([]string, error) { set := map[string]struct{}{} - runner, err := util.FindRunner(runnerType, l.pool.Runners) - if err != nil { - return nil, errors.Wrapf(err, "finding runner of type %s", runnerType) - } - profiles, err := l.cli.GetProfileNames() if err != nil { return nil, errors.Wrap(err, "fetching profile names") @@ -128,21 +134,132 @@ func (l *LXD) getProfiles(runnerType string) ([]string, error) { return ret, nil } -func (l *LXD) getCloudConfig(runnerType string) (string, error) { - return "", nil +// TODO: Add image details cache. Avoid doing a request if not necessary. +func (l *LXD) getImageDetails(runner config.Runner) (*api.Image, error) { + alias, _, err := l.cli.GetImageAlias(runner.Image) + if err != nil { + return nil, errors.Wrapf(err, "resolving alias: %s", runner.Image) + } + + image, _, err := l.cli.GetImage(alias.Target) + if err != nil { + return nil, errors.Wrap(err, "fetching image details") + } + return image, nil } -func (l *LXD) getCreateInstanceArgs(runnerType string) (api.InstancesPost, error) { +func (l *LXD) getCloudConfig(runner config.Runner, bootstrapParams params.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) { + cloudCfg := cloudconfig.NewDefaultCloudInitConfig() + + installRunnerParams := cloudconfig.InstallRunnerParams{ + FileName: *tools.Filename, + DownloadURL: *tools.DownloadURL, + GithubToken: bootstrapParams.GithubRunnerAccessToken, + RunnerUsername: config.DefaultUser, + RunnerGroup: config.DefaultUser, + RepoURL: bootstrapParams.RepoURL, + RunnerName: runnerName, + RunnerLabels: strings.Join(runner.Labels, ","), + } + + installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams) + if err != nil { + return "", errors.Wrap(err, "generating script") + } + + cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...) + cloudCfg.AddFile(installScript, "/var/run/install_runner.sh", "root:root", "755") + cloudCfg.AddRunCmd("/var/run/install_runner.sh") + + asStr, err := cloudCfg.Serialize() + if err != nil { + return "", errors.Wrap(err, "creating cloud config") + } + return asStr, nil +} + +func (l *LXD) getTools(image *api.Image, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) { + if image == nil { + return github.RunnerApplicationDownload{}, fmt.Errorf("nil image received") + } + osName, ok := image.ImagePut.Properties["os"] + if !ok { + return github.RunnerApplicationDownload{}, fmt.Errorf("missing OS info in image properties") + } + + osType, err := util.OSToOSType(osName) + if err != nil { + return github.RunnerApplicationDownload{}, errors.Wrap(err, "fetching OS type") + } + + switch osType { + case config.Linux: + default: + return github.RunnerApplicationDownload{}, fmt.Errorf("this provider does not support OS type: %s", osType) + } + + for _, tool := range tools { + if tool == nil { + continue + } + if tool.OS == nil || tool.Architecture == nil { + continue + } + + fmt.Println(*tool.Architecture, *tool.OS) + fmt.Printf("image arch: %s --> osType: %s\n", image.Architecture, string(osType)) + if *tool.Architecture == image.Architecture && *tool.OS == string(osType) { + return *tool, nil + } + + arch, ok := archMap[image.Architecture] + if ok && arch == *tool.Architecture { + return *tool, nil + } + } + return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, image.Architecture) +} + +func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance) (api.InstancesPost, error) { name := fmt.Sprintf("runner-manager-%s", uuid.New()) - profiles, err := l.getProfiles(runnerType) + runner, err := util.FindRunnerType(bootstrapParams.RunnerType, l.pool.Runners) + + if err != nil { + return api.InstancesPost{}, errors.Wrap(err, "fetching runner") + } + + profiles, err := l.getProfiles(runner) if err != nil { return api.InstancesPost{}, errors.Wrap(err, "fetching profiles") } + image, err := l.getImageDetails(runner) + if err != nil { + return api.InstancesPost{}, errors.Wrap(err, "getting image details") + } + + tools, err := l.getTools(image, bootstrapParams.Tools) + if err != nil { + return api.InstancesPost{}, errors.Wrap(err, "getting tools") + } + + cloudCfg, err := l.getCloudConfig(runner, bootstrapParams, tools, name) + if err != nil { + return api.InstancesPost{}, errors.Wrap(err, "generating cloud-config") + } + args := api.InstancesPost{ InstancePut: api.InstancePut{ - Profiles: profiles, - Description: "Github runner provisioned by runner-manager", + Architecture: image.Architecture, + Profiles: profiles, + Description: "Github runner provisioned by runner-manager", + Config: map[string]string{ + "user.user-data": cloudCfg, + }, + }, + Source: api.InstanceSource{ + Type: "image", + Alias: runner.Image, }, Name: name, Type: api.InstanceTypeVM, @@ -151,7 +268,14 @@ func (l *LXD) getCreateInstanceArgs(runnerType string) (api.InstancesPost, error } // CreateInstance creates a new compute instance in the provider. -func (l *LXD) CreateInstance(ctx context.Context, runnerType string, tools github.RunnerApplicationDownload) error { +func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) error { + args, err := l.getCreateInstanceArgs(bootstrapParams) + if err != nil { + return errors.Wrap(err, "fetching create args") + } + + asJs, err := json.MarshalIndent(args, "", " ") + fmt.Println(string(asJs), err) return nil } diff --git a/runner/runner.go b/runner/runner.go index 0f8c85fe..3ef49333 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2,23 +2,112 @@ package runner import ( "context" + "io/ioutil" + "os" + "path/filepath" "runner-manager/config" "runner-manager/runner/common" + "runner-manager/util" + "sync" - "github.com/google/go-github/github" + "github.com/google/go-github/v43/github" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" ) func NewRunner(ctx context.Context, cfg *config.Config) (*Runner, error) { - return &Runner{ + runner := &Runner{ ctx: ctx, config: cfg, - }, nil + } + + if err := runner.ensureSSHKeys(); err != nil { + return nil, errors.Wrap(err, "ensuring SSH keys") + } + + return runner, nil } type Runner struct { + mux sync.Mutex + ctx context.Context ghc *github.Client config *config.Config pools []common.PoolManager } + +func (r *Runner) sshDir() string { + return filepath.Join(r.config.ConfigDir, "ssh") +} + +func (r *Runner) sshKeyPath() string { + keyPath := filepath.Join(r.sshDir(), "runner_rsa_key") + return keyPath +} + +func (r *Runner) sshPubKeyPath() string { + keyPath := filepath.Join(r.sshDir(), "runner_rsa_key.pub") + return keyPath +} + +func (r *Runner) parseSSHKey() (ssh.Signer, error) { + r.mux.Lock() + defer r.mux.Unlock() + + key, err := ioutil.ReadFile(r.sshKeyPath()) + if err != nil { + return nil, errors.Wrapf(err, "reading private key %s", r.sshKeyPath()) + } + + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, errors.Wrapf(err, "parsing private key %s", r.sshKeyPath()) + } + + return signer, nil +} + +func (r *Runner) sshPubKey() ([]byte, error) { + key, err := ioutil.ReadFile(r.sshPubKeyPath()) + if err != nil { + return nil, errors.Wrapf(err, "reading public key %s", r.sshPubKeyPath()) + } + return key, nil +} + +func (r *Runner) ensureSSHKeys() error { + sshDir := r.sshDir() + + if _, err := os.Stat(sshDir); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return errors.Wrapf(err, "checking SSH dir %s", sshDir) + } + if err := os.MkdirAll(sshDir, 0o700); err != nil { + return errors.Wrapf(err, "creating ssh dir %s", sshDir) + } + } + + privKeyFile := r.sshKeyPath() + pubKeyFile := r.sshPubKeyPath() + + if _, err := os.Stat(privKeyFile); err == nil { + return nil + } + + pubKey, privKey, err := util.GenerateSSHKeyPair() + if err != nil { + errors.Wrap(err, "generating keypair") + } + + if err := ioutil.WriteFile(privKeyFile, privKey, 0o600); err != nil { + return errors.Wrap(err, "writing private key") + } + + if err := ioutil.WriteFile(pubKeyFile, pubKey, 0o600); err != nil { + return errors.Wrap(err, "writing public key") + } + + return nil +} diff --git a/testdata/config.toml b/testdata/config.toml index 2bc27ede..0994c10d 100644 --- a/testdata/config.toml +++ b/testdata/config.toml @@ -1,4 +1,4 @@ -log_file = "/tmp/runner-manager.log" +# log_file = "/tmp/runner-manager.log" [apiserver] bind = "0.0.0.0" diff --git a/util/util.go b/util/util.go index e61181c7..ec05d203 100644 --- a/util/util.go +++ b/util/util.go @@ -1,15 +1,37 @@ package util import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/pem" "fmt" "io" + "io/ioutil" "os" "path" + "strings" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" lumberjack "gopkg.in/natefinch/lumberjack.v2" "runner-manager/config" - "runner-manager/errors" + runnerErrors "runner-manager/errors" +) + +var ( + OSToOSTypeMap map[string]config.OSType = map[string]config.OSType{ + "ubuntu": config.Linux, + "rhel": config.Linux, + "centos": config.Linux, + "suse": config.Linux, + "fedora": config.Linux, + "flatcar": config.Linux, + "windows": config.Windows, + } ) // GetLoggingWriter returns a new io.Writer suitable for logging. @@ -36,12 +58,54 @@ func GetLoggingWriter(cfg *config.Config) (io.Writer, error) { return writer, nil } -func FindRunner(runnerType string, runners []config.Runner) (config.Runner, error) { +func FindRunnerType(runnerType string, runners []config.Runner) (config.Runner, error) { for _, runner := range runners { if runner.Name == runnerType { return runner, nil } } - return config.Runner{}, errors.ErrNotFound + return config.Runner{}, runnerErrors.ErrNotFound +} + +func ConvertFileToBase64(file string) (string, error) { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return "", errors.Wrap(err, "reading file") + } + + return base64.StdEncoding.EncodeToString(bytes), nil +} + +// GenerateSSHKeyPair generates a private/public key-pair. +// Shamlessly copied from: https://stackoverflow.com/questions/21151714/go-generate-an-ssh-public-key +func GenerateSSHKeyPair() (pubKey, privKey []byte, err error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + // generate and write private key as PEM + var privKeyBuf bytes.Buffer + + privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} + if err := pem.Encode(&privKeyBuf, privateKeyPEM); err != nil { + return nil, nil, err + } + + // generate and write public key + pub, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return nil, nil, err + } + + return ssh.MarshalAuthorizedKey(pub), privKeyBuf.Bytes(), nil +} + +func OSToOSType(os string) (config.OSType, error) { + osType, ok := OSToOSTypeMap[strings.ToLower(os)] + if !ok { + return config.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os) + } + return osType, nil }