DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Update deps

Browse source 
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2023-07-23 15:48:46 +00:00
parent e33b64aacb
commit 7e4dcea056
8 changed files with 235 additions and 109 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -4,7 +4,7 @@ go 1.20
require (
github.com/BurntSushi/toml v1.2.1
github.com/cloudbase/garm-provider-common v0.0.0-20230723124240-9ad9282807e5
github.com/cloudbase/garm-provider-common v0.0.0-20230723154741-c0a12146e69e
github.com/go-openapi/errors v0.20.4
github.com/go-openapi/runtime v0.26.0
github.com/go-openapi/strfmt v0.21.7

12
go.sum
View file

@ -25,10 +25,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudbase/garm-provider-common v0.0.0-20230722221640-5f53e8404727 h1:/xosrBTRvakPPFBEzfZGruD0TXI5zy0q1ACu/jplbZk=
github.com/cloudbase/garm-provider-common v0.0.0-20230722221640-5f53e8404727/go.mod h1:RKzgL0MXkNeGfloQpE2swz/y4LWJr5+2Wd45bSXPB0k=
github.com/cloudbase/garm-provider-common v0.0.0-20230723124240-9ad9282807e5 h1:x0Z4i2ep2xos2I/qBkk/b9RNebufpY+l4GwJQaIAQyA=
github.com/cloudbase/garm-provider-common v0.0.0-20230723124240-9ad9282807e5/go.mod h1:RKzgL0MXkNeGfloQpE2swz/y4LWJr5+2Wd45bSXPB0k=
github.com/cloudbase/garm-provider-common v0.0.0-20230723154741-c0a12146e69e h1:a0vWKKqUKEGs4jiXw/RSZ72AkbyBwPscU49IeTMMSQY=
github.com/cloudbase/garm-provider-common v0.0.0-20230723154741-c0a12146e69e/go.mod h1:RKzgL0MXkNeGfloQpE2swz/y4LWJr5+2Wd45bSXPB0k=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
@ -50,7 +48,6 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D
github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@ -191,8 +188,6 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/juju/clock v1.0.3 h1:yJHIsWXeU8j3QcBdiess09SzfiXRRrsjKPn2whnMeds=
github.com/juju/clock v1.0.3/go.mod h1:HIBvJ8kiV/n7UHwKuCkdYL4l/MDECztHR2sAvWDxxf0=
github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM=
@ -253,10 +248,7 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

4
runner/providers/lxd/lxd.go
View file

@ -22,7 +22,6 @@ import (
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
@ -32,6 +31,7 @@ import (
"github.com/lxc/lxd/shared/api"
"github.com/pkg/errors"
"github.com/cloudbase/garm-provider-common/cloudconfig"
commonParams "github.com/cloudbase/garm-provider-common/params"
)
@ -239,7 +239,7 @@ func (l *LXD) getCreateInstanceArgs(bootstrapParams commonParams.BootstrapInstan
bootstrapParams.UserDataOptions.DisableUpdatesOnBoot = specs.DisableUpdates
bootstrapParams.UserDataOptions.ExtraPackages = specs.ExtraPackages
cloudCfg, err := util.GetCloudConfig(bootstrapParams, tools, bootstrapParams.Name)
cloudCfg, err := cloudconfig.GetCloudConfig(bootstrapParams, tools, bootstrapParams.Name)
if err != nil {
return api.InstancesPost{}, errors.Wrap(err, "generating cloud-config")
}

72
vendor/github.com/cloudbase/garm-provider-common/cloudconfig/templates.go generated vendored
View file

@ -28,6 +28,10 @@ var CloudConfigTemplate = `#!/bin/bash
set -e
set -o pipefail
{{- if .EnableBootDebug }}
set -x
{{- end }}
CALLBACK_URL="{{ .CallbackURL }}"
METADATA_URL="{{ .MetadataURL }}"
BEARER_TOKEN="{{ .CallbackToken }}"
@ -410,31 +414,59 @@ function Install-Runner() {
Install-Runner
`
// InstallRunnerParams holds the parameters needed to render the runner install script.
type InstallRunnerParams struct {
FileName string
DownloadURL string
RunnerUsername string
RunnerGroup string
RepoURL string
MetadataURL string
RunnerName string
RunnerLabels string
CallbackURL string
CallbackToken string
// FileName is the name of the file that will be downloaded from the download URL.
// This will be the runner archive downloaded from GitHub.
FileName string
// DownloadURL is the URL from which the runner archive will be downloaded.
DownloadURL string
// RunnerUsername is the username of the user that will run the runner service.
RunnerUsername string
// RunnerGroup is the group of the user that will run the runner service.
RunnerGroup string
// RepoURL is the URL or the github repo the github runner agent needs to configure itself.
RepoURL string
// MetadataURL is the URL where instances can fetch information needed to set themselves up.
// This URL is set in the GARM config file.
MetadataURL string
// RunnerName is the name of the runner. GARM will use this to register the runner with GitHub.
RunnerName string
// RunnerLabels is a comma separated list of labels that will be added to the runner.
RunnerLabels string
// CallbackURL is the URL where the instance can send a post, signaling progress or status.
// This URL is set in the GARM config file.
CallbackURL string
// CallbackToken is the token that needs to be set by the instance in the headers in order to call
// the CallbackURL.
CallbackToken string
// TempDownloadToken is the token that needs to be set by the instance in the headers in order to download
// the githun runner. This is usually needed when using garm against a GHES instance.
TempDownloadToken string
CABundle string
// CABundle is a CA certificate bundle which will be sent to instances and which will tipically be installed
// as a system wide trusted root CA by either cloud-init or whatever mechanism the provider will use to set
// up the runner.
CABundle string
// GitHubRunnerGroup is the github runner group in which the newly installed runner should be added to.
GitHubRunnerGroup string
// EnableBootDebug will enable bash debug mode.
EnableBootDebug bool
// ExtraContext is a map of extra context that will be passed to the runner install template.
// This option is useful for situations in which you're supplying your own template and you need
// to pass in information that is not available in the default template.
ExtraContext map[string]string
}
func InstallRunnerScript(installParams InstallRunnerParams, osType params.OSType) ([]byte, error) {
var tpl string
switch osType {
case params.Linux:
tpl = CloudConfigTemplate
case params.Windows:
tpl = WindowsSetupScriptTemplate
default:
return nil, fmt.Errorf("unsupported os type: %s", osType)
func InstallRunnerScript(installParams InstallRunnerParams, osType params.OSType, tpl string) ([]byte, error) {
if tpl == "" {
switch osType {
case params.Linux:
tpl = CloudConfigTemplate
case params.Windows:
tpl = WindowsSetupScriptTemplate
default:
return nil, fmt.Errorf("unsupported os type: %s", osType)
}
}
t, err := template.New("").Parse(tpl)

176
vendor/github.com/cloudbase/garm-provider-common/cloudconfig/util.go generated vendored Normal file
View file

@ -0,0 +1,176 @@
package cloudconfig
import (
"encoding/json"
"fmt"
"strings"
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/pkg/errors"
)
type CloudConfigSpec struct {
// RunnerInstallTemplate can be used to override the default runner install template.
// If used, the caller is responsible for the correctness of the template as well as the
// suitability of the template for the target OS.
RunnerInstallTemplate []byte `json:"runner_install_template"`
// PreInstallScripts is a map of pre-install scripts that will be run before the
// runner install script. These will run as root and can be used to prep a generic image
// before we attempt to install the runner. The key of the map is the name of the script
// as it will be written to disk. The value is a byte array with the contents of the script.
// On Linux, we will set the executable flag. On Windows, the name matters as Windows looks for an
// extension to determine if the file is an executable or not. In theory this can hold binaries,
// but in most cases this will most likely hold scripts. We do not currenly validate the payload,
// so it's up to the user what they upload here.
// Caution needs to be exercised when using this feature, as the total size of userdata is limited
// on most providers.
PreInstallScripts map[string][]byte `json:"pre_install_scripts"`
// ExtraContext is a map of extra context that will be passed to the runner install template.
ExtraContext map[string]string `json:"extra_context"`
}
// GetSpecs returns the cloud config specific extra specs from the bootstrap params.
func GetSpecs(bootstrapParams commonParams.BootstrapInstance) (CloudConfigSpec, error) {
var extraSpecs CloudConfigSpec
if err := json.Unmarshal(bootstrapParams.ExtraSpecs, &extraSpecs); err != nil {
return CloudConfigSpec{}, errors.Wrap(err, "unmarshaling extra specs")
}
if extraSpecs.ExtraContext == nil {
extraSpecs.ExtraContext = map[string]string{}
}
if extraSpecs.PreInstallScripts == nil {
extraSpecs.PreInstallScripts = map[string][]byte{}
}
return extraSpecs, nil
}
// GetRunnerInstallScript returns the runner install script for the given bootstrap params.
// This function will return either the default script for the given OS type or will use the supplied template
// if one is provided.
func GetRunnerInstallScript(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) ([]byte, error) {
if tools.Filename == nil {
return nil, fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return nil, fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
extraSpecs, err := GetSpecs(bootstrapParams)
if err != nil {
return nil, errors.Wrap(err, "getting specs")
}
installRunnerParams := InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: defaults.DefaultUser,
RunnerGroup: defaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
ExtraContext: extraSpecs.ExtraContext,
EnableBootDebug: bootstrapParams.UserDataOptions.EnableBootDebug,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := InstallRunnerScript(installRunnerParams, bootstrapParams.OSType, string(extraSpecs.RunnerInstallTemplate))
if err != nil {
return nil, errors.Wrap(err, "generating script")
}
return installScript, nil
}
// GetCloudInitConfig returns the cloud-init specific userdata config. This config can be used on most clouds
// for most Linux machines. The install runner script must be generated separately either by GetRunnerInstallScript()
// or some other means.
func GetCloudInitConfig(bootstrapParams commonParams.BootstrapInstance, installScript []byte) (string, error) {
extraSpecs, err := GetSpecs(bootstrapParams)
if err != nil {
return "", errors.Wrap(err, "getting specs")
}
cloudCfg := NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
if len(extraSpecs.PreInstallScripts) > 0 {
for name, script := range extraSpecs.PreInstallScripts {
cloudCfg.AddFile(script, fmt.Sprintf("/garm-pre-install/%s", name), "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("/garm-pre-install/%s", name))
}
}
cloudCfg.AddRunCmd("rm -rf /garm-pre-install")
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
asStr, err := cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
return asStr, nil
}
// GetCloudConfig is a helper function that generates a cloud-init config for Linux and a powershell script for Windows.
// In most cases this function should do, but in situations where a more custom approach is needed, you may need to call
// GetCloudInitConfig() or GetRunnerInstallScript() directly and compose the final userdata in a different way.
// The extra specs PreInstallScripts is only supported on Linux via cloud-init by this function. On some providers, like Azure
// Windows initialization scripts are run by creating a separate CustomScriptExtension resource for each individual script.
// On other clouds it may be different. This function aims to be generic, which is why it only supports the PreInstallScripts
// via cloud-init.
func GetCloudConfig(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
installScript, err := GetRunnerInstallScript(bootstrapParams, tools, runnerName)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case commonParams.Linux:
cloudCfg, err := GetCloudInitConfig(bootstrapParams, installScript)
if err != nil {
return "", errors.Wrap(err, "getting cloud init config")
}
return cloudCfg, nil
case commonParams.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}

1
vendor/github.com/cloudbase/garm-provider-common/params/params.go generated vendored
View file

@ -45,6 +45,7 @@ const (
type UserDataOptions struct {
DisableUpdatesOnBoot bool `json:"disable_updates_on_boot"`
ExtraPackages []string `json:"extra_packages"`
EnableBootDebug bool `json:"enable_boot_debug"`
}
type BootstrapInstance struct {

75
vendor/github.com/cloudbase/garm-provider-common/util/util.go generated vendored
View file

@ -33,10 +33,8 @@ import (
"unicode"
"unicode/utf16"
"github.com/cloudbase/garm-provider-common/cloudconfig"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
@ -190,79 +188,6 @@ func OSToOSType(os string) (commonParams.OSType, error) {
return osType, nil
}
func GetCloudConfig(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
if tools.Filename == nil {
return "", fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return "", fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
installRunnerParams := cloudconfig.InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: defaults.DefaultUser,
RunnerGroup: defaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case commonParams.Linux:
cloudCfg := cloudconfig.NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
var err error
asStr, err = cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
case commonParams.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}
func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {

2
vendor/modules.txt vendored
View file

@ -34,7 +34,7 @@ github.com/cespare/xxhash/v2
# github.com/chzyer/readline v1.5.1
## explicit; go 1.15
github.com/chzyer/readline
# github.com/cloudbase/garm-provider-common v0.0.0-20230723124240-9ad9282807e5
# github.com/cloudbase/garm-provider-common v0.0.0-20230723154741-c0a12146e69e
## explicit; go 1.20
github.com/cloudbase/garm-provider-common/cloudconfig
github.com/cloudbase/garm-provider-common/defaults