diff --git a/cloudconfig/templates.go b/cloudconfig/templates.go index e83deffa..fbea1b20 100644 --- a/cloudconfig/templates.go +++ b/cloudconfig/templates.go @@ -56,7 +56,7 @@ curl -L -o "/home/runner/{{ .FileName }}" "{{ .DownloadURL }}" || fail "failed t mkdir -p /home/runner/actions-runner || fail "failed to create actions-runner folder" sendStatus "extracting runner" -tar xf "/home/runner/{{ .FileName }}" -C /home/runner/actions-runner/ || fail "failed to extract runner" +tar xf "/home/{{ .RunnerUsername }}/{{ .FileName }}" -C /home/{{ .RunnerUsername }}/actions-runner/ || fail "failed to extract runner" chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R /home/{{ .RunnerUsername }}/actions-runner/ || fail "failed to change owner" sendStatus "installing dependencies" diff --git a/config/config.go b/config/config.go index 6c3a9bba..15c96f2f 100644 --- a/config/config.go +++ b/config/config.go @@ -46,6 +46,8 @@ const ( // LXDProvider represents the LXD provider. LXDProvider ProviderType = "lxd" + // ExternalProvider represents an external provider. + ExternalProvider ProviderType = "external" // DefaultConfigFilePath is the default path on disk to the garm // configuration file. @@ -182,6 +184,7 @@ type Provider struct { ProviderType ProviderType `toml:"provider_type" json:"provider-type"` Description string `toml:"description" json:"description"` LXD LXD `toml:"lxd" json:"lxd"` + External External `toml:"external" json:"external"` } func (p *Provider) Validate() error { @@ -194,6 +197,10 @@ func (p *Provider) Validate() error { if err := p.LXD.Validate(); err != nil { return errors.Wrap(err, "validating LXD provider info") } + case ExternalProvider: + if err := p.External.Validate(); err != nil { + return errors.Wrap(err, "validating external provider info") + } default: return fmt.Errorf("unknown provider type: %s", p.ProviderType) } diff --git a/config/external.go b/config/external.go new file mode 100644 index 00000000..b4bfc53a --- /dev/null +++ b/config/external.go @@ -0,0 +1,67 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + + "garm/util/exec" + + "github.com/pkg/errors" +) + +// External represents the config for an external provider. +// The external provider is a provider that delegates all operations +// to an external binary. This way, you can write your own logic in +// whatever programming language you wish, while still remaining compatible +// with garm. +type External struct { + // ConfigFile is the path on disk to a file which will be passed to + // the external binary as an environment variable: GARM_PROVIDER_CONFIG + // You can use this file for any configuration you need to do for the + // cloud your calling into, to create the compute resources. + ConfigFile string `toml:"config_file" json:"config-file"` + // ProviderDir is the path on disk to a folder containing an executable + // called "garm-external-provider". + ProviderDir string `toml:"provider_dir" json:"provider-dir"` +} + +func (e *External) ExecutablePath() (string, error) { + execPath := filepath.Join(e.ProviderDir, "garm-external-provider") + if !filepath.IsAbs(execPath) { + return "", fmt.Errorf("executable path must be an absolut epath") + } + return filepath.Join(e.ProviderDir, "garm-external-provider"), nil +} + +func (e *External) Validate() error { + if e.ConfigFile != "" { + if _, err := os.Stat(e.ConfigFile); err != nil { + return fmt.Errorf("failed to access cofig file %s", e.ConfigFile) + } + if !filepath.IsAbs(e.ConfigFile) { + return fmt.Errorf("path to config file must be an absolute path") + } + } + + if e.ProviderDir == "" { + return fmt.Errorf("missing provider dir") + } + + if !filepath.IsAbs(e.ProviderDir) { + return fmt.Errorf("path to provider dir must be absolute") + } + + execPath, err := e.ExecutablePath() + if err != nil { + return errors.Wrap(err, "fetching executable path") + } + if _, err := os.Stat(execPath); err != nil { + return errors.Wrap(err, "checking provider executable") + } + if !exec.IsExecutable(execPath) { + return fmt.Errorf("external provider binary %s is not executable", execPath) + } + + return nil +} diff --git a/contrib/providers.d/openstack/README.md b/contrib/providers.d/openstack/README.md new file mode 100644 index 00000000..6aa88f35 --- /dev/null +++ b/contrib/providers.d/openstack/README.md @@ -0,0 +1,5 @@ +# OpenStack external provider for GARM + +This is an example external provider, written for OpenStack. It is a simple bash script that implements the external provider interface, in order to supply ```garm``` with compute instances. This is just an example, complete with a sample config file. + +Not all functions are implemented, just the bare minimum to get it to work with the current feature set of ```garm```. It is not meant for production, as it needs a lot more error checking, retries, and potentially more flexibility to be of any use in a real environment. \ No newline at end of file diff --git a/contrib/providers.d/openstack/cloudconfig/install_runner.tpl b/contrib/providers.d/openstack/cloudconfig/install_runner.tpl new file mode 100644 index 00000000..0453fe8c --- /dev/null +++ b/contrib/providers.d/openstack/cloudconfig/install_runner.tpl @@ -0,0 +1,61 @@ +#!/bin/bash + +set -ex +set -o pipefail + +CALLBACK_URL="GARM_CALLBACK_URL" +BEARER_TOKEN="GARM_CALLBACK_TOKEN" +DOWNLOAD_URL="GH_DOWNLOAD_URL" +FILENAME="GH_FILENAME" +TARGET_URL="GH_TARGET_URL" +RUNNER_TOKEN="GH_RUNNER_TOKEN" +RUNNER_NAME="GH_RUNNER_NAME" +RUNNER_LABELS="GH_RUNNER_LABELS" + +function call() { + PAYLOAD="$1" + curl -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)" +} + +function sendStatus() { + MSG="$1" + call "{\"status\": \"installing\", \"message\": \"$MSG\"}" +} + +function success() { + MSG="$1" + call "{\"status\": \"idle\", \"message\": \"$MSG\"}" +} + +function fail() { + MSG="$1" + call "{\"status\": \"failed\", \"message\": \"$MSG\"}" + exit 1 +} + + + +sendStatus "downloading tools from ${DOWNLOAD_URL}" +curl -L -o "/home/runner/${FILENAME}" "${DOWNLOAD_URL}" || fail "failed to download tools" + +mkdir -p /home/runner/actions-runner || fail "failed to create actions-runner folder" + +sendStatus "extracting runner" +tar xf "/home/runner/${FILENAME}" -C /home/runner/actions-runner/ || fail "failed to extract runner" +chown runner:runner -R /home/runner/actions-runner/ || fail "failed to change owner" + +sendStatus "installing dependencies" +cd /home/runner/actions-runner +sudo ./bin/installdependencies.sh || fail "failed to install dependencies" + +sendStatus "configuring runner" +sudo -u runner -- ./config.sh --unattended --url "${TARGET_URL}" --token "${RUNNER_TOKEN}" --name "${RUNNER_NAME}" --labels "${RUNNER_LABELS}" --ephemeral || fail "failed to configure runner" + +sendStatus "installing runner service" +./svc.sh install runner || fail "failed to install service" + +sendStatus "starting service" +./svc.sh start || fail "failed to start service" + +success "runner successfully installed" + diff --git a/contrib/providers.d/openstack/cloudconfig/userdata.tpl b/contrib/providers.d/openstack/cloudconfig/userdata.tpl new file mode 100644 index 00000000..cb9fe6ad --- /dev/null +++ b/contrib/providers.d/openstack/cloudconfig/userdata.tpl @@ -0,0 +1,29 @@ +#cloud-config +package_upgrade: true +packages: + - curl + - tar +system_info: + default_user: + name: runner + home: /home/runner + shell: /bin/bash + groups: + - sudo + - adm + - cdrom + - dialout + - dip + - video + - plugdev + - netdev + sudo: ALL=(ALL) NOPASSWD:ALL +runcmd: + - /install_runner.sh + - rm -f /install_runner.sh +write_files: + - encoding: b64 + content: RUNNER_INSTALL_B64 + owner: root:root + path: /install_runner.sh + permissions: "755" diff --git a/contrib/providers.d/openstack/garm-external-provider b/contrib/providers.d/openstack/garm-external-provider new file mode 100755 index 00000000..3552d317 --- /dev/null +++ b/contrib/providers.d/openstack/garm-external-provider @@ -0,0 +1,385 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ ! -t 0 ] +then + INPUT=$(cat -) +fi +MYPATH=$(realpath ${BASH_SOURCE[0]}) +MYDIR=$(dirname "${MYPATH}") +TEMPLATES="$MYDIR/cloudconfig" + +# Defaults +# set this variable to 0 in the provider config to disable. +BOOT_FROM_VOLUME=${BOOT_FROM_VOLUME:-1} + +# END Defaults + +if [ -z "$GARM_PROVIDER_CONFIG_FILE" ] +then + echo "no config file specified in env" + exit 1 +fi + +source "$GARM_PROVIDER_CONFIG_FILE" + +declare -A OS_TO_GH_ARCH_MAP +OS_TO_GH_ARCH_MAP["x86_64"]="x64" +OS_TO_GH_ARCH_MAP["armv7l"]="arm64" +OS_TO_GH_ARCH_MAP["mips64"]="arm64" +OS_TO_GH_ARCH_MAP["mips64el"]="arm64" +OS_TO_GH_ARCH_MAP["mips"]="arm" +OS_TO_GH_ARCH_MAP["mipsel"]="arm" + +declare -A OS_TO_GARM_ARCH_MAP +OS_TO_GARM_ARCH_MAP["x86_64"]="amd64" +OS_TO_GARM_ARCH_MAP["armv7l"]="arm64" +OS_TO_GARM_ARCH_MAP["mips64"]="arm64" +OS_TO_GARM_ARCH_MAP["mips64el"]="arm64" +OS_TO_GARM_ARCH_MAP["mips"]="arm" +OS_TO_GARM_ARCH_MAP["mipsel"]="arm" + +declare -A GARM_TO_GH_ARCH_MAP +GARM_TO_GH_ARCH_MAP["amd64"]="x64" +GARM_TO_GH_ARCH_MAP["arm"]="arm" +GARM_TO_GH_ARCH_MAP["arm64"]="arm64" + +function checkValNotNull() { + if [ -z "$1" -o "$1" == "null" ];then + echo "failed to fetch value $2" + return 1 + fi + return 0 +} + +function getOSImageDetails() { + IMAGE_ID=$(echo "$INPUT" | jq -r -c '.image') + OS_IMAGE=$(openstack image show "$IMAGE_ID" -f json) + echo "$OS_IMAGE" +} + +function getOpenStackNetworkID() { + if [ -z "$OPENSTACK_PRIVATE_NETWORK" ] + then + echo "no network specified in config" + return 1 + fi + + NET_ID=$(openstack network show ${OPENSTACK_PRIVATE_NETWORK} -f value -c id) + if [ -z "$NET_ID" ];then + echo "failed to find network $OPENSTACK_PRIVATE_NETWORK" + fi + echo ${NET_ID} +} + +function getVolumeSizeFromFlavor() { + local flavor="$1" + + FLAVOR_DETAILS=$(openstack flavor show "${flavor}" -f json) + DISK_SIZE=$(echo "$FLAVOR_DETAILS" | jq -c -r '.disk') + if [ -z "$DISK_SIZE" ];then + echo "failed to get disk size from flavor" + return 1 + fi + + echo ${DISK_SIZE} +} + +function waitForVolume() { + local volumeName=$1 + set +e + status=$(openstack volume show "${volumeName}" -f json | jq -r -c '.status') + if [ $? -ne 0 ];then + set -e + return $? + fi + set -e + while [ "${status}" != "available" -a "${status}" != "error" ];do + status=$(openstack volume show "${volumeName}" -f json | jq -r -c '.status') + done +} + +function createVolumeFromImage() { + local image="$1" + local disk_size="$2" + local instance_name="$3" + if [ -z ${image} -o -z ${disk_size} -o -z "${instance_name}" ];then + echo "missing image, disk size or instance name in function call" + return 1 + fi + # Instance names contain a UUID. It should be safe to create a volume with the same name and + # expect it to be unique. + set +e + VOLUME_INFO=$(openstack volume create -f json --image "${image}" --size "${disk_size}" "${instance_name}") + if [ $? -ne 0 ]; then + CODE=$? + openstack volume delete "${instance_name}" || true + set -e + return $CODE + fi + waitForVolume "${instance_name}" + echo "${VOLUME_INFO}" +} + +function requestedArch() { + ARCH=$(echo "$INPUT" | jq -c -r '.arch') + checkValNotNull "${ARCH}" "arch" || return $? + echo "${ARCH}" +} + +function downloadURL() { + [ -z "$1" -o -z "$2" ] && return 1 + GH_ARCH="${GARM_TO_GH_ARCH_MAP[$2]}" + URL=$(echo "$INPUT" | jq -c -r --arg OS "$1" --arg ARCH "$GH_ARCH" '(.tools[] | select( .os == $OS and .architecture == $ARCH)).download_url') + checkValNotNull "${URL}" "download URL" || return $? + echo "${URL}" +} + +function downloadFilename() { + [ -z "$1" -o -z "$2" ] && return 1 + GH_ARCH="${GARM_TO_GH_ARCH_MAP[$2]}" + FN=$(echo "$INPUT" | jq -c -r --arg OS "$1" --arg ARCH "$GH_ARCH" '(.tools[] | select( .os == $OS and .architecture == $ARCH)).filename') + checkValNotNull "${FN}" "download filename" || return $? + echo "${FN}" +} + +function poolID() { + POOL_ID=$(echo "$INPUT" | jq -c -r '.pool_id') + checkValNotNull "${POOL_ID}" "pool_id" || return $? + echo "${POOL_ID}" +} + +function flavor() { + FLAVOR=$(echo "$INPUT" | jq -c -r '.flavor') + checkValNotNull "${FLAVOR}" "flavor" || return $? + echo "${FLAVOR}" +} + +function image() { + IMG=$(echo "$INPUT" | jq -c -r '.image') + checkValNotNull "${IMG}" "image" || return $? + echo "${IMG}" +} + +function repoURL() { + REPO=$(echo "$INPUT" | jq -c -r '.repo_url') + checkValNotNull "${REPO}" "repo_url" || return $? + echo "${REPO}" +} + +function ghAccessToken() { + TOKEN=$(echo "$INPUT" | jq -c -r '.github_runner_access_token') + checkValNotNull "${TOKEN}" "github_runner_access_token" || return $? + echo "${TOKEN}" +} + +function callbackURL() { + CB_URL=$(echo "$INPUT" | jq -c -r '."callback-url"') + checkValNotNull "${CB_URL}" "callback-url" || return $? + echo "${CB_URL}" +} + +function callbackToken() { + CB_TK=$(echo "$INPUT" | jq -c -r '."instance-token"') + checkValNotNull "${CB_TK}" "instance-token" || return $? + echo "${CB_TK}" +} + +function instanceName() { + NAME=$(echo "$INPUT" | jq -c -r '.name') + checkValNotNull "${NAME}" "name" || return $? + echo "${NAME}" +} + +function labels() { + LBL=$(echo "$INPUT" | jq -c -r '.labels | join(",")') + checkValNotNull "${LBL}" "labels" || return $? + echo "${LBL}" +} + +function getCloudConfig() { + IMAGE_DETAILS=$(getOSImageDetails) + + OS_TYPE=$(echo "${IMAGE_DETAILS}" | jq -c -r '.properties.os_type') + checkValNotNull "${OS_TYPE}" "os_type" || return $? + + ARCH=$(requestedArch) + DW_URL=$(downloadURL "${OS_TYPE}" "${ARCH}") + DW_FILENAME=$(downloadFilename "${OS_TYPE}" "${ARCH}") + LABELS=$(labels) + + TMP_SCRIPT=$(mktemp) + TMP_CC=$(mktemp) + + INSTALL_TPL=$(cat "${TEMPLATES}/install_runner.tpl") + CC_TPL=$(cat "${TEMPLATES}/userdata.tpl") + echo "$INSTALL_TPL" | sed -e "s|GARM_CALLBACK_URL|$(callbackURL)|g" \ + -e "s|GARM_CALLBACK_TOKEN|$(callbackToken)|g" \ + -e "s|GH_DOWNLOAD_URL|${DW_URL}|g" \ + -e "s|GH_FILENAME|${DW_FILENAME}|g" \ + -e "s|GH_TARGET_URL|$(repoURL)|g" \ + -e "s|GH_RUNNER_TOKEN|$(ghAccessToken)|g" \ + -e "s|GH_RUNNER_NAME|$(instanceName)|g" \ + -e "s|GH_RUNNER_LABELS|${LABELS}|g" > ${TMP_SCRIPT} + + AS_B64=$(base64 -w0 ${TMP_SCRIPT}) + echo "${CC_TPL}" | sed "s|RUNNER_INSTALL_B64|${AS_B64}|g" > ${TMP_CC} + echo "${TMP_CC}" +} + +function waitForServer() { + local srv_id="$1" + + srv_info=$(openstack server show -f json "${srv_id}") + [ $? -ne 0 ] && return $? + + status=$(echo "${srv_info}" | jq -r -c '.status') + + while [ "${status}" != "ERROR" -a "${status}" != "ACTIVE" ];do + sleep 2 + srv_info=$(openstack server show -f json "${srv_id}") + [ $? -ne 0 ] && return $? + status=$(echo "${srv_info}" | jq -r -c '.status') + done + echo "${srv_info}" +} + +function CreateInstance() { + if [ -z "$INPUT" ];then + echo "expected build params in stdin" + exit 1 + fi + + CC_FILE=$(getCloudConfig) + FLAVOR=$(flavor) + IMAGE=$(image) + INSTANCE_NAME=$(instanceName) + NET=$(getOpenStackNetworkID) + IMAGE_DETAILS=$(getOSImageDetails) + + OS_TYPE=$(echo "${IMAGE_DETAILS}" | jq -c -r '.properties.os_type') + checkValNotNull "${OS_TYPE}" "os_type" || return $? + DISTRO=$(echo "${IMAGE_DETAILS}" | jq -c -r '.properties.os_distro') + checkValNotNull "${OS_TYPE}" "os_distro" || return $? + VERSION=$(echo "${IMAGE_DETAILS}" | jq -c -r '.properties.os_version') + checkValNotNull "${VERSION}" "os_version" || return $? + ARCH=$(echo "${IMAGE_DETAILS}" | jq -c -r '.properties.architecture') + checkValNotNull "${ARCH}" "architecture" || return $? + + + SOURCE_ARGS="" + + if [ "${BOOT_FROM_VOLUME}" -eq 1 ];then + VOL_SIZE=$(getVolumeSizeFromFlavor "${FLAVOR}") + VOL_INFO=$(createVolumeFromImage "${IMAGE}" "${VOL_SIZE}" "${INSTANCE_NAME}") + if [ $? -ne 0 ];then + openstack volume delete "${INSTANCE_NAME}" || true + fi + SOURCE_ARGS="--volume ${INSTANCE_NAME}" + else + SOURCE_ARGS="--image ${IMAGE}" + fi + + set +e + SRV_DETAILS=$(openstack server create ${SOURCE_ARGS} --flavor "${FLAVOR}" --user-data="${CC_FILE}" --network="${NET}" "${INSTANCE_NAME}") + if [ $? -ne 0 ];then + openstack volume delete "${INSTANCE_NAME}" || true + exit 1 + fi + SRV_DETAILS=$(waitForServer "${INSTANCE_NAME}") + if [ $? -ne 0 ];then + CODE="$?" + # cleanup + rm -f "${CC_FILE}" || true + openstack server delete "${INSTANCE_NAME}" || true + openstack volume delete "${INSTANCE_NAME}" || true + set -e + return $CODE + fi + set -e + rm -f "${CC_FILE}" || true + + SRV_ID=$(echo "${SRV_DETAILS}" | jq -r -c '.id') + STATUS=$(echo "${SRV_DETAILS}" | jq -r -c '.status') + + jq -rnc \ + --arg PROVIDER_ID ${SRV_ID} \ + --arg NAME "${INSTANCE_NAME}" \ + --arg OS_TYPE "${OS_TYPE}" \ + --arg OS_NAME "${DISTRO}" \ + --arg OS_VERSION "${VERSION}" \ + --arg ARCH "${ARCH}" \ + --arg STATUS "${STATUS}" \ + '{"id": "", "provider_id": $PROVIDER_ID, "name": $NAME, "os_type": $OS_TYPE, "os_name": $OS_NAME, "os_version": $OS_VERSION, "os_arch": $ARCH, "status": $STATUS, "runner_status": "", "pool_id": ""}' +} + +function DeleteInstance() { + local instance_id="${GARM_INSTANCE_ID}" + if [ -z "${instance_id}" ];then + echo "missing instance ID in env" + return 1 + fi + + instance_info=$(openstack server show "${instance_id}" -f json) + VOLUMES=$(echo "${instance_info}" | jq -r -c '.volumes_attached[] | .id') + + openstack server delete "${instance_id}" + for vol in "$VOLUMES";do + waitForVolume "${vol}" + openstack volume delete $vol || true + done +} + +function StartInstance() { + local instance_id="${GARM_INSTANCE_ID}" + if [ -z "${instance_id}" ];then + echo "missing instance ID in env" + return 1 + fi + + openstack server start "${instance_id}" +} + +function StopServer() { + local instance_id="${GARM_INSTANCE_ID}" + if [ -z "${instance_id}" ];then + echo "missing instance ID in env" + return 1 + fi + + openstack server stop "${instance_id}" +} + +case "$GARM_COMMAND" in + "CreateInstance") + CreateInstance + ;; + "DeleteInstance") + DeleteInstance + ;; + "GetInstance") + echo "GetInstance not implemented" + exit 1 + ;; + "ListInstances") + echo "ListInstances not implemented" + exit 1 + ;; + "StartInstance") + StartInstance + ;; + "StopInstance") + StopServer + ;; + "RemoveAllInstances") + echo "RemoveAllInstances not implemented" + exit 1 + ;; + *) + echo "Invalid GARM provider command: \"$GARM_COMMAND\"" + exit 1 + ;; +esac + diff --git a/contrib/providers.d/openstack/keystonerc b/contrib/providers.d/openstack/keystonerc new file mode 100644 index 00000000..1b702dd7 --- /dev/null +++ b/contrib/providers.d/openstack/keystonerc @@ -0,0 +1,16 @@ +# OpenStack client config +export OS_REGION_NAME=RegionOne +export OS_AUTH_VERSION=3 +export OS_AUTH_URL=http://10.0.8.36:5000/v3 +export OS_PROJECT_DOMAIN_NAME=admin_domain +export OS_USERNAME=admin +export OS_AUTH_TYPE=password +export OS_USER_DOMAIN_NAME=admin_domain +export OS_PROJECT_NAME=admin +export OS_PASSWORD=Iegeehahth4suSie +export OS_IDENTITY_API_VERSION=3 + + +# GARM config +export OPENSTACK_PRIVATE_NETWORK="int_net" +export BOOT_FROM_VOLUME=1 diff --git a/errors/errors.go b/errors/errors.go index 20c0245d..b205fa45 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -40,6 +40,20 @@ func (b *baseError) Error() string { return b.msg } +// NewProviderError returns a new ProviderError +func NewProviderError(msg string, a ...interface{}) error { + return &ProviderError{ + baseError{ + msg: fmt.Sprintf(msg, a...), + }, + } +} + +// UnauthorizedError is returned when a request is unauthorized +type ProviderError struct { + baseError +} + // NewUnauthorizedError returns a new UnauthorizedError func NewUnauthorizedError(msg string) error { return &UnauthorizedError{ diff --git a/go.mod b/go.mod index 105877dd..6b1cc79b 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/cobra v1.4.1-0.20220504202302-9e88759b19cd golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 + golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b gorm.io/driver/mysql v1.3.3 @@ -52,7 +53,6 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect - golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/go.sum b/go.sum index 49da1471..64c0134f 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,6 @@ github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybL github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.4.1-0.20220504202302-9e88759b19cd h1:0Hv1DPpsKWp/xjP1sQRfLDIymRDu79mErd9H9+l0uaE= github.com/spf13/cobra v1.4.1-0.20220504202302-9e88759b19cd/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/runner/pool/common.go b/runner/pool/common.go index 2043af68..1a83aa9e 100644 --- a/runner/pool/common.go +++ b/runner/pool/common.go @@ -298,6 +298,7 @@ func (r *basePool) addInstanceToProvider(instance params.Instance) error { Flavor: pool.Flavor, Image: pool.Image, Labels: labels, + PoolID: instance.PoolID, } providerInstance, err := provider.CreateInstance(r.ctx, bootstrapArgs) @@ -548,6 +549,7 @@ func (r *basePool) deletePendingInstances() { } for _, instance := range instances { + log.Printf("instance status for %s is %s", instance.Name, instance.Status) if instance.Status != providerCommon.InstancePendingDelete { // not in pending_delete status. Skip. continue diff --git a/runner/providers/external/external.go b/runner/providers/external/external.go new file mode 100644 index 00000000..fa95dfd1 --- /dev/null +++ b/runner/providers/external/external.go @@ -0,0 +1,175 @@ +package external + +import ( + "context" + "encoding/json" + "fmt" + + "garm/config" + garmErrors "garm/errors" + "garm/params" + "garm/runner/common" + "garm/util/exec" + + "github.com/pkg/errors" +) + +var _ common.Provider = (*external)(nil) + +func NewProvider(ctx context.Context, cfg *config.Provider, controllerID string) (common.Provider, error) { + if cfg.ProviderType != config.ExternalProvider { + return nil, garmErrors.NewBadRequestError("invalid provider config") + } + + execPath, err := cfg.External.ExecutablePath() + if err != nil { + return nil, errors.Wrap(err, "fetching executable path") + } + return &external{ + ctx: ctx, + controllerID: controllerID, + cfg: cfg, + execPath: execPath, + }, nil +} + +type external struct { + ctx context.Context + controllerID string + cfg *config.Provider + execPath string +} + +func (e *external) configEnvVar() string { + return fmt.Sprintf("GARM_PROVIDER_CONFIG_FILE=%s", e.cfg.External.ConfigFile) +} + +// CreateInstance creates a new compute instance in the provider. +func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { + asEnv := bootstrapParamsToEnv(bootstrapParams) + asEnv = append(asEnv, createInstanceCommand) + asEnv = append(asEnv, fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID)) + asEnv = append(asEnv, fmt.Sprintf("GARM_POOL_ID=%s", bootstrapParams.PoolID)) + asEnv = append(asEnv, e.configEnvVar()) + + asJs, err := json.Marshal(bootstrapParams) + if err != nil { + return params.Instance{}, errors.Wrap(err, "serializing bootstrap params") + } + + out, err := exec.Exec(ctx, e.execPath, asJs, asEnv) + if err != nil { + return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + + var param params.Instance + if err := json.Unmarshal(out, ¶m); err != nil { + return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) + } + return param, nil +} + +// Delete instance will delete the instance in a provider. +func (e *external) DeleteInstance(ctx context.Context, instance string) error { + asEnv := []string{ + deleteInstanceCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_INSTANCE_ID=%s", instance), + } + + _, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + return nil +} + +// GetInstance will return details about one instance. +func (e *external) GetInstance(ctx context.Context, instance string) (params.Instance, error) { + asEnv := []string{ + getInstanceCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_INSTANCE_ID=%s", instance), + } + + out, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + + var param params.Instance + if err := json.Unmarshal(out, ¶m); err != nil { + return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) + } + return param, nil +} + +// ListInstances will list all instances for a provider. +func (e *external) ListInstances(ctx context.Context, poolID string) ([]params.Instance, error) { + asEnv := []string{ + listInstancesCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_POOL_ID=%s", poolID), + } + + out, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return []params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + + var param []params.Instance + if err := json.Unmarshal(out, ¶m); err != nil { + return []params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) + } + return param, nil +} + +// RemoveAllInstances will remove all instances created by this provider. +func (e *external) RemoveAllInstances(ctx context.Context) error { + asEnv := []string{ + removeAllInstancesCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID), + } + _, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + return nil +} + +// Stop shuts down the instance. +func (e *external) Stop(ctx context.Context, instance string, force bool) error { + asEnv := []string{ + stopInstanceCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_INSTANCE_ID=%s", instance), + } + _, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + return nil +} + +// Start boots up an instance. +func (e *external) Start(ctx context.Context, instance string) error { + asEnv := []string{ + startInstanceCommand, + e.configEnvVar(), + fmt.Sprintf("GARM_INSTANCE_ID=%s", instance), + } + _, err := exec.Exec(ctx, e.execPath, nil, asEnv) + if err != nil { + return garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) + } + return nil +} + +func (e *external) AsParams() params.Provider { + return params.Provider{ + Name: e.cfg.Name, + Description: e.cfg.Description, + ProviderType: e.cfg.ProviderType, + } +} diff --git a/runner/providers/external/util.go b/runner/providers/external/util.go new file mode 100644 index 00000000..4a39b2e3 --- /dev/null +++ b/runner/providers/external/util.go @@ -0,0 +1,49 @@ +package external + +import ( + "fmt" + "garm/params" + "strings" +) + +const ( + envPrefix = "GARM" + + createInstanceCommand = "GARM_COMMAND=CreateInstance" + deleteInstanceCommand = "GARM_COMMAND=DeleteInstance" + getInstanceCommand = "GARM_COMMAND=GetInstance" + listInstancesCommand = "GARM_COMMAND=ListInstances" + startInstanceCommand = "GARM_COMMAND=StartInstance" + stopInstanceCommand = "GARM_COMMAND=StopInstance" + removeAllInstancesCommand = "GARM_COMMAND=RemoveAllInstances" +) + +func bootstrapParamsToEnv(param params.BootstrapInstance) []string { + ret := []string{ + fmt.Sprintf("%s_BOOTSTRAP_NAME='%s'", envPrefix, param.Name), + fmt.Sprintf("%s_BOOTSTRAP_OS_ARCH='%s'", envPrefix, param.OSArch), + fmt.Sprintf("%s_BOOTSTRAP_FLAVOR='%s'", envPrefix, param.Flavor), + fmt.Sprintf("%s_BOOTSTRAP_IMAGE='%s'", envPrefix, param.Image), + fmt.Sprintf("%s_BOOTSTRAP_POOL_ID='%s'", envPrefix, param.PoolID), + fmt.Sprintf("%s_BOOTSTRAP_INSTANCE_TOKEN='%s'", envPrefix, param.InstanceToken), + fmt.Sprintf("%s_BOOTSTRAP_CALLBACK_URL='%s'", envPrefix, param.CallbackURL), + fmt.Sprintf("%s_BOOTSTRAP_REPO_URL='%s'", envPrefix, param.RepoURL), + fmt.Sprintf("%s_BOOTSTRAP_LABELS='%s'", envPrefix, strings.Join(param.Labels, ",")), + fmt.Sprintf("%s_BOOTSTRAP_GITHUB_ACCESS_TOKEN='%s'", envPrefix, param.GithubRunnerAccessToken), + } + + for idx, tool := range param.Tools { + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_TOOLS_DOWNLOAD_URL_%d='%s'", envPrefix, idx, *tool.DownloadURL)) + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_TOOLS_ARCH_%d='%s'", envPrefix, idx, *tool.Architecture)) + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_TOOLS_OS_%d='%s'", envPrefix, idx, *tool.OS)) + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_TOOLS_FILENAME_%d='%s'", envPrefix, idx, *tool.Filename)) + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_TOOLS_SHA256_%d='%s'", envPrefix, idx, *tool.SHA256Checksum)) + } + + for idx, sshKey := range param.SSHKeys { + ret = append(ret, fmt.Sprintf("%s_BOOTSTRAP_SSH_KEY_%d='%s'", envPrefix, idx, sshKey)) + + } + + return ret +} diff --git a/runner/providers/lxd/lxd.go b/runner/providers/lxd/lxd.go index 313631ec..f0f7bea5 100644 --- a/runner/providers/lxd/lxd.go +++ b/runner/providers/lxd/lxd.go @@ -16,6 +16,7 @@ package lxd import ( "context" + "encoding/json" "fmt" "log" @@ -224,6 +225,8 @@ func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance) (a return api.InstancesPost{}, errors.Wrap(err, "generating cloud-config") } + fmt.Printf(">>> Cloud-config: \n%s\n", cloudCfg) + args := api.InstancesPost{ InstancePut: api.InstancePut{ Architecture: image.Architecture, @@ -289,6 +292,8 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error { // CreateInstance creates a new compute instance in the provider. func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { + asJs, _ := json.MarshalIndent(bootstrapParams, "", " ") + fmt.Printf(">>> %s\n", string(asJs)) args, err := l.getCreateInstanceArgs(bootstrapParams) if err != nil { return params.Instance{}, errors.Wrap(err, "fetching create args") @@ -300,7 +305,14 @@ func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.Bootstr return params.Instance{}, errors.Wrap(err, "creating instance") } - return l.GetInstance(ctx, args.Name) + ret, err := l.GetInstance(ctx, args.Name) + if err != nil { + return params.Instance{}, errors.Wrap(err, "fetching instance") + } + + asJs2, _ := json.MarshalIndent(ret, "", " ") + fmt.Printf(">>>22 %s\n", string(asJs2)) + return ret, nil } // GetInstance will return details about one instance. diff --git a/runner/providers/providers.go b/runner/providers/providers.go index 7258090a..2b494faa 100644 --- a/runner/providers/providers.go +++ b/runner/providers/providers.go @@ -20,6 +20,7 @@ import ( "garm/config" "garm/runner/common" + "garm/runner/providers/external" "garm/runner/providers/lxd" "github.com/pkg/errors" @@ -39,6 +40,13 @@ func LoadProvidersFromConfig(ctx context.Context, cfg config.Config, controllerI return nil, errors.Wrap(err, "creating provider") } providers[providerCfg.Name] = provider + case config.ExternalProvider: + conf := providerCfg + provider, err := external.NewProvider(ctx, &conf, controllerID) + if err != nil { + return nil, errors.Wrap(err, "creating provider") + } + providers[providerCfg.Name] = provider } } return providers, nil diff --git a/testdata/config.toml b/testdata/config.toml index 7dd2a271..480ae1fe 100644 --- a/testdata/config.toml +++ b/testdata/config.toml @@ -136,6 +136,20 @@ time_to_live = "8760h" protocol = "simplestreams" skip_verify = false +# This is an example external provider. External providers are executables that +# implement the needed interface to create/delete/list compute systems that are used +# by garm to create runners. +[[provider]] +name = "openstack_external" +description = "external openstack provider" +provider_type = "external" + [provider.external] + # config file passed to the executable via GARM_PROVIDER_CONFIG_FILE environment variable + config_file = "/home/ubuntu/garm/providers.d/openstack/keystonerc" + # path on disk to a folder that contains a "garm-external-provider" executable. The executable + # can be anything (bash, a binary, python, etc) + provider_dir = "/home/ubuntu/garm/providers.d/openstack" + # This is a list of credentials that you can define as part of the repository # or organization definitions. They are not saved inside the database, as there # is no Vault integration (yet). This will change in the future. diff --git a/util/exec/exec.go b/util/exec/exec.go new file mode 100644 index 00000000..654b0955 --- /dev/null +++ b/util/exec/exec.go @@ -0,0 +1,25 @@ +package exec + +import ( + "bytes" + "context" + "os/exec" + + "github.com/pkg/errors" +) + +func Exec(ctx context.Context, providerBin string, stdinData []byte, environ []string) ([]byte, error) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + c := exec.CommandContext(ctx, providerBin) + c.Env = environ + c.Stdin = bytes.NewBuffer(stdinData) + c.Stdout = stdout + c.Stderr = stderr + + if err := c.Run(); err != nil { + return nil, errors.Wrapf(err, "provider binary failed with stdout: %s; stderr: %s", stdout.String(), stderr.String()) + } + + return stdout.Bytes(), nil +} diff --git a/util/exec/exec_nix.go b/util/exec/exec_nix.go new file mode 100644 index 00000000..76f8ba46 --- /dev/null +++ b/util/exec/exec_nix.go @@ -0,0 +1,16 @@ +//go:build !windows +// +build !windows + +package exec + +import ( + "golang.org/x/sys/unix" +) + +func IsExecutable(path string) bool { + if unix.Access(path, unix.X_OK) == nil { + return true + } + + return false +} diff --git a/util/exec/exec_windows.go b/util/exec/exec_windows.go new file mode 100644 index 00000000..0c17839c --- /dev/null +++ b/util/exec/exec_windows.go @@ -0,0 +1,18 @@ +package exec + +import ( + "os" + "strings" +) + +func IsExecutable(path string) bool { + pathExt := os.Getenv("PATHEXT") + execList := strings.Split(pathExt, ";") + for _, ext := range execList { + if strings.HasSuffix(path, ext) { + return true + } + } + + return false +}