bare-metal: Ues Terraform templating for Container Linux configs

* Template bare-metal Container Linux configs with Terraform's (limited) template_file module. This allows rendering problems to be identified during `terraform plan` and is favored over using the Matchbox templating feature when the configs are served to PXE booting nodes. * Writes a Matchbox profile for each machine, which will be served as-is. The effect is the same, each node gets provisioned with its own Container Linux config.
2024-06-01 12:26:12 +02:00 · 2017-09-23 11:49:12 -07:00 · 2017-09-23 11:49:12 -07:00 · bca96bb124
parent cd368c123f
commit bca96bb124
4 changed files with 56 additions and 41 deletions
--- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl
+++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl
@ -9,12 +9,12 @@ systemd:
          contents: |
            [Service]
            Environment="ETCD_IMAGE_TAG=v3.2.0"
-            Environment="ETCD_NAME={{.etcd_name}}"
-            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://{{.domain_name}}:2379"
-            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{.domain_name}}:2380"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380"
            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
-            Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
@ -79,11 +79,11 @@ systemd:
          --allow-privileged \
          --anonymous-auth=false \
          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns={{.k8s_dns_service_ip}} \
+          --cluster_dns=${k8s_dns_service_ip} \
          --cluster_domain=cluster.local \
          --cni-conf-dir=/etc/kubernetes/cni/net.d \
          --exit-on-lock-contention \
-          --hostname-override={{.domain_name}} \
+          --hostname-override=${domain_name} \
          --kubeconfig=/etc/kubernetes/kubeconfig \
          --lock-file=/var/run/lock/kubelet.lock \
          --network-plugin=cni \
@ -137,7 +137,7 @@ storage:
      mode: 0644
      contents:
        inline:
-          {{.domain_name}}
+          ${domain_name}
    - path: /etc/sysctl.d/max-user-watches.conf
      filesystem: root
      contents:
@ -159,17 +159,17 @@ storage:
          [ -d /opt/bootkube/assets/manifests-* ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
          [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
          [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
-          BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.6.2}"
-          BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
+          BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.6.2}"
+          BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
          exec /usr/bin/rkt run \
            --trust-keys-from-https \
            --volume assets,kind=host,source=$BOOTKUBE_ASSETS \
            --mount volume=assets,target=/assets \
            --volume bootstrap,kind=host,source=/etc/kubernetes \
            --mount volume=bootstrap,target=/etc/kubernetes \
-            $RKT_OPTS \
-            ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} \
+            $$RKT_OPTS \
+            $${BOOTKUBE_ACI}:$${BOOTKUBE_VERSION} \
            --net=host \
            --dns=host \
            --exec=/bootkube -- start --asset-dir=/assets "$@"
@ -177,5 +177,4 @@ passwd:
  users:
    - name: core
      ssh_authorized_keys:
-        - {{.ssh_authorized_key}}
-
+        - ${ssh_authorized_key}
--- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl
+++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl
@ -54,11 +54,11 @@ systemd:
          --allow-privileged \
          --anonymous-auth=false \
          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns={{.k8s_dns_service_ip}} \
+          --cluster_dns=${k8s_dns_service_ip} \
          --cluster_domain=cluster.local \
          --cni-conf-dir=/etc/kubernetes/cni/net.d \
          --exit-on-lock-contention \
-          --hostname-override={{.domain_name}} \
+          --hostname-override=${domain_name} \
          --kubeconfig=/etc/kubernetes/kubeconfig \
          --lock-file=/var/run/lock/kubelet.lock \
          --network-plugin=cni \
@ -101,7 +101,7 @@ storage:
      mode: 0644
      contents:
        inline:
-          {{.domain_name}}
+          ${domain_name}
    - path: /etc/sysctl.d/max-user-watches.conf
      filesystem: root
      contents:
@ -111,5 +111,5 @@ passwd:
  users:
    - name: core
      ssh_authorized_keys:
-        - {{.ssh_authorized_key}}
+        - ${ssh_authorized_key}

--- a/bare-metal/container-linux/kubernetes/groups.tf
+++ b/bare-metal/container-linux/kubernetes/groups.tf
@ -17,7 +17,7 @@ resource "matchbox_group" "container-linux-install" {
 resource "matchbox_group" "controller" {
  count   = "${length(var.controller_names)}"
  name    = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}"
-  profile = "${matchbox_profile.controller.name}"
+  profile = "${element(matchbox_profile.controllers.*.name, count.index)}"

  selector {
    mac = "${element(var.controller_macs, count.index)}"
@ -25,29 +25,17 @@ resource "matchbox_group" "controller" {
  }

  metadata {
-    domain_name          = "${element(var.controller_domains, count.index)}"
-    etcd_name            = "${element(var.controller_names, count.index)}"
-    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
-    etcd_on_host         = "${var.experimental_self_hosted_etcd ? "false" : "true"}"
-    k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key   = "${var.ssh_authorized_key}"
+    etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}"
  }
 }

 resource "matchbox_group" "worker" {
  count   = "${length(var.worker_names)}"
  name    = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}"
-  profile = "${matchbox_profile.worker.name}"
+  profile = "${element(matchbox_profile.workers.*.name, count.index)}"

  selector {
    mac = "${element(var.worker_macs, count.index)}"
    os  = "installed"
  }
-
-  metadata {
-    domain_name        = "${element(var.worker_domains, count.index)}"
-    etcd_on_host       = "${var.experimental_self_hosted_etcd ? "false" : "true"}"
-    k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key = "${var.ssh_authorized_key}"
-  }
 }
--- a/bare-metal/container-linux/kubernetes/profiles.tf
+++ b/bare-metal/container-linux/kubernetes/profiles.tf
@ -67,14 +67,42 @@ data "template_file" "cached-container-linux-install-config" {
  }
 }

-// Kubernetes Controller profile
-resource "matchbox_profile" "controller" {
-  name                   = "controller"
-  container_linux_config = "${file("${path.module}/cl/controller.yaml.tmpl")}"
+// Kubernetes Controller profiles
+resource "matchbox_profile" "controllers" {
+  count                  = "${length(var.controller_names)}"
+  name                   = "${format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index))}"
+  container_linux_config = "${element(data.template_file.controller-configs.*.rendered, count.index)}"
 }

-// Kubernetes Worker profile
-resource "matchbox_profile" "worker" {
-  name                   = "worker"
-  container_linux_config = "${file("${path.module}/cl/worker.yaml.tmpl")}"
+data "template_file" "controller-configs" {
+  count = "${length(var.controller_names)}"
+
+  template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
+
+  vars {
+    domain_name          = "${element(var.controller_domains, count.index)}"
+    etcd_name            = "${element(var.controller_names, count.index)}"
+    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
+    k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
+    ssh_authorized_key   = "${var.ssh_authorized_key}"
+  }
+}
+
+// Kubernetes Worker profiles
+resource "matchbox_profile" "workers" {
+  count                  = "${length(var.worker_names)}"
+  name                   = "${format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index))}"
+  container_linux_config = "${element(data.template_file.worker-configs.*.rendered, count.index)}"
+}
+
+data "template_file" "worker-configs" {
+  count = "${length(var.worker_names)}"
+
+  template = "${file("${path.module}/cl/worker.yaml.tmpl")}"
+
+  vars {
+    domain_name        = "${element(var.worker_domains, count.index)}"
+    k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+    ssh_authorized_key = "${var.ssh_authorized_key}"
+  }
 }