From 590d941f50dc77815c9e4c3b6ec27678c3638325 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 16 Mar 2020 21:21:41 -0700 Subject: [PATCH] Switch from upstream hyperkube image to individual images * Kubernetes plans to stop releasing the hyperkube container image * Upstream will continue to publish `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images to `k8s.gcr.io` * Upstream will publish Kubelet only as a binary for distros to package, either as a DEB/RPM on traditional distros or a container image on container-optimized operating systems * Typhoon will package the upstream Kubelet (checksummed) and its dependencies as a container image for use on CoreOS Container Linux, Flatcar Linux, and Fedora CoreOS * Update the Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact Hyperkube: https://github.com/kubernetes/kubernetes/pull/88676 Kubelet Container Image: https://github.com/poseidon/kubelet Kubelet Quay Repo: https://quay.io/repository/poseidon/kubelet --- CHANGES.md | 9 +++++++++ aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 5 ++--- aws/container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- bare-metal/container-linux/kubernetes/cl/worker.yaml | 3 +-- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/cl/worker.yaml | 7 +++---- docs/architecture/operating-systems.md | 4 ++-- docs/topics/security.md | 9 +++++++-- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 5 ++--- .../container-linux/kubernetes/workers/cl/worker.yaml | 11 +++++------ google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 4 ++-- .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 4 ++-- 27 files changed, 66 insertions(+), 62 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3bd9663e..f19f4305 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,15 @@ Notable changes between versions. ## Latest * Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5) +* Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669)) + * Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images + * Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary (checksummed) and + its dependencies as an automated build container image [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) + * Update base images for control plane and Kubelet images used by Typhoon from upstream's debian 9 (stretch) to debian 10 + (buster) base + * Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact + * Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container + image and provide the Kubelet as a binary for distros to package #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index f55e011d..65431247 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index b0a4258c..59b634f8 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -91,8 +91,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -136,7 +135,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index a41dc853..a7733b7e 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -64,8 +64,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -80,9 +79,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -128,11 +127,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:612b947 \ --net=host \ --dns=host \ -- \ - kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 556cc760..5036643e 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 93297d7d..b9cac354 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 07492c3e..c840da1d 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index d7656e2c..3f8118a6 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 4cafaa00..a8dc507d 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -90,8 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index 749cb9bc..6ca5794f 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,8 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -78,9 +77,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -126,11 +125,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 25c4f686..f60e7b21 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 4f90796a..db0b430b 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -103,8 +103,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -149,7 +148,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index f4b923d3..09450435 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -76,8 +76,7 @@ systemd: --mount volume=etc-iscsi,target=/etc/iscsi \ --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 077df9a4..6941dd26 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 1fd0d53b..69176ebc 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -80,7 +80,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +134,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 54395326..e02083d1 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -50,7 +50,7 @@ systemd: --volume /opt/cni/bin:/opt/cni/bin:z \ --volume /etc/iscsi:/etc/iscsi \ --volume /sbin/iscsiadm:/sbin/iscsiadm \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index bfec3b98..e35dd0e6 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index acd643af..cb190e39 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -101,8 +101,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -146,7 +145,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 230e05a8..2a36572c 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -74,8 +74,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -132,8 +131,8 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 9db88649..50f30a6d 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -31,8 +31,8 @@ Together, they diversify Typhoon to support a range of container technologies. | single-master | all platforms | all platforms | | multi-master | all platforms | all platforms | | control plane | static pods | static pods | -| kubelet image | upstream hyperkube | upstream hyperkube | -| control plane images | upstream hyperkube | upstream hyperkube | +| kubelet image | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | +| control plane images | upstream images | upstream images | | on-host etcd | rkt-fly | podman | | on-host kubelet | rkt-fly | podman | | CNI plugins | calico or flannel | calico or flannel | diff --git a/docs/topics/security.md b/docs/topics/security.md index 60934e4c..fc4cbb42 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -40,9 +40,14 @@ Typhoon limits exposure to many security threats, but it is not a silver bullet. * Do not give untrusted users a shell behind your firewall * Define network policies for your namespaces -## OpenPGP Signing +## Container Images -Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own. If you find artifacts claiming to be from Typhoon, please send a note. +Typhoon uses upstream container images (where possible) and upstream binaries. + +!!! note + Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems. + +Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents. ## Disclosures diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f3244b99..83cd7ba7 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index cdf1c57e..a8d97a54 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -90,8 +90,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,7 +133,7 @@ systemd: --volume script,kind=host,source=/opt/bootstrap/apply \ --mount volume=script,target=/apply \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ --exec=/apply diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 4cf23d9b..6ccd5254 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -63,8 +63,7 @@ systemd: --mount volume=var-log,target=/var/log \ --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ --mount volume=opt-cni-bin,target=/opt/cni/bin \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ - --exec=/usr/local/bin/kubelet -- \ + docker://quay.io/poseidon/kubelet:v1.17.4 -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -78,9 +77,9 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ - %{ for label in split(",", node_labels) } + %{~ for label in split(",", node_labels) ~} --node-labels=${label} \ - %{ endfor ~} + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins @@ -126,11 +125,11 @@ storage: --volume config,kind=host,source=/etc/kubernetes \ --mount volume=config,target=/etc/kubernetes \ --insecure-options=image \ - docker://k8s.gcr.io/hyperkube:v1.17.4 \ + docker://quay.io/poseidon/kubelet:v1.17.4 \ --net=host \ --dns=host \ -- \ - kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) + /usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index fb05144e..b8d9c1b5 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 93297d7d..b9cac354 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -79,7 +79,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -123,7 +123,7 @@ systemd: --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ --entrypoint=/apply \ - k8s.gcr.io/hyperkube:v1.17.4 + quay.io/poseidon/kubelet:v1.17.4 ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done ExecStartPost=-/usr/bin/podman stop bootstrap storage: diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 07492c3e..17a19c4f 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -49,7 +49,7 @@ systemd: --volume /var/log:/var/log \ --volume /var/run/lock:/var/run/lock:z \ --volume /opt/cni/bin:/opt/cni/bin:z \ - k8s.gcr.io/hyperkube:v1.17.4 kubelet \ + quay.io/poseidon/kubelet:v1.17.4 \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -87,7 +87,7 @@ systemd: Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' + ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: