From 2325a503e1a294792470d51d9f638a52bb1e700c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 24 Feb 2024 18:48:06 -0800 Subject: [PATCH] Add an `install_container_networking` variable (default `true`) * When `true`, the chosen container `networking` provider is installed during cluster bootstrap * Set `false` to self-manage the container networking provider. This allows flannel, Calico, or Cilium to be managed via Terraform (like any other Kubernetes resources). Nodes will be NotReady until you apply the self-managed container networking provider. This may become the default in future. --- CHANGES.md | 8 +++++++- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/variables.tf | 6 ++++++ aws/flatcar-linux/kubernetes/bootstrap.tf | 2 +- aws/flatcar-linux/kubernetes/variables.tf | 6 ++++++ azure/fedora-coreos/kubernetes/bootstrap.tf | 3 +-- azure/fedora-coreos/kubernetes/variables.tf | 6 ++++++ azure/flatcar-linux/kubernetes/bootstrap.tf | 3 +-- azure/flatcar-linux/kubernetes/variables.tf | 6 ++++++ bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/variables.tf | 6 ++++++ bare-metal/flatcar-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/flatcar-linux/kubernetes/variables.tf | 6 ++++++ digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 3 +-- digital-ocean/fedora-coreos/kubernetes/variables.tf | 6 ++++++ digital-ocean/flatcar-linux/kubernetes/bootstrap.tf | 3 +-- digital-ocean/flatcar-linux/kubernetes/variables.tf | 6 ++++++ google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/variables.tf | 6 ++++++ google-cloud/flatcar-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/flatcar-linux/kubernetes/variables.tf | 6 ++++++ 21 files changed, 77 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ac406b42..5ba612ec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,13 @@ Notable changes between versions. * Kubernetes [v1.29.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.29.md#v1292) * Update Cilium from v1.14.3 to [v1.15.1](https://github.com/cilium/cilium/releases/tag/v1.15.1) * Update flannel from v0.22.2 to [v0.24.2](https://github.com/flannel-io/flannel/releases/tag/v0.24.2) -* Allow CNI `networking` to be set to "none" to skip bootstrapping flannel, Calico, or Cilium ([#1419](https://github.com/poseidon/typhoon/pull/1419)) +* Add an `install_container_networking` variable (default `true`) + * When `true`, the chosen container `networking` provider is installed during cluster bootstrap + * Set `false` to self-manage the container networking provider. This allows flannel, Calico, or Cilium + to be managed via Terraform (like any other Kubernetes resources). Nodes will be NotReady until you + apply the self-managed container networking provider. This may become the default in future. + * Continue to set `networking` to one of the three supported container networking providers. Most + require custom firewall / security policies be present across nodes so they have some infra tie-ins. ## v1.29.1 diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 24cc5f08..37f47553 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = aws_route53_record.etcds.*.fqdn - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = var.network_mtu pod_cidr = var.pod_cidr service_cidr = var.service_cidr diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index b8679bf9..52f6bd46 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -107,6 +107,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "network_mtu" { type = number description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." diff --git a/aws/flatcar-linux/kubernetes/bootstrap.tf b/aws/flatcar-linux/kubernetes/bootstrap.tf index 24cc5f08..37f47553 100644 --- a/aws/flatcar-linux/kubernetes/bootstrap.tf +++ b/aws/flatcar-linux/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = aws_route53_record.etcds.*.fqdn - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = var.network_mtu pod_cidr = var.pod_cidr service_cidr = var.service_cidr diff --git a/aws/flatcar-linux/kubernetes/variables.tf b/aws/flatcar-linux/kubernetes/variables.tf index 81e62ed2..25839dd6 100644 --- a/aws/flatcar-linux/kubernetes/variables.tf +++ b/aws/flatcar-linux/kubernetes/variables.tf @@ -107,6 +107,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "network_mtu" { type = number description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 16cc0c16..29ca8471 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -6,8 +6,7 @@ module "bootstrap" { api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) - networking = var.networking - + networking = var.install_container_networking ? var.networking : "none" # only effective with Calico networking # we should be able to use 1450 MTU, but in practice, 1410 was needed network_encapsulation = "vxlan" diff --git a/azure/fedora-coreos/kubernetes/variables.tf b/azure/fedora-coreos/kubernetes/variables.tf index 05ae4496..05e3050b 100644 --- a/azure/fedora-coreos/kubernetes/variables.tf +++ b/azure/fedora-coreos/kubernetes/variables.tf @@ -94,6 +94,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "host_cidr" { type = string description = "CIDR IPv4 range to assign to instances" diff --git a/azure/flatcar-linux/kubernetes/bootstrap.tf b/azure/flatcar-linux/kubernetes/bootstrap.tf index 16cc0c16..29ca8471 100644 --- a/azure/flatcar-linux/kubernetes/bootstrap.tf +++ b/azure/flatcar-linux/kubernetes/bootstrap.tf @@ -6,8 +6,7 @@ module "bootstrap" { api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) - networking = var.networking - + networking = var.install_container_networking ? var.networking : "none" # only effective with Calico networking # we should be able to use 1450 MTU, but in practice, 1410 was needed network_encapsulation = "vxlan" diff --git a/azure/flatcar-linux/kubernetes/variables.tf b/azure/flatcar-linux/kubernetes/variables.tf index 7b2dd15a..e14b871a 100644 --- a/azure/flatcar-linux/kubernetes/variables.tf +++ b/azure/flatcar-linux/kubernetes/variables.tf @@ -100,6 +100,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "host_cidr" { type = string description = "CIDR IPv4 range to assign to instances" diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 9ce564bd..77d9b605 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] etcd_servers = var.controllers.*.domain - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = var.network_mtu network_ip_autodetection_method = var.network_ip_autodetection_method pod_cidr = var.pod_cidr diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 943c2f0c..daee1916 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -92,6 +92,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "network_mtu" { type = number description = "CNI interface MTU (applies to calico only)" diff --git a/bare-metal/flatcar-linux/kubernetes/bootstrap.tf b/bare-metal/flatcar-linux/kubernetes/bootstrap.tf index ecc38fbb..d21f3a98 100644 --- a/bare-metal/flatcar-linux/kubernetes/bootstrap.tf +++ b/bare-metal/flatcar-linux/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] etcd_servers = var.controllers.*.domain - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = var.network_mtu network_ip_autodetection_method = var.network_ip_autodetection_method pod_cidr = var.pod_cidr diff --git a/bare-metal/flatcar-linux/kubernetes/variables.tf b/bare-metal/flatcar-linux/kubernetes/variables.tf index 2f379887..422a1a66 100644 --- a/bare-metal/flatcar-linux/kubernetes/variables.tf +++ b/bare-metal/flatcar-linux/kubernetes/variables.tf @@ -91,6 +91,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "network_mtu" { type = number description = "CNI interface MTU (applies to calico only)" diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index b7fd4273..17b238e0 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -6,8 +6,7 @@ module "bootstrap" { api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = digitalocean_record.etcds.*.fqdn - networking = var.networking - + networking = var.install_container_networking ? var.networking : "none" # only effective with Calico networking network_encapsulation = "vxlan" network_mtu = "1450" diff --git a/digital-ocean/fedora-coreos/kubernetes/variables.tf b/digital-ocean/fedora-coreos/kubernetes/variables.tf index 4a6dd8ad..4dc67bd2 100644 --- a/digital-ocean/fedora-coreos/kubernetes/variables.tf +++ b/digital-ocean/fedora-coreos/kubernetes/variables.tf @@ -71,6 +71,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods" diff --git a/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf b/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf index b7fd4273..17b238e0 100644 --- a/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf @@ -6,8 +6,7 @@ module "bootstrap" { api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = digitalocean_record.etcds.*.fqdn - networking = var.networking - + networking = var.install_container_networking ? var.networking : "none" # only effective with Calico networking network_encapsulation = "vxlan" network_mtu = "1450" diff --git a/digital-ocean/flatcar-linux/kubernetes/variables.tf b/digital-ocean/flatcar-linux/kubernetes/variables.tf index 7c755af6..3748b69b 100644 --- a/digital-ocean/flatcar-linux/kubernetes/variables.tf +++ b/digital-ocean/flatcar-linux/kubernetes/variables.tf @@ -71,6 +71,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods" diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index f0b7e02f..dff45bcb 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")] - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = 1440 pod_cidr = var.pod_cidr service_cidr = var.service_cidr diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf index 4ea49983..ad561e13 100644 --- a/google-cloud/fedora-coreos/kubernetes/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -94,6 +94,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods" diff --git a/google-cloud/flatcar-linux/kubernetes/bootstrap.tf b/google-cloud/flatcar-linux/kubernetes/bootstrap.tf index f0b7e02f..dff45bcb 100644 --- a/google-cloud/flatcar-linux/kubernetes/bootstrap.tf +++ b/google-cloud/flatcar-linux/kubernetes/bootstrap.tf @@ -5,7 +5,7 @@ module "bootstrap" { cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")] - networking = var.networking + networking = var.install_container_networking ? var.networking : "none" network_mtu = 1440 pod_cidr = var.pod_cidr service_cidr = var.service_cidr diff --git a/google-cloud/flatcar-linux/kubernetes/variables.tf b/google-cloud/flatcar-linux/kubernetes/variables.tf index e13da824..3a510f49 100644 --- a/google-cloud/flatcar-linux/kubernetes/variables.tf +++ b/google-cloud/flatcar-linux/kubernetes/variables.tf @@ -94,6 +94,12 @@ variable "networking" { default = "cilium" } +variable "install_container_networking" { + type = bool + description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)" + default = true +} + variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods"