From ce45e123fe047441efb503c683a652103cc2f573 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 16 Jul 2019 23:32:49 -0700 Subject: [PATCH] Port Typhoon Fedora CoreOS support to AWS * Use the newly minted "Fedora CoreOS Preview" AMI * Remove iscsi, kubelet.path activation, and kubeconfig distribution * As usual, bare-metal efforts make cloud provider ports much easier --- aws/fedora-coreos/kubernetes/LICENSE | 23 ++ aws/fedora-coreos/kubernetes/README.md | 23 ++ aws/fedora-coreos/kubernetes/ami.tf | 21 + aws/fedora-coreos/kubernetes/bootkube.tf | 19 + aws/fedora-coreos/kubernetes/controllers.tf | 86 +++++ .../kubernetes/fcc/controller.yaml | 179 +++++++++ aws/fedora-coreos/kubernetes/network.tf | 67 ++++ aws/fedora-coreos/kubernetes/nlb.tf | 94 +++++ aws/fedora-coreos/kubernetes/outputs.tf | 54 +++ aws/fedora-coreos/kubernetes/security.tf | 364 ++++++++++++++++++ aws/fedora-coreos/kubernetes/ssh.tf | 92 +++++ aws/fedora-coreos/kubernetes/variables.tf | 156 ++++++++ aws/fedora-coreos/kubernetes/versions.tf | 11 + aws/fedora-coreos/kubernetes/workers.tf | 23 ++ aws/fedora-coreos/kubernetes/workers/ami.tf | 21 + .../kubernetes/workers/fcc/worker.yaml | 108 ++++++ .../kubernetes/workers/ingress.tf | 48 +++ .../kubernetes/workers/outputs.tf | 10 + .../kubernetes/workers/variables.tf | 107 +++++ .../kubernetes/workers/versions.tf | 4 + .../kubernetes/workers/workers.tf | 89 +++++ 21 files changed, 1599 insertions(+) create mode 100644 aws/fedora-coreos/kubernetes/LICENSE create mode 100644 aws/fedora-coreos/kubernetes/README.md create mode 100644 aws/fedora-coreos/kubernetes/ami.tf create mode 100644 aws/fedora-coreos/kubernetes/bootkube.tf create mode 100644 aws/fedora-coreos/kubernetes/controllers.tf create mode 100644 aws/fedora-coreos/kubernetes/fcc/controller.yaml create mode 100644 aws/fedora-coreos/kubernetes/network.tf create mode 100644 aws/fedora-coreos/kubernetes/nlb.tf create mode 100644 aws/fedora-coreos/kubernetes/outputs.tf create mode 100644 aws/fedora-coreos/kubernetes/security.tf create mode 100644 aws/fedora-coreos/kubernetes/ssh.tf create mode 100644 aws/fedora-coreos/kubernetes/variables.tf create mode 100644 aws/fedora-coreos/kubernetes/versions.tf create mode 100644 aws/fedora-coreos/kubernetes/workers.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/ami.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml create mode 100644 aws/fedora-coreos/kubernetes/workers/ingress.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/outputs.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/variables.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/versions.tf create mode 100644 aws/fedora-coreos/kubernetes/workers/workers.tf diff --git a/aws/fedora-coreos/kubernetes/LICENSE b/aws/fedora-coreos/kubernetes/LICENSE new file mode 100644 index 00000000..bd9a5eea --- /dev/null +++ b/aws/fedora-coreos/kubernetes/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2017 Typhoon Authors +Copyright (c) 2017 Dalton Hubble + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md new file mode 100644 index 00000000..ce8ee9a3 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/README.md @@ -0,0 +1,23 @@ +# Typhoon + +Typhoon is a minimal and free Kubernetes distribution. + +* Minimal, stable base Kubernetes distribution +* Declarative infrastructure and configuration +* Free (freedom and cost) and privacy-respecting +* Practical for labs, datacenters, and clouds + +Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components. + +## Features + +* Kubernetes v1.15.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/cl/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization +* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) + +## Docs + +Please see the [official docs](https://typhoon.psdn.io) and the AWS [tutorial](https://typhoon.psdn.io/cl/aws/). + diff --git a/aws/fedora-coreos/kubernetes/ami.tf b/aws/fedora-coreos/kubernetes/ami.tf new file mode 100644 index 00000000..f4ecec66 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/ami.tf @@ -0,0 +1,21 @@ + +data "aws_ami" "fedora-coreos" { + most_recent = true + owners = ["125523088429"] + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + // pin on known ok versions as preview matures + filter { + name = "name" + values = ["fedora-coreos-30.20190716.1-hvm"] + } +} diff --git a/aws/fedora-coreos/kubernetes/bootkube.tf b/aws/fedora-coreos/kubernetes/bootkube.tf new file mode 100644 index 00000000..9c58da8e --- /dev/null +++ b/aws/fedora-coreos/kubernetes/bootkube.tf @@ -0,0 +1,19 @@ +# Self-hosted Kubernetes assets (kubeconfig, manifests) +module "bootkube" { + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + + cluster_name = var.cluster_name + api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] + etcd_servers = aws_route53_record.etcds.*.fqdn + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation + + trusted_certs_dir = "/etc/pki/tls/certs" +} + diff --git a/aws/fedora-coreos/kubernetes/controllers.tf b/aws/fedora-coreos/kubernetes/controllers.tf new file mode 100644 index 00000000..821ad648 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/controllers.tf @@ -0,0 +1,86 @@ +# Discrete DNS records for each controller's private IPv4 for etcd usage +resource "aws_route53_record" "etcds" { + count = var.controller_count + + # DNS Zone where record should be created + zone_id = var.dns_zone_id + + name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone) + type = "A" + ttl = 300 + + # private IPv4 address for etcd + records = [aws_instance.controllers.*.private_ip[count.index]] +} + +# Controller instances +resource "aws_instance" "controllers" { + count = var.controller_count + + tags = { + Name = "${var.cluster_name}-controller-${count.index}" + } + + instance_type = var.controller_type + + ami = data.aws_ami.fedora-coreos.image_id + user_data = data.ct_config.controller-ignitions.*.rendered[count.index] + + # storage + root_block_device { + volume_type = var.disk_type + volume_size = var.disk_size + iops = var.disk_iops + } + + # network + associate_public_ip_address = true + subnet_id = aws_subnet.public.*.id[count.index] + vpc_security_group_ids = [aws_security_group.controller.id] + + lifecycle { + ignore_changes = [ + ami, + user_data, + ] + } +} + +# Controller Ignition configs +data "ct_config" "controller-ignitions" { + count = var.controller_count + content = data.template_file.controller-configs.*.rendered[count.index] + strict = true + snippets = var.controller_snippets +} + +# Controller Fedora CoreOS configs +data "template_file" "controller-configs" { + count = var.controller_count + + template = file("${path.module}/fcc/controller.yaml") + + vars = { + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered) + kubeconfig = indent(10, module.bootkube.kubeconfig-kubelet) + ssh_authorized_key = var.ssh_authorized_key + cluster_dns_service_ip = cidrhost(var.service_cidr, 10) + cluster_domain_suffix = var.cluster_domain_suffix + } +} + +data "template_file" "etcds" { + count = var.controller_count + template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380" + + vars = { + index = count.index + cluster_name = var.cluster_name + dns_zone = var.dns_zone + } +} + diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 00000000..7130c304 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,179 @@ +--- +variant: fcos +version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + ExecStartPre=/bin/mkdir -p /var/lib/etcd + ExecStartPre=-/usr/bin/podman rm etcd + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + ExecStart=/usr/bin/podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.3.13 + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target + - name: docker.service + enabled: true + - name: wait-for-dns.service + enabled: true + contents: | + [Unit] + Description=Wait for DNS entries + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + RequiredBy=etcd-member.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/master \ + --node-labels=node-role.kubernetes.io/controller="true" \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: bootkube.service + contents: | + [Unit] + Description=Bootstrap a Kubernetes control plane + ConditionPathExists=!/opt/bootkube/init_bootkube.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootkube + ExecStart=/usr/bin/bash -c 'set -x && \ + [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \ + --network host \ + --volume /opt/bootkube/assets:/assets \ + --volume /etc/kubernetes:/etc/kubernetes \ + quay.io/coreos/bootkube:v0.14.0 \ + /bootkube start --asset-dir=/assets' + ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + # TODO: Use a systemd dropin once podman v1.4.5 is avail. + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} diff --git a/aws/fedora-coreos/kubernetes/network.tf b/aws/fedora-coreos/kubernetes/network.tf new file mode 100644 index 00000000..a93b3f0c --- /dev/null +++ b/aws/fedora-coreos/kubernetes/network.tf @@ -0,0 +1,67 @@ +data "aws_availability_zones" "all" { +} + +# Network VPC, gateway, and routes + +resource "aws_vpc" "network" { + cidr_block = var.host_cidr + assign_generated_ipv6_cidr_block = true + enable_dns_support = true + enable_dns_hostnames = true + + tags = { + "Name" = var.cluster_name + } +} + +resource "aws_internet_gateway" "gateway" { + vpc_id = aws_vpc.network.id + + tags = { + "Name" = var.cluster_name + } +} + +resource "aws_route_table" "default" { + vpc_id = aws_vpc.network.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.gateway.id + } + + route { + ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.gateway.id + } + + tags = { + "Name" = var.cluster_name + } +} + +# Subnets (one per availability zone) + +resource "aws_subnet" "public" { + count = length(data.aws_availability_zones.all.names) + + vpc_id = aws_vpc.network.id + availability_zone = data.aws_availability_zones.all.names[count.index] + + cidr_block = cidrsubnet(var.host_cidr, 4, count.index) + ipv6_cidr_block = cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index) + map_public_ip_on_launch = true + assign_ipv6_address_on_creation = true + + tags = { + "Name" = "${var.cluster_name}-public-${count.index}" + } +} + +resource "aws_route_table_association" "public" { + count = length(data.aws_availability_zones.all.names) + + route_table_id = aws_route_table.default.id + subnet_id = aws_subnet.public.*.id[count.index] +} + diff --git a/aws/fedora-coreos/kubernetes/nlb.tf b/aws/fedora-coreos/kubernetes/nlb.tf new file mode 100644 index 00000000..2b87366a --- /dev/null +++ b/aws/fedora-coreos/kubernetes/nlb.tf @@ -0,0 +1,94 @@ +# Network Load Balancer DNS Record +resource "aws_route53_record" "apiserver" { + zone_id = var.dns_zone_id + + name = format("%s.%s.", var.cluster_name, var.dns_zone) + type = "A" + + # AWS recommends their special "alias" records for NLBs + alias { + name = aws_lb.nlb.dns_name + zone_id = aws_lb.nlb.zone_id + evaluate_target_health = true + } +} + +# Network Load Balancer for apiservers and ingress +resource "aws_lb" "nlb" { + name = "${var.cluster_name}-nlb" + load_balancer_type = "network" + internal = false + + subnets = aws_subnet.public.*.id + + enable_cross_zone_load_balancing = true +} + +# Forward TCP apiserver traffic to controllers +resource "aws_lb_listener" "apiserver-https" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = "6443" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.controllers.arn + } +} + +# Forward HTTP ingress traffic to workers +resource "aws_lb_listener" "ingress-http" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = 80 + + default_action { + type = "forward" + target_group_arn = module.workers.target_group_http + } +} + +# Forward HTTPS ingress traffic to workers +resource "aws_lb_listener" "ingress-https" { + load_balancer_arn = aws_lb.nlb.arn + protocol = "TCP" + port = 443 + + default_action { + type = "forward" + target_group_arn = module.workers.target_group_https + } +} + +# Target group of controllers +resource "aws_lb_target_group" "controllers" { + name = "${var.cluster_name}-controllers" + vpc_id = aws_vpc.network.id + target_type = "instance" + + protocol = "TCP" + port = 6443 + + # TCP health check for apiserver + health_check { + protocol = "TCP" + port = 6443 + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + +# Attach controller instances to apiserver NLB +resource "aws_lb_target_group_attachment" "controllers" { + count = var.controller_count + + target_group_arn = aws_lb_target_group.controllers.arn + target_id = aws_instance.controllers.*.id[count.index] + port = 6443 +} + diff --git a/aws/fedora-coreos/kubernetes/outputs.tf b/aws/fedora-coreos/kubernetes/outputs.tf new file mode 100644 index 00000000..471c3300 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/outputs.tf @@ -0,0 +1,54 @@ +output "kubeconfig-admin" { + value = module.bootkube.kubeconfig-admin +} + +# Outputs for Kubernetes Ingress + +output "ingress_dns_name" { + value = aws_lb.nlb.dns_name + description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" +} + +output "ingress_zone_id" { + value = aws_lb.nlb.zone_id + description = "Route53 zone id of the network load balancer DNS name that can be used in Route53 alias records" +} + +# Outputs for worker pools + +output "vpc_id" { + value = aws_vpc.network.id + description = "ID of the VPC for creating worker instances" +} + +output "subnet_ids" { + value = aws_subnet.public.*.id + description = "List of subnet IDs for creating worker instances" +} + +output "worker_security_groups" { + value = [aws_security_group.worker.id] + description = "List of worker security group IDs" +} + +output "kubeconfig" { + value = module.bootkube.kubeconfig-kubelet +} + +# Outputs for custom load balancing + +output "nlb_id" { + description = "ARN of the Network Load Balancer" + value = aws_lb.nlb.id +} + +output "worker_target_group_http" { + description = "ARN of a target group of workers for HTTP traffic" + value = module.workers.target_group_http +} + +output "worker_target_group_https" { + description = "ARN of a target group of workers for HTTPS traffic" + value = module.workers.target_group_https +} + diff --git a/aws/fedora-coreos/kubernetes/security.tf b/aws/fedora-coreos/kubernetes/security.tf new file mode 100644 index 00000000..aa2f84cb --- /dev/null +++ b/aws/fedora-coreos/kubernetes/security.tf @@ -0,0 +1,364 @@ +# Security Groups (instance firewalls) + +# Controller security group + +resource "aws_security_group" "controller" { + name = "${var.cluster_name}-controller" + description = "${var.cluster_name} controller security group" + + vpc_id = aws_vpc.network.id + + tags = { + "Name" = "${var.cluster_name}-controller" + } +} + +resource "aws_security_group_rule" "controller-ssh" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "controller-etcd" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 2379 + to_port = 2380 + self = true +} + +# Allow Prometheus to scrape etcd metrics +resource "aws_security_group_rule" "controller-etcd-metrics" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 2381 + to_port = 2381 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-vxlan" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-vxlan-self" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +resource "aws_security_group_rule" "controller-apiserver" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + cidr_blocks = ["0.0.0.0/0"] +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "aws_security_group_rule" "controller-node-exporter" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + source_security_group_id = aws_security_group.worker.id +} + +# Allow apiserver to access kubelets for exec, log, port-forward +resource "aws_security_group_rule" "controller-kubelet" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-kubelet-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "controller-bgp" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-bgp-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "controller-ipip" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-ipip-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-ipip-legacy" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.worker.id +} + +resource "aws_security_group_rule" "controller-ipip-legacy-self" { + security_group_id = aws_security_group.controller.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-egress" { + security_group_id = aws_security_group.controller.id + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} + +# Worker security group + +resource "aws_security_group" "worker" { + name = "${var.cluster_name}-worker" + description = "${var.cluster_name} worker security group" + + vpc_id = aws_vpc.network.id + + tags = { + "Name" = "${var.cluster_name}-worker" + } +} + +resource "aws_security_group_rule" "worker-ssh" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-http" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-https" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-vxlan" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-vxlan-self" { + count = var.networking == "flannel" ? 1 : 0 + + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "udp" + from_port = 4789 + to_port = 4789 + self = true +} + +# Allow Prometheus to scrape node-exporter daemonset +resource "aws_security_group_rule" "worker-node-exporter" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + self = true +} + +resource "aws_security_group_rule" "ingress-health" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10254 + to_port = 10254 + cidr_blocks = ["0.0.0.0/0"] +} + +# Allow apiserver to access kubelets for exec, log, port-forward +resource "aws_security_group_rule" "worker-kubelet" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + source_security_group_id = aws_security_group.controller.id +} + +# Allow Prometheus to scrape kubelet metrics +resource "aws_security_group_rule" "worker-kubelet-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "worker-bgp" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-bgp-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "worker-ipip" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-ipip-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-ipip-legacy" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = aws_security_group.controller.id +} + +resource "aws_security_group_rule" "worker-ipip-legacy-self" { + security_group_id = aws_security_group.worker.id + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-egress" { + security_group_id = aws_security_group.worker.id + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} + diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 00000000..09e31c19 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,92 @@ +# Secure copy etcd TLS assets to controllers. +resource "null_resource" "copy-controller-secrets" { + count = var.controller_count + + connection { + type = "ssh" + host = aws_instance.controllers.*.public_ip[count.index] + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = module.bootkube.etcd_ca_cert + destination = "$HOME/etcd-client-ca.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_cert + destination = "$HOME/etcd-client.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_key + destination = "$HOME/etcd-client.key" + } + + provisioner "file" { + content = module.bootkube.etcd_server_cert + destination = "$HOME/etcd-server.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_server_key + destination = "$HOME/etcd-server.key" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_cert + destination = "$HOME/etcd-peer.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_key + destination = "$HOME/etcd-peer.key" + } + + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /etc/ssl/etcd/etcd", + "sudo mv etcd-client* /etc/ssl/etcd/", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", + "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", + "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", + "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", + "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", + "sudo chown -R etcd:etcd /etc/ssl/etcd", + "sudo chmod -R 500 /etc/ssl/etcd", + ] + } +} + +# Secure copy bootkube assets to ONE controller and start bootkube to perform +# one-time self-hosted cluster bootstrapping. +resource "null_resource" "bootkube-start" { + depends_on = [ + module.bootkube, + module.workers, + aws_route53_record.apiserver, + null_resource.copy-controller-secrets, + ] + + connection { + type = "ssh" + host = aws_instance.controllers[0].public_ip + user = "core" + timeout = "15m" + } + + provisioner "file" { + source = var.asset_dir + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv $HOME/assets /opt/bootkube", + "sudo systemctl start bootkube", + ] + } +} + diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 00000000..8df68d85 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,156 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name (prepended to dns_zone)" +} + +# AWS + +variable "dns_zone" { + type = string + description = "AWS Route53 DNS Zone (e.g. aws.example.com)" +} + +variable "dns_zone_id" { + type = string + description = "AWS Route53 DNS Zone ID (e.g. Z3PAABBCFAKEC0)" +} + +# instances + +variable "controller_count" { + type = string + default = "1" + description = "Number of controllers (i.e. masters)" +} + +variable "worker_count" { + type = string + default = "1" + description = "Number of workers" +} + +variable "controller_type" { + type = string + default = "t3.small" + description = "EC2 instance type for controllers" +} + +variable "worker_type" { + type = string + default = "t3.small" + description = "EC2 instance type for workers" +} + +variable "os_image" { + type = string + default = "coreos-stable" + description = "AMI channel for Fedora CoreOS (not yet used)" +} + +variable "disk_size" { + type = string + default = "40" + description = "Size of the EBS volume in GB" +} + +variable "disk_type" { + type = string + default = "gp2" + description = "Type of the EBS volume (e.g. standard, gp2, io1)" +} + +variable "disk_iops" { + type = string + default = "0" + description = "IOPS of the EBS volume (e.g. 100)" +} + +variable "worker_price" { + type = string + default = "" + description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" +} + +variable "worker_target_groups" { + type = list(string) + description = "Additional target group ARNs to which worker instances should be added" + default = [] +} + +variable "controller_snippets" { + type = list(string) + description = "Controller Fedora CoreOS Config snippets" + default = [] +} + +variable "worker_snippets" { + type = list(string) + description = "Worker Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "asset_dir" { + description = "Path to a directory where generated assets should be placed (contains secrets)" + type = string +} + +variable "networking" { + description = "Choice of networking provider (calico or flannel)" + type = string + default = "calico" +} + +variable "network_mtu" { + description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." + type = string + default = "1480" +} + +variable "host_cidr" { + description = "CIDR IPv4 range to assign to EC2 nodes" + type = string + default = "10.0.0.0/16" +} + +variable "pod_cidr" { + description = "CIDR IPv4 range to assign Kubernetes pods" + type = string + default = "10.2.0.0/16" +} + +variable "service_cidr" { + description = < /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enabled: true + contents: | + [Unit] + Description=Kubelet via Hyperkube (System Container) + Wants=rpc-statd.service + [Service] + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /var/lib/calico + ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins + ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/podman rm kubelet + ExecStart=/usr/bin/podman run --name kubelet \ + --privileged \ + --pid host \ + --network host \ + --volume /etc/kubernetes:/etc/kubernetes:ro,z \ + --volume /usr/lib/os-release:/etc/os-release:ro \ + --volume /etc/ssl/certs:/etc/ssl/certs:ro \ + --volume /lib/modules:/lib/modules:ro \ + --volume /run:/run \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + --volume /etc/pki/tls/certs:/usr/share/ca-certificates:ro \ + --volume /var/lib/calico:/var/lib/calico \ + --volume /var/lib/docker:/var/lib/docker \ + --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ + --volume /var/log:/var/log \ + --volume /var/run:/var/run \ + --volume /var/run/lock:/var/run/lock:z \ + --volume /opt/cni/bin:/opt/cni/bin:z \ + k8s.gcr.io/hyperkube:v1.15.0 /hyperkube kubelet \ + --anonymous-auth=false \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --cgroup-driver=systemd \ + --cgroups-per-qos=true \ + --enforce-node-allocatable=pods \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --cluster_dns=${cluster_dns_service_ip} \ + --cluster_domain=${cluster_domain_suffix} \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --read-only-port=0 \ + --volume-plugin-dir=/var/lib/kubelet/volumeplugins + ExecStop=-/usr/bin/podman stop kubelet + Delegate=yes + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target +storage: + directories: + - path: /etc/kubernetes + - path: /opt/bootkube + files: + - path: /etc/kubernetes/kubeconfig + mode: 0644 + contents: + inline: | + ${kubeconfig} + - path: /etc/sysctl.d/reverse-path-filter.conf + contents: + inline: | + net.ipv4.conf.all.rp_filter=1 + - path: /etc/sysctl.d/max-user-watches.conf + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/systemd/system.conf.d/accounting.conf + contents: + inline: | + [Manager] + DefaultCPUAccounting=yes + DefaultMemoryAccounting=yes + DefaultBlockIOAccounting=yes +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/aws/fedora-coreos/kubernetes/workers/ingress.tf b/aws/fedora-coreos/kubernetes/workers/ingress.tf new file mode 100644 index 00000000..6b25152f --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/ingress.tf @@ -0,0 +1,48 @@ +# Target groups of instances for use with load balancers + +resource "aws_lb_target_group" "workers-http" { + name = "${var.name}-workers-http" + vpc_id = var.vpc_id + target_type = "instance" + + protocol = "TCP" + port = 80 + + # HTTP health check for ingress + health_check { + protocol = "HTTP" + port = 10254 + path = "/healthz" + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + +resource "aws_lb_target_group" "workers-https" { + name = "${var.name}-workers-https" + vpc_id = var.vpc_id + target_type = "instance" + + protocol = "TCP" + port = 443 + + # HTTP health check for ingress + health_check { + protocol = "HTTP" + port = 10254 + path = "/healthz" + + # NLBs required to use same healthy and unhealthy thresholds + healthy_threshold = 3 + unhealthy_threshold = 3 + + # Interval between health checks required to be 10 or 30 + interval = 10 + } +} + diff --git a/aws/fedora-coreos/kubernetes/workers/outputs.tf b/aws/fedora-coreos/kubernetes/workers/outputs.tf new file mode 100644 index 00000000..22f37885 --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/outputs.tf @@ -0,0 +1,10 @@ +output "target_group_http" { + description = "ARN of a target group of workers for HTTP traffic" + value = aws_lb_target_group.workers-http.arn +} + +output "target_group_https" { + description = "ARN of a target group of workers for HTTPS traffic" + value = aws_lb_target_group.workers-https.arn +} + diff --git a/aws/fedora-coreos/kubernetes/workers/variables.tf b/aws/fedora-coreos/kubernetes/workers/variables.tf new file mode 100644 index 00000000..a1de562f --- /dev/null +++ b/aws/fedora-coreos/kubernetes/workers/variables.tf @@ -0,0 +1,107 @@ +variable "name" { + type = string + description = "Unique name for the worker pool" +} + +# AWS + +variable "vpc_id" { + type = string + description = "Must be set to `vpc_id` output by cluster" +} + +variable "subnet_ids" { + type = list(string) + description = "Must be set to `subnet_ids` output by cluster" +} + +variable "security_groups" { + type = list(string) + description = "Must be set to `worker_security_groups` output by cluster" +} + +# instances + +variable "worker_count" { + type = string + default = "1" + description = "Number of instances" +} + +variable "instance_type" { + type = string + default = "t3.small" + description = "EC2 instance type" +} + +variable "os_image" { + type = string + default = "coreos-stable" + description = "AMI channel for Fedora CoreOS (not yet used)" +} + +variable "disk_size" { + type = string + default = "40" + description = "Size of the EBS volume in GB" +} + +variable "disk_type" { + type = string + default = "gp2" + description = "Type of the EBS volume (e.g. standard, gp2, io1)" +} + +variable "disk_iops" { + type = string + default = "0" + description = "IOPS of the EBS volume (required for io1)" +} + +variable "spot_price" { + type = string + default = "" + description = "Spot price in USD for autoscaling group spot instances. Leave as default empty string for autoscaling group to use on-demand instances. Note, switching in-place from spot to on-demand is not possible: https://github.com/terraform-providers/terraform-provider-aws/issues/4320" +} + +variable "target_groups" { + type = list(string) + description = "Additional target group ARNs to which instances should be added" + default = [] +} + +variable "snippets" { + type = list(string) + description = "Fedora CoreOS Config snippets" + default = [] +} + +# configuration + +variable "kubeconfig" { + type = string + description = "Must be set to `kubeconfig` output by cluster" +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "service_cidr" { + description = <