mirror of
https://github.com/poseidon/typhoon
synced 2024-05-23 07:56:14 +02:00
0e7977694f
* Set CNI networking to "none" to skip installing any CNI provider (i.e. no flannel, Calico, or Cilium). In this mode, cluster nodes will be NotReady until you add your own CNI stack * Motivation: I now tend to manage CNI components as addon modules just like other applications overlaid onto a cluster. It allows for faster iteration and may eventually become the recommendation
264 lines
9.6 KiB
YAML
264 lines
9.6 KiB
YAML
---
|
|
variant: fcos
|
|
version: 1.5.0
|
|
systemd:
|
|
units:
|
|
- name: etcd-member.service
|
|
enabled: true
|
|
contents: |
|
|
[Unit]
|
|
Description=etcd (System Container)
|
|
Documentation=https://github.com/etcd-io/etcd
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
[Service]
|
|
Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.5.12
|
|
Type=exec
|
|
ExecStartPre=/bin/mkdir -p /var/lib/etcd
|
|
ExecStartPre=-/usr/bin/podman rm etcd
|
|
ExecStart=/usr/bin/podman run --name etcd \
|
|
--env-file /etc/etcd/etcd.env \
|
|
--log-driver k8s-file \
|
|
--network host \
|
|
--volume /var/lib/etcd:/var/lib/etcd:rw,Z \
|
|
--volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
|
|
$${ETCD_IMAGE}
|
|
ExecStop=/usr/bin/podman stop etcd
|
|
Restart=on-failure
|
|
RestartSec=10s
|
|
TimeoutStartSec=0
|
|
LimitNOFILE=40000
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
- name: containerd.service
|
|
enabled: true
|
|
- name: docker.service
|
|
mask: true
|
|
- name: wait-for-dns.service
|
|
enabled: true
|
|
contents: |
|
|
[Unit]
|
|
Description=Wait for DNS entries
|
|
Before=kubelet.service
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=true
|
|
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
|
|
[Install]
|
|
RequiredBy=kubelet.service
|
|
RequiredBy=etcd-member.service
|
|
- name: kubelet.service
|
|
enabled: true
|
|
contents: |
|
|
[Unit]
|
|
Description=Kubelet (System Container)
|
|
Wants=rpc-statd.service
|
|
[Service]
|
|
Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.29.2
|
|
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
|
|
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
|
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
|
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
|
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
|
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
|
|
ExecStartPre=-/usr/bin/podman rm kubelet
|
|
ExecStart=/usr/bin/podman run --name kubelet \
|
|
--log-driver k8s-file \
|
|
--privileged \
|
|
--pid host \
|
|
--network host \
|
|
--volume /etc/cni/net.d:/etc/cni/net.d:ro,z \
|
|
--volume /etc/kubernetes:/etc/kubernetes:ro,z \
|
|
--volume /usr/lib/os-release:/etc/os-release:ro \
|
|
--volume /etc/machine-id:/etc/machine-id:ro \
|
|
--volume /lib/modules:/lib/modules:ro \
|
|
--volume /run:/run \
|
|
--volume /sys/fs/cgroup:/sys/fs/cgroup \
|
|
--volume /etc/selinux:/etc/selinux \
|
|
--volume /sys/fs/selinux:/sys/fs/selinux \
|
|
--volume /var/lib/calico:/var/lib/calico:ro \
|
|
--volume /var/lib/containerd:/var/lib/containerd \
|
|
--volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
|
|
--volume /var/log:/var/log \
|
|
--volume /var/run/lock:/var/run/lock:z \
|
|
--volume /opt/cni/bin:/opt/cni/bin:z \
|
|
$${KUBELET_IMAGE} \
|
|
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
|
|
--config=/etc/kubernetes/kubelet.yaml \
|
|
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \
|
|
--kubeconfig=/var/lib/kubelet/kubeconfig \
|
|
--node-labels=node.kubernetes.io/controller="true" \
|
|
--register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
|
|
ExecStop=-/usr/bin/podman stop kubelet
|
|
Delegate=yes
|
|
Restart=always
|
|
RestartSec=10
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
- name: bootstrap.service
|
|
contents: |
|
|
[Unit]
|
|
Description=Kubernetes control plane
|
|
ConditionPathExists=!/opt/bootstrap/bootstrap.done
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=true
|
|
WorkingDirectory=/opt/bootstrap
|
|
ExecStartPre=-/usr/bin/podman rm bootstrap
|
|
ExecStart=/usr/bin/podman run --name bootstrap \
|
|
--network host \
|
|
--volume /etc/kubernetes/pki:/etc/kubernetes/pki:ro,z \
|
|
--volume /opt/bootstrap/assets:/assets:ro,Z \
|
|
--volume /opt/bootstrap/apply:/apply:ro,Z \
|
|
--entrypoint=/apply \
|
|
quay.io/poseidon/kubelet:v1.29.2
|
|
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
|
|
ExecStartPost=-/usr/bin/podman stop bootstrap
|
|
storage:
|
|
directories:
|
|
- path: /var/lib/etcd
|
|
mode: 0700
|
|
- path: /etc/kubernetes
|
|
- path: /opt/bootstrap
|
|
files:
|
|
- path: /etc/kubernetes/kubeconfig
|
|
mode: 0644
|
|
contents:
|
|
inline: |
|
|
${kubeconfig}
|
|
- path: /etc/kubernetes/kubelet.yaml
|
|
contents:
|
|
inline: |
|
|
apiVersion: kubelet.config.k8s.io/v1beta1
|
|
kind: KubeletConfiguration
|
|
authentication:
|
|
anonymous:
|
|
enabled: false
|
|
webhook:
|
|
enabled: true
|
|
x509:
|
|
clientCAFile: /etc/kubernetes/ca.crt
|
|
authorization:
|
|
mode: Webhook
|
|
cgroupDriver: systemd
|
|
clusterDNS:
|
|
- ${cluster_dns_service_ip}
|
|
clusterDomain: ${cluster_domain_suffix}
|
|
healthzPort: 0
|
|
rotateCertificates: true
|
|
shutdownGracePeriod: 45s
|
|
shutdownGracePeriodCriticalPods: 30s
|
|
staticPodPath: /etc/kubernetes/manifests
|
|
readOnlyPort: 0
|
|
resolvConf: /run/systemd/resolve/resolv.conf
|
|
volumePluginDir: /var/lib/kubelet/volumeplugins
|
|
- path: /opt/bootstrap/layout
|
|
mode: 0544
|
|
contents:
|
|
inline: |
|
|
#!/bin/bash -e
|
|
mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
|
|
awk '/#####/ {filename=$2; next} {print > filename}' assets
|
|
mkdir -p /etc/ssl/etcd/etcd
|
|
mkdir -p /etc/kubernetes/pki
|
|
mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
|
|
mv tls/etcd/etcd-client* /etc/kubernetes/pki/
|
|
chown -R etcd:etcd /etc/ssl/etcd
|
|
chmod -R 500 /etc/ssl/etcd
|
|
mv auth/* /etc/kubernetes/pki/
|
|
mv tls/k8s/* /etc/kubernetes/pki/
|
|
mkdir -p /etc/kubernetes/manifests
|
|
mv static-manifests/* /etc/kubernetes/manifests/
|
|
mkdir -p /opt/bootstrap/assets
|
|
mv manifests /opt/bootstrap/assets/manifests
|
|
mv manifests-networking/* /opt/bootstrap/assets/manifests/ 2>/dev/null || true
|
|
rm -rf assets auth static-manifests tls manifests-networking manifests
|
|
chcon -R -u system_u -t container_file_t /etc/kubernetes/pki
|
|
- path: /opt/bootstrap/apply
|
|
mode: 0544
|
|
contents:
|
|
inline: |
|
|
#!/bin/bash -e
|
|
export KUBECONFIG=/etc/kubernetes/pki/admin.conf
|
|
until kubectl version; do
|
|
echo "Waiting for static pod control plane"
|
|
sleep 5
|
|
done
|
|
until kubectl apply -f /assets/manifests -R; do
|
|
echo "Retry applying manifests"
|
|
sleep 5
|
|
done
|
|
- path: /etc/systemd/logind.conf.d/inhibitors.conf
|
|
contents:
|
|
inline: |
|
|
[Login]
|
|
InhibitDelayMaxSec=45s
|
|
- path: /etc/sysctl.d/max-user-watches.conf
|
|
contents:
|
|
inline: |
|
|
fs.inotify.max_user_watches=16184
|
|
- path: /etc/sysctl.d/reverse-path-filter.conf
|
|
contents:
|
|
inline: |
|
|
net.ipv4.conf.default.rp_filter=0
|
|
net.ipv4.conf.*.rp_filter=0
|
|
- path: /etc/systemd/network/50-flannel.link
|
|
contents:
|
|
inline: |
|
|
[Match]
|
|
OriginalName=flannel*
|
|
[Link]
|
|
MACAddressPolicy=none
|
|
- path: /etc/systemd/system.conf.d/accounting.conf
|
|
contents:
|
|
inline: |
|
|
[Manager]
|
|
DefaultCPUAccounting=yes
|
|
DefaultMemoryAccounting=yes
|
|
DefaultBlockIOAccounting=yes
|
|
- path: /etc/etcd/etcd.env
|
|
mode: 0644
|
|
contents:
|
|
inline: |
|
|
ETCD_NAME=${etcd_name}
|
|
ETCD_DATA_DIR=/var/lib/etcd
|
|
ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379
|
|
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380
|
|
ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
|
|
ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
|
|
ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
|
|
ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
|
|
ETCD_STRICT_RECONFIG_CHECK=true
|
|
ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt
|
|
ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt
|
|
ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key
|
|
ETCD_CLIENT_CERT_AUTH=true
|
|
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt
|
|
ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
|
|
ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
|
|
ETCD_PEER_CLIENT_CERT_AUTH=true
|
|
- path: /etc/containerd/config.toml
|
|
overwrite: true
|
|
contents:
|
|
inline: |
|
|
version = 2
|
|
root = "/var/lib/containerd"
|
|
state = "/run/containerd"
|
|
subreaper = true
|
|
oom_score = -999
|
|
[grpc]
|
|
address = "/run/containerd/containerd.sock"
|
|
uid = 0
|
|
gid = 0
|
|
[plugins."io.containerd.grpc.v1.cri"]
|
|
enable_selinux = true
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
|
runtime_type = "io.containerd.runc.v2"
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
|
SystemdCgroup = true
|
|
passwd:
|
|
users:
|
|
- name: core
|
|
ssh_authorized_keys:
|
|
- ${ssh_authorized_key}
|