2020-04-13 00:49:48 +02:00
|
|
|
---
|
|
|
|
variant: fcos
|
2021-07-18 18:30:24 +02:00
|
|
|
version: 1.4.0
|
2020-04-13 00:49:48 +02:00
|
|
|
systemd:
|
|
|
|
units:
|
|
|
|
- name: etcd-member.service
|
|
|
|
enabled: true
|
|
|
|
contents: |
|
|
|
|
[Unit]
|
|
|
|
Description=etcd (System Container)
|
2020-11-08 20:46:39 +01:00
|
|
|
Documentation=https://github.com/etcd-io/etcd
|
2020-04-13 00:49:48 +02:00
|
|
|
Wants=network-online.target network.target
|
|
|
|
After=network-online.target
|
|
|
|
[Service]
|
2022-04-28 05:51:51 +02:00
|
|
|
Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.5.4
|
2020-04-13 00:49:48 +02:00
|
|
|
Type=exec
|
|
|
|
ExecStartPre=/bin/mkdir -p /var/lib/etcd
|
|
|
|
ExecStartPre=-/usr/bin/podman rm etcd
|
|
|
|
ExecStart=/usr/bin/podman run --name etcd \
|
|
|
|
--env-file /etc/etcd/etcd.env \
|
|
|
|
--network host \
|
|
|
|
--volume /var/lib/etcd:/var/lib/etcd:rw,Z \
|
|
|
|
--volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
|
2020-11-08 20:46:39 +01:00
|
|
|
$${ETCD_IMAGE}
|
2020-04-13 00:49:48 +02:00
|
|
|
ExecStop=/usr/bin/podman stop etcd
|
2020-11-08 20:46:39 +01:00
|
|
|
Restart=on-failure
|
|
|
|
RestartSec=10s
|
|
|
|
TimeoutStartSec=0
|
|
|
|
LimitNOFILE=40000
|
2020-04-13 00:49:48 +02:00
|
|
|
[Install]
|
|
|
|
WantedBy=multi-user.target
|
2022-01-12 05:40:17 +01:00
|
|
|
- name: containerd.service
|
2020-04-13 00:49:48 +02:00
|
|
|
enabled: true
|
2022-01-14 19:10:32 +01:00
|
|
|
- name: docker.service
|
|
|
|
mask: true
|
2020-04-13 00:49:48 +02:00
|
|
|
- name: wait-for-dns.service
|
|
|
|
enabled: true
|
|
|
|
contents: |
|
|
|
|
[Unit]
|
|
|
|
Description=Wait for DNS entries
|
|
|
|
Before=kubelet.service
|
|
|
|
[Service]
|
|
|
|
Type=oneshot
|
|
|
|
RemainAfterExit=true
|
|
|
|
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
|
|
|
|
[Install]
|
|
|
|
RequiredBy=kubelet.service
|
|
|
|
RequiredBy=etcd-member.service
|
|
|
|
- name: kubelet.service
|
|
|
|
enabled: true
|
|
|
|
contents: |
|
|
|
|
[Unit]
|
2020-04-26 01:50:51 +02:00
|
|
|
Description=Kubelet (System Container)
|
2020-04-13 00:49:48 +02:00
|
|
|
Wants=rpc-statd.service
|
|
|
|
[Service]
|
2022-08-24 02:32:13 +02:00
|
|
|
Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.25.0
|
2021-04-02 08:45:19 +02:00
|
|
|
ExecStartPre=/bin/mkdir -p /etc/cni/net.d
|
2020-04-13 00:49:48 +02:00
|
|
|
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
|
|
|
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
|
|
|
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
|
|
|
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
|
|
|
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
|
|
|
|
ExecStartPre=-/usr/bin/podman rm kubelet
|
|
|
|
ExecStart=/usr/bin/podman run --name kubelet \
|
|
|
|
--privileged \
|
|
|
|
--pid host \
|
|
|
|
--network host \
|
2021-04-02 08:45:19 +02:00
|
|
|
--volume /etc/cni/net.d:/etc/cni/net.d:ro,z \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume /etc/kubernetes:/etc/kubernetes:ro,z \
|
2022-04-02 00:32:29 +02:00
|
|
|
--volume /etc/machine-id:/etc/machine-id:ro \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume /usr/lib/os-release:/etc/os-release:ro \
|
|
|
|
--volume /lib/modules:/lib/modules:ro \
|
|
|
|
--volume /run:/run \
|
2021-04-17 20:45:52 +02:00
|
|
|
--volume /sys/fs/cgroup:/sys/fs/cgroup \
|
2022-04-26 19:01:09 +02:00
|
|
|
--volume /etc/selinux:/etc/selinux \
|
|
|
|
--volume /sys/fs/selinux:/sys/fs/selinux \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume /var/lib/calico:/var/lib/calico:ro \
|
2022-01-12 05:40:17 +01:00
|
|
|
--volume /var/lib/containerd:/var/lib/containerd \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
|
|
|
|
--volume /var/log:/var/log \
|
|
|
|
--volume /var/run/lock:/var/run/lock:z \
|
|
|
|
--volume /opt/cni/bin:/opt/cni/bin:z \
|
2020-05-20 07:39:53 +02:00
|
|
|
$${KUBELET_IMAGE} \
|
2020-04-13 00:49:48 +02:00
|
|
|
--anonymous-auth=false \
|
|
|
|
--authentication-token-webhook \
|
|
|
|
--authorization-mode=Webhook \
|
2020-04-26 01:50:51 +02:00
|
|
|
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
|
2020-04-13 00:49:48 +02:00
|
|
|
--cgroup-driver=systemd \
|
|
|
|
--cgroups-per-qos=true \
|
2022-01-12 05:40:17 +01:00
|
|
|
--container-runtime=remote \
|
|
|
|
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \
|
2020-04-13 00:49:48 +02:00
|
|
|
--enforce-node-allocatable=pods \
|
|
|
|
--client-ca-file=/etc/kubernetes/ca.crt \
|
|
|
|
--cluster_dns=${cluster_dns_service_ip} \
|
|
|
|
--cluster_domain=${cluster_domain_suffix} \
|
|
|
|
--healthz-port=0 \
|
2020-04-26 01:50:51 +02:00
|
|
|
--kubeconfig=/var/lib/kubelet/kubeconfig \
|
2020-04-13 00:49:48 +02:00
|
|
|
--node-labels=node.kubernetes.io/controller="true" \
|
|
|
|
--pod-manifest-path=/etc/kubernetes/manifests \
|
|
|
|
--read-only-port=0 \
|
2021-12-10 17:22:30 +01:00
|
|
|
--resolv-conf=/run/systemd/resolve/resolv.conf \
|
2020-06-18 07:50:12 +02:00
|
|
|
--register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
|
2020-04-26 01:50:51 +02:00
|
|
|
--rotate-certificates \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
|
|
|
|
ExecStop=-/usr/bin/podman stop kubelet
|
|
|
|
Delegate=yes
|
|
|
|
Restart=always
|
|
|
|
RestartSec=10
|
|
|
|
[Install]
|
|
|
|
WantedBy=multi-user.target
|
|
|
|
- name: bootstrap.service
|
|
|
|
contents: |
|
|
|
|
[Unit]
|
|
|
|
Description=Kubernetes control plane
|
|
|
|
ConditionPathExists=!/opt/bootstrap/bootstrap.done
|
|
|
|
[Service]
|
|
|
|
Type=oneshot
|
|
|
|
RemainAfterExit=true
|
|
|
|
WorkingDirectory=/opt/bootstrap
|
|
|
|
ExecStartPre=-/usr/bin/podman rm bootstrap
|
|
|
|
ExecStart=/usr/bin/podman run --name bootstrap \
|
|
|
|
--network host \
|
2020-12-03 08:22:34 +01:00
|
|
|
--volume /etc/kubernetes/pki:/etc/kubernetes/pki:ro,z \
|
2020-04-13 00:49:48 +02:00
|
|
|
--volume /opt/bootstrap/assets:/assets:ro,Z \
|
|
|
|
--volume /opt/bootstrap/apply:/apply:ro,Z \
|
|
|
|
--entrypoint=/apply \
|
2022-08-24 02:32:13 +02:00
|
|
|
quay.io/poseidon/kubelet:v1.25.0
|
2020-04-13 00:49:48 +02:00
|
|
|
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
|
|
|
|
ExecStartPost=-/usr/bin/podman stop bootstrap
|
|
|
|
storage:
|
|
|
|
directories:
|
2020-07-25 22:50:08 +02:00
|
|
|
- path: /var/lib/etcd
|
|
|
|
mode: 0700
|
2020-04-13 00:49:48 +02:00
|
|
|
- path: /etc/kubernetes
|
|
|
|
- path: /opt/bootstrap
|
|
|
|
files:
|
|
|
|
- path: /etc/kubernetes/kubeconfig
|
|
|
|
mode: 0644
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
${kubeconfig}
|
|
|
|
- path: /opt/bootstrap/layout
|
|
|
|
mode: 0544
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
#!/bin/bash -e
|
|
|
|
mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
|
|
|
|
awk '/#####/ {filename=$2; next} {print > filename}' assets
|
|
|
|
mkdir -p /etc/ssl/etcd/etcd
|
2020-12-03 08:22:34 +01:00
|
|
|
mkdir -p /etc/kubernetes/pki
|
2020-04-13 00:49:48 +02:00
|
|
|
mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
|
2020-12-03 08:22:34 +01:00
|
|
|
mv tls/etcd/etcd-client* /etc/kubernetes/pki/
|
2020-04-13 00:49:48 +02:00
|
|
|
chown -R etcd:etcd /etc/ssl/etcd
|
|
|
|
chmod -R 500 /etc/ssl/etcd
|
2020-12-03 08:22:34 +01:00
|
|
|
mv auth/* /etc/kubernetes/pki/
|
|
|
|
mv tls/k8s/* /etc/kubernetes/pki/
|
2020-04-23 05:27:08 +02:00
|
|
|
mkdir -p /etc/kubernetes/manifests
|
|
|
|
mv static-manifests/* /etc/kubernetes/manifests/
|
|
|
|
mkdir -p /opt/bootstrap/assets
|
|
|
|
mv manifests /opt/bootstrap/assets/manifests
|
|
|
|
mv manifests-networking/* /opt/bootstrap/assets/manifests/
|
2020-04-13 00:49:48 +02:00
|
|
|
rm -rf assets auth static-manifests tls manifests-networking
|
2020-12-03 08:22:34 +01:00
|
|
|
chcon -R -u system_u -t container_file_t /etc/kubernetes/pki
|
2020-04-13 00:49:48 +02:00
|
|
|
- path: /opt/bootstrap/apply
|
|
|
|
mode: 0544
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
#!/bin/bash -e
|
2020-12-03 08:22:34 +01:00
|
|
|
export KUBECONFIG=/etc/kubernetes/pki/admin.conf
|
2020-04-13 00:49:48 +02:00
|
|
|
until kubectl version; do
|
|
|
|
echo "Waiting for static pod control plane"
|
|
|
|
sleep 5
|
|
|
|
done
|
|
|
|
until kubectl apply -f /assets/manifests -R; do
|
|
|
|
echo "Retry applying manifests"
|
|
|
|
sleep 5
|
|
|
|
done
|
|
|
|
- path: /etc/sysctl.d/max-user-watches.conf
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
fs.inotify.max_user_watches=16184
|
2020-06-16 07:42:57 +02:00
|
|
|
- path: /etc/sysctl.d/reverse-path-filter.conf
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
net.ipv4.conf.default.rp_filter=0
|
|
|
|
net.ipv4.conf.*.rp_filter=0
|
2020-08-02 06:00:39 +02:00
|
|
|
- path: /etc/systemd/network/50-flannel.link
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
[Match]
|
|
|
|
OriginalName=flannel*
|
|
|
|
[Link]
|
|
|
|
MACAddressPolicy=none
|
2020-04-13 00:49:48 +02:00
|
|
|
- path: /etc/systemd/system.conf.d/accounting.conf
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
[Manager]
|
|
|
|
DefaultCPUAccounting=yes
|
|
|
|
DefaultMemoryAccounting=yes
|
|
|
|
DefaultBlockIOAccounting=yes
|
|
|
|
- path: /etc/etcd/etcd.env
|
|
|
|
mode: 0644
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
ETCD_NAME=${etcd_name}
|
|
|
|
ETCD_DATA_DIR=/var/lib/etcd
|
|
|
|
ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379
|
|
|
|
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380
|
|
|
|
ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
|
|
|
|
ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
|
|
|
|
ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
|
|
|
|
ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
|
|
|
|
ETCD_STRICT_RECONFIG_CHECK=true
|
|
|
|
ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt
|
|
|
|
ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt
|
|
|
|
ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key
|
|
|
|
ETCD_CLIENT_CERT_AUTH=true
|
|
|
|
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt
|
|
|
|
ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
|
|
|
|
ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
|
|
|
|
ETCD_PEER_CLIENT_CERT_AUTH=true
|
2022-01-12 05:40:17 +01:00
|
|
|
- path: /etc/containerd/config.toml
|
|
|
|
overwrite: true
|
|
|
|
contents:
|
|
|
|
inline: |
|
|
|
|
version = 2
|
|
|
|
root = "/var/lib/containerd"
|
|
|
|
state = "/run/containerd"
|
|
|
|
subreaper = true
|
|
|
|
oom_score = -999
|
|
|
|
[grpc]
|
|
|
|
address = "/run/containerd/containerd.sock"
|
|
|
|
uid = 0
|
|
|
|
gid = 0
|
|
|
|
[plugins."io.containerd.grpc.v1.cri"]
|
|
|
|
enable_selinux = true
|
|
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
|
|
|
|
runtime_type = "io.containerd.runc.v2"
|
|
|
|
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
|
|
|
|
SystemdCgroup = true
|
2020-04-13 00:49:48 +02:00
|
|
|
passwd:
|
|
|
|
users:
|
|
|
|
- name: core
|
|
|
|
ssh_authorized_keys:
|
|
|
|
- ${ssh_authorized_key}
|
|
|
|
|