Kubernetes the hard way

## RUN ON EACH MASTER.
## REQUIRED VARS: 
export BASE_DOMAIN=dobry-kot.ru
export CLUSTER_NAME=example
export BASE_CLUSTER_DOMAIN=${CLUSTER_NAME}.${BASE_DOMAIN}

# Порты для ETCD
export ETCD_SERVER_PORT="2379"
export ETCD_PEER_PORT="2380"
export ETCD_METRICS_PORT="2381"

# Порты для KUBERNETES
export KUBE_APISERVER_PORT="6443"
export KUBE_CONTROLLER_MANAGER_PORT="10257"
export KUBE_SCHEDULER_PORT="10259"

# Установите значение 1, 3, 5
export MASTER_COUNT=1

# Для Kube-apiserver 
export ETCD_SERVERS=$(echo \
$(for INDEX in `seq 0 $(($MASTER_COUNT-1))`; \
do \
echo https://master-${INDEX}.${BASE_CLUSTER_DOMAIN}:${ETCD_SERVER_PORT} ; \
done) | 
sed "s/,//" | 
sed "s/ /,/g")

# Для формирования ETCD кластера
export ETCD_INITIAL_CLUSTER=$(echo \
$(for INDEX in `seq 0 $(($MASTER_COUNT-1))`; \
do \
echo master-${INDEX}.${BASE_CLUSTER_DOMAIN}=https://master-${INDEX}.${BASE_CLUSTER_DOMAIN}:${ETCD_PEER_PORT} ; \
done) | 
sed "s/,//" | 
sed "s/ /,/g")


export KUBERNETES_VERSION="v1.23.12"
export ETCD_VERSION="3.5.3-0"
export ETCD_TOOL_VERSION="v3.5.5"
export RUNC_VERSION="v1.1.3"
export CONTAINERD_VERSION="1.6.8"
export CRICTL_VERSION=$(echo $KUBERNETES_VERSION | 
sed -r 's/^v([0-9]*).([0-9]*).([0-9]*)/v\1.\2.0/')

export BASE_K8S_PATH="/etc/kubernetes"

export SERVICE_CIDR="29.64.0.0/16"
# Не обижайтесь - regexp сами напишите)
export SERVICE_DNS="29.64.0.10"

export VAULT_MASTER_TOKEN="hvs.vy0dqWuHkJpiwtYhw4yPT6cC"
export VAULT_SERVER="http://193.32.219.99:9200/"

export VAULT_MASTER_TOKEN="root"
export VAULT_SERVER="http://master-0.${CLUSTER_NAME}.${BASE_DOMAIN}:9200/"

## RUN ON EACH MASTER.
wget -O /usr/bin/key-keeper   "https://storage.yandexcloud.net/m.images/key-keeper-T2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=YCAJEhOlYpv1GRY7hghCojNX5%2F20221020%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20221020T123413Z&X-Amz-Expires=2592000&X-Amz-Signature=138701723B70343E38D82791A28AD1DB87040677F7C94D83610FF26ED9AF1954&X-Amz-SignedHeaders=host"
wget -O /usr/bin/kubectl       https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
wget -O /usr/bin/kubelet       https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubelet
wget -O /usr/bin/kubeadm       https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubeadm
wget -O /usr/bin/runc          https://github.com/opencontainers/runc/releases/download/${RUNC_VERSION}/runc.amd64
wget -O /tmp/etcd.tar.gz       https://github.com/etcd-io/etcd/releases/download/${ETCD_TOOL_VERSION}/etcd-${ETCD_TOOL_VERSION}-linux-amd64.tar.gz
wget -O /tmp/containerd.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz
wget -O /tmp/crictl.tar.gz     https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz

chmod +x /usr/bin/key-keeper 
chmod +x /usr/bin/kubelet 
chmod +x /usr/bin/kubectl 
chmod +x /usr/bin/kubeadm
chmod +x /usr/bin/runc

mkdir -p /tmp/containerd
mkdir -p /tmp/etcd

tar -C "/tmp/etcd"        -xvf /tmp/etcd.tar.gz
tar -C "/tmp/containerd"  -xvf /tmp/containerd.tar.gz
tar -C "/usr/bin"         -xvf /tmp/crictl.tar.gz

cp /tmp/etcd/etcd*/etcdctl /usr/bin/
cp /tmp/containerd/bin/*   /usr/bin/

## RUN ON EACH MASTER.
## SETUP SERVICE FOR CONTAINERD

cat <<EOF > /etc/systemd/system/containerd.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target

[Service]
ExecStartPre=/sbin/modprobe overlay
ExecStart=/usr/bin/containerd
Restart=always
RestartSec=5
Delegate=yes
KillMode=process
OOMScoreAdjust=-999
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity

[Install]
WantedBy=multi-user.target
EOF

## RUN ON EACH MASTER.
## SETUP SERVICE FOR KEY-KEEPER
cat <<EOF > /etc/systemd/system/key-keeper.service
[Unit]
Description=key-keeper-agent

Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/key-keeper -config-dir ${BASE_K8S_PATH}/pki -config-regexp .*vault-config 

Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

## RUN ON EACH MASTER.
## SETUP SERVICE FOR KUBELET
cat <<EOF > /etc/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/home/
Wants=network-online.target
After=network-online.target


[Service]
ExecStart=/usr/bin/kubelet

Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

## RUN ON EACH MASTER.
## SETUP SERVICE-CONFIG FOR KUBELET

mkdir -p /etc/systemd/system/kubelet.service.d

cat <<EOF > /etc/systemd/system/kubelet.service.d/10-fraima.conf
[Service]
EnvironmentFile=-${BASE_K8S_PATH}/kubelet/service/kubelet-args.env

ExecStart=
ExecStart=/usr/bin/kubelet \
\$KUBELET_HOSTNAME \
\$KUBELET_CNI_ARGS \
\$KUBELET_RUNTIME_ARGS \
\$KUBELET_AUTH_ARGS \
\$KUBELET_CONFIGS_ARGS \
\$KUBELET_BASIC_ARGS \
\$KUBELET_KUBECONFIG_ARGS
EOF

## RUN ON EACH MASTER.
## SETUP SERVICE-CONFIG FOR KUBELET

mkdir -p  ${BASE_K8S_PATH}/kubelet/service/

cat <<EOF > ${BASE_K8S_PATH}/kubelet/service/kubelet-args.env
KUBELET_HOSTNAME=""
KUBELET_BASIC_ARGS="
    --register-node=true
    --cloud-provider=external
    --image-pull-progress-deadline=2m
    --feature-gates=RotateKubeletServerCertificate=true
    --cert-dir=/etc/kubernetes/pki/certs/kubelet
    --authorization-mode=Webhook
    --v=2
"
KUBELET_AUTH_ARGS="
    --anonymous-auth="false"
"
KUBELET_CNI_ARGS="
    --cni-bin-dir=/opt/cni/bin
    --cni-conf-dir=/etc/cni/net.d
    --network-plugin=cni
"
KUBELET_CONFIGS_ARGS="
    --config=${BASE_K8S_PATH}/kubelet/config.yaml
    --root-dir=/var/lib/kubelet
    --register-node=true
    --image-pull-progress-deadline=2m
    --v=2
"
KUBELET_KUBECONFIG_ARGS="
    --kubeconfig=${BASE_K8S_PATH}/kubelet/kubeconfig
"
KUBELET_RUNTIME_ARGS="
    --container-runtime=remote
    --container-runtime-endpoint=/run/containerd/containerd.sock
    --pod-infra-container-image=k8s.gcr.io/pause:3.6
"
EOF

** Please note that if you will deploy K8S in the cloud in the future and integrate it with it, then set –cloud-provider=external

*** A useful feature may be the automatic labeling of the node when registering in the cluster
--node-labels=node.kubernetes.io/master,foo=bar

Below is a list of available system labels that you can change:
kubelet.kubernetes.io
node.kubernetes.io
beta.kubernetes.io/arch,
beta.kubernetes.io/instance-type,
beta.kubernetes.io/os,
failure-domain.beta.kubernetes.io/region,
failure-domain.beta.kubernetes.io/zone,
kubernetes.io/arch,
kubernetes.io/hostname,
kubernetes.io/os,
node.kubernetes.io/instance-type,
topology.kubernetes.io/region,
topology.kubernetes.io/zone

For example, you cannot set a system label that is not from the list:
--node-labels=node-role.kubernetes.io/master

## RUN ON MASTER-0.
export VAULT_VERSION="1.12.1"
export VAULT_ADDR=${VAULT_SERVER}
export VAULT_TOKEN=${VAULT_MASTER_TOKEN}

wget -O /tmp/vault_${VAULT_VERSION}_linux_amd64.zip https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip
unzip /tmp/vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/bin

## RUN ON MASTER-0.
cat <<EOF > /etc/systemd/system/vault.service
[Unit]
Description=Vault secret management tool
After=consul.service


[Service]
PermissionsStartOnly=true
ExecStart=/usr/bin/vault server -log-level=debug -dev -dev-root-token-id="${VAULT_MASTER_TOKEN}" -dev-listen-address=0.0.0.0:9200
Restart=on-failure
LimitMEMLOCK=infinity

[Install]
WantedBy=multi-user.target
EOF

## RUN ON MASTER-0.
#enable Vault PKI secret engine 
vault secrets enable -path=pki-root pki

#set default ttl
vault secrets tune -max-lease-ttl=87600h pki-root

#generate root CA
vault write -format=json pki-root/root/generate/internal \
common_name="ROOT PKI" ttl=8760h

*Please note that if you are located in Russia, you will have problems accessing Vault and Terraform downloads.

**pki-root/root/generate/internal – Indicates that a CA will be formed, and only the public key will arrive in response, the private key will be closed.

*** pki-root – the base name of the safe for Root-CA, the change is made through the customization of the terraform module, which we will discuss below.

**** This vault installation is deployed as an overview and cannot be used for production workload.

## RUN ON MASTER-0.
export TERRAFORM_VERSION="1.3.4"

wget -O /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip

unzip /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/bin

## RUN ON MASTER-0.
mkdir terraform

cat <<EOF > terraform/main.tf
terraform {
  required_version = ">= 0.13"

}

provider "vault" {
    
    address = "http://127.0.0.1:9200/"
    token = "${VAULT_MASTER_TOKEN}"
}


variable "master-instance-count" {
  type = number
  default = 1
}

variable "base_domain" {
  type = string
  default = "${BASE_DOMAIN}"
}

variable "cluster_name" {
  type = string
  default = "${CLUSTER_NAME}"
}

variable "vault_server" {
  type = string
  default = "http://master-0.${BASE_CLUSTER_DOMAIN}:9200/"
}

# Данный модуль генерит весь набор переменных, 
# который потребуется в следующих статьях и модулях.
module "k8s-global-vars" {
    source = "git::https://github.com/fraima/kubernetes.git//modules/k8s-config-vars"
    cluster_name          = var.cluster_name
    base_domain           = var.base_domain
    master_instance_count = var.master-instance-count
    vault_server          = var.vault_server
}

# Тут происходит вся магия с Vault.
module "k8s-vault" {
    source = "git::https://github.com/fraima/kubernetes.git//modules/k8s-vault"
    k8s_global_vars   = module.k8s-global-vars
}
EOF

cd terraform 
terraform init --upgrade
terraform plan
terraform apply

## RUN ON EACH MASTER.
# Для каждой ноды свое имя!!!!
export MASTER_NAME="master-0"

In the first part of the config, specify the name of the node, all other variables are specified above.

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}/pki/

cat <<EOF > ${BASE_K8S_PATH}/pki/vault-config
---
issuers:

  - name: kube-apiserver-sa
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-apiserver-sa-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-sa/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-sa/role-id
      resource:
        kv:
          path: clusters/${CLUSTER_NAME}/kv
      timeout: 15s

  - name: etcd-ca
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: etcd-ca-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/etcd-ca/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/etcd-ca/role-id
      resource:
        CAPath: "clusters/${CLUSTER_NAME}/pki/etcd"
        rootCAPath: "clusters/${CLUSTER_NAME}/pki/root"
      timeout: 15s

  - name: etcd-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: etcd-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/etcd-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/etcd-client/role-id
      resource:
        role: etcd-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/etcd"
      timeout: 15s

  - name: etcd-peer
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: etcd-peer-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/etcd-peer/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/etcd-peer/role-id
      resource:
        role: etcd-peer
        CAPath: "clusters/${CLUSTER_NAME}/pki/etcd"
      timeout: 15s

  - name: front-proxy-ca
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: front-proxy-ca-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/front-proxy-ca/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/front-proxy-ca/role-id
      resource:
        CAPath: "clusters/${CLUSTER_NAME}/pki/front-proxy"
        rootCAPath: "clusters/${CLUSTER_NAME}/pki/root"
      timeout: 15s

  - name: front-proxy-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: front-proxy-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/front-proxy-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/front-proxy-client/role-id
      resource:
        role: front-proxy-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/front-proxy"
      timeout: 15s

  - name: kubernetes-ca
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kubernetes-ca-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kubernetes-ca/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kubernetes-ca/role-id
      resource:
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
        rootCAPath: "clusters/${CLUSTER_NAME}/pki/root"
      timeout: 15s

  - name: kube-apiserver
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-apiserver-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver/role-id
      resource:
        role: kube-apiserver
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-apiserver-cluster-admin-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-apiserver-cluster-admin-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-cluster-admin-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-cluster-admin-client/role-id
      resource:
        role: kube-apiserver-cluster-admin-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-apiserver-kubelet-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-apiserver-kubelet-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-kubelet-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-apiserver-kubelet-client/role-id
      resource:
        role: kube-apiserver-kubelet-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-controller-manager-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-controller-manager-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-controller-manager-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-controller-manager-client/role-id
      resource:
        role: kube-controller-manager-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-controller-manager-server
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-controller-manager-server-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-controller-manager-server/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-controller-manager-server/role-id
      resource:
        role: kube-controller-manager-server
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-scheduler-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-scheduler-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-scheduler-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-scheduler-client/role-id
      resource:
        role: kube-scheduler-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kube-scheduler-server
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kube-scheduler-server-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kube-scheduler-server/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kube-scheduler-server/role-id
      resource:
        role: kube-scheduler-server
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kubeadm-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kubeadm-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kubeadm-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kubeadm-client/role-id
      resource:
        role: kubeadm-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kubelet-client
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kubelet-client-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kubelet-client/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kubelet-client/role-id
      resource:
        role: kubelet-client
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s

  - name: kubelet-server
    vault:
      server: ${VAULT_SERVER}
      auth:
        caBundle: 
        tlsInsecure: true
        bootstrap:
          file: /var/lib/key-keeper/bootstrap.token
        appRole:
          name: kubelet-server-${MASTER_NAME}
          path: "clusters/${CLUSTER_NAME}/approle"
          secretIDLocalPath: /var/lib/key-keeper/vault/kubelet-server/secret-id
          roleIDLocalPath: /var/lib/key-keeper/vault/kubelet-server/role-id
      resource:
        role: kubelet-server
        CAPath: "clusters/${CLUSTER_NAME}/pki/kubernetes"
      timeout: 15s
EOF

## RUN ON EACH MASTER.
cat <<EOF >> ${BASE_K8S_PATH}/pki/vault-config
certificates:

  - name: etcd-ca
    issuerRef:
      name: etcd-ca
    isCa: true
    ca:
      exportedKey: false
      generate: false
    hostPath: "${BASE_K8S_PATH}/pki/ca"

  - name: kube-apiserver-etcd-client
    issuerRef:
      name: etcd-client
    spec:
      subject:
        commonName: "system:kube-apiserver-etcd-client"
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-apiserver"
    withUpdate: true

  - name: etcd-peer
    issuerRef:
      name: etcd-peer
    spec:
      subject:
        commonName: "system:etcd-peer"
      usage:
        - server auth
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        interfaces:
          - lo
          - eth*
      ttl: 10m
      hostnames:
        - localhost
        - $HOSTNAME
        - "${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}"
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/etcd"
    withUpdate: true

  - name: etcd-server
    issuerRef:
      name: etcd-peer
    spec:
      subject:
        commonName: "system:etcd-server"
      usage:
        - server auth
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        static:
          - 127.0.1.1
        interfaces:
          - lo
          - eth*
      ttl: 10m
      hostnames:
        - localhost
        - $HOSTNAME
        - "${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}"
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/etcd"
    withUpdate: true

  - name: front-proxy-ca
    issuerRef:
      name: front-proxy-ca
    isCa: true
    ca:
      exportedKey: false
      generate: false
    hostPath: "${BASE_K8S_PATH}/pki/ca"

  - name: front-proxy-client
    issuerRef:
      name: front-proxy-client
    spec:
      subject:
        commonName: "custom:kube-apiserver-front-proxy-client"
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-apiserver"
    withUpdate: true

  - name: kubernetes-ca
    issuerRef:
      name: kubernetes-ca
    isCa: true
    ca:
      exportedKey: false
      generate: false
    hostPath: "${BASE_K8S_PATH}/pki/ca"

  - name: kube-apiserver
    issuerRef:
      name: kube-apiserver
    spec:
      subject:
        commonName: "custom:kube-apiserver"
      usage:
        - server auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        static:
          - 29.64.0.1
        interfaces:
          - lo
          - eth*
        dnsLookup:
          - api.${BASE_CLUSTER_DOMAIN}
      ttl: 10m
      hostnames:
        - localhost
        - kubernetes
        - kubernetes.default
        - kubernetes.default.svc
        - kubernetes.default.svc.cluster
        - kubernetes.default.svc.cluster.local
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-apiserver"
    withUpdate: true

  - name: kube-apiserver-kubelet-client
    issuerRef:
      name: kube-apiserver-kubelet-client
    spec:
      subject:
        commonName: "custom:kube-apiserver-kubelet-client"
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-apiserver"
    withUpdate: true

  - name: kube-controller-manager-client
    issuerRef:
      name: kube-controller-manager-client
    spec:
      subject:
        commonName: "system:kube-controller-manager"
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-controller-manager"
    withUpdate: true

  - name: kube-controller-manager-server
    issuerRef:
      name: kube-controller-manager-server
    spec:
      subject:
        commonName: "custom:kube-controller-manager"
      usage:
        - server auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        interfaces:
          - lo
          - eth*
      ttl: 10m
      hostnames:
        - localhost
        - kube-controller-manager.default
        - kube-controller-manager.default.svc
        - kube-controller-manager.default.svc.cluster
        - kube-controller-manager.default.svc.cluster.local
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-controller-manager"
    withUpdate: true

  - name: kube-scheduler-client
    issuerRef:
      name: kube-scheduler-client
    spec:
      subject:
        commonName: "system:kube-scheduler"
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-scheduler"
    withUpdate: true

  - name: kube-scheduler-server
    issuerRef:
      name: kube-scheduler-server
    spec:
      subject:
        commonName: "custom:kube-scheduler"
      usage:
        - server auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        interfaces:
          - lo
          - eth*
      ttl: 10m
      hostnames:
        - localhost
        - kube-scheduler.default
        - kube-scheduler.default.svc
        - kube-scheduler.default.svc.cluster
        - kube-scheduler.default.svc.cluster.local
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-scheduler"
    withUpdate: true

  - name: kubeadm-client
    issuerRef:
      name: kubeadm-client
    spec:
      subject:
        commonName: "custom:kubeadm-client"
        organizationalUnit:
          - system:masters
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kube-apiserver"
    withUpdate: true

  - name: kubelet-client
    issuerRef:
      name: kubelet-client
    spec:
      subject:
        commonName: "system:node:${MASTER_NAME}-${CLUSTER_NAME}"
        organization:
          - system:nodes
      usage:
        - client auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ttl: 10m
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kubelet"
    withUpdate: true

  - name: kubelet-server
    issuerRef:
      name: kubelet-server
    spec:
      subject:
        commonName: "system:node:${MASTER_NAME}-${CLUSTER_NAME}"
      usage:
        - server auth
      privateKey:
        algorithm: "RSA"
        encoding: "PKCS1"
        size: 4096
      ipAddresses:
        interfaces:
          - lo
          - eth*
      ttl: 10m
      hostnames:
        - localhost
        - $HOSTNAME
        - "${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}"
    renewBefore: 7m
    hostPath: "${BASE_K8S_PATH}/pki/certs/kubelet"
    withUpdate: true

secrets:
  - name: kube-apiserver-sa
    issuerRef:
      name: kube-apiserver-sa
    key: private  
    hostPath: ${BASE_K8S_PATH}/pki/certs/kube-apiserver/kube-apiserver-sa.pem

  - name: kube-apiserver-sa
    issuerRef:
      name: kube-apiserver-sa
    key: public  
    hostPath: ${BASE_K8S_PATH}/pki/certs/kube-apiserver/kube-apiserver-sa.pub
EOF

## RUN ON EACH MASTER.
mkdir -p /var/lib/key-keeper/

cat <<EOF > /var/lib/key-keeper/bootstrap.token
${VAULT_MASTER_TOKEN}
EOF

** Do not be surprised that in this configuration file the master key from the Vault Server, as I said earlier, is a simplified version of the configuration.

*** If you study our Vault module for Terraform a little deeper, you will understand that temporary tokens are created there, which must be specified in bootstrap in the key-keeper config. Each issuer has its own token. Example -> https://github.com/fraima/kubernetes/blob/f0e4c7bc8f8d2695c419b17fec4bacc2dd7c5f18/modules/k8s-templates/cloud-init/templates/cloud-init-kubeadm-master.tftpl#L115

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}/kubelet

cat <<EOF >> ${BASE_K8S_PATH}/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: "${BASE_K8S_PATH}/pki/ca/kubernetes-ca.pem"

tlsCertFile: ${BASE_K8S_PATH}/pki/certs/kubelet/kubelet-server.pem
tlsPrivateKeyFile: ${BASE_K8S_PATH}/pki/certs/kubelet/kubelet-server-key.pem

authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
  - "${SERVICE_DNS}"
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 1s
nodeStatusUpdateFrequency: 1s
resolvConf: /run/systemd/resolve/resolv.conf
rotateCertificates: false
runtimeRequestTimeout: 0s
serverTLSBootstrap: true
shutdownGracePeriod: 15s
shutdownGracePeriodCriticalPods: 5s
staticPodPath: "${BASE_K8S_PATH}/manifests"
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
containerLogMaxSize: 50Mi
maxPods: 250
kubeAPIQPS: 50
kubeAPIBurst: 100
podPidsLimit: 4096
serializeImagePulls: false
systemReserved:
  ephemeral-storage: 1Gi
featureGates:
  APIPriorityAndFairness: true
  DownwardAPIHugePages: true
  PodSecurity: true
  CSIMigrationAWS: false
  CSIMigrationAzureFile: false
  CSIMigrationGCE: false
  CSIMigrationvSphere: false
rotateCertificates: false
serverTLSBootstrap: true
tlsMinVersion: VersionTLS12
tlsCipherSuites:
  - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
  - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
  - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
  - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
  - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
  - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
allowedUnsafeSysctls:
  - "net.core.somaxconn"
evictionSoft: 
  memory.available: 3Gi 
  nodefs.available: 25%
  nodefs.inodesFree: 15%
  imagefs.available: 30%
  imagefs.inodesFree: 25%
evictionSoftGracePeriod:  
  memory.available: 2m30s
  nodefs.available: 2m30s
  nodefs.inodesFree: 2m30s
  imagefs.available: 2m30s
  imagefs.inodesFree: 2m30s
evictionHard:
  memory.available: 2Gi
  nodefs.available: 20%
  nodefs.inodesFree: 10%
  imagefs.available: 25%
  imagefs.inodesFree: 15%
evictionPressureTransitionPeriod: 5s 
imageMinimumGCAge: 12h 
imageGCHighThresholdPercent: 55
imageGCLowThresholdPercent: 50
EOF

** clusterDNS – it’s easy to get burned if you specify an incorrect value.

*** resolvConf – in Centos, Rhel, Almalinux can swear on the path, solved by the commands:

systemctl daemon-reload
systemctl enable systemd-resolved.service
systemctl start systemd-resolved.service

Documentation describing the problem:
https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/#known-issues

## RUN ON EACH MASTER.
cat <<EOF >> /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

## RUN ON EACH MASTER.
cat <<EOF >> /etc/sysctl.d/99-network.conf
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
EOF

sysctl --system

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}

cat <<EOF >> ${BASE_K8S_PATH}/admin.conf
---
apiVersion: v1
clusters:
- cluster:
    certificate-authority: ${BASE_K8S_PATH}/pki/ca/kubernetes-ca.pem
    server: https://127.0.0.1:${KUBE_APISERVER_PORT}
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    namespace: default
    user: kubeadm
  name: kubeadm@kubernetes
current-context: kubeadm@kubernetes
kind: Config
preferences: {}
users:
- name: kubeadm
  user:
    client-certificate: ${BASE_K8S_PATH}/pki/certs/kube-apiserver/kubeadm-client.pem
    client-key: ${BASE_K8S_PATH}/pki/certs/kube-apiserver/kubeadm-client-key.pem
EOF

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}/kube-scheduler/

cat <<EOF >> ${BASE_K8S_PATH}/kube-scheduler/kubeconfig
---
apiVersion: v1
clusters:
- cluster:
    certificate-authority: ${BASE_K8S_PATH}/pki/ca/kubernetes-ca.pem
    server: https://127.0.0.1:${KUBE_APISERVER_PORT}
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    namespace: default
    user: kube-scheduler
  name: kube-scheduler@kubernetes
current-context: kube-scheduler@kubernetes
kind: Config
preferences: {}
users:
- name: kube-scheduler
  user:
    client-certificate: ${BASE_K8S_PATH}/pki/certs/kube-scheduler/kube-scheduler-client.pem
    client-key: ${BASE_K8S_PATH}/pki/certs/kube-scheduler/kube-scheduler-client-key.pem
EOF

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}/kube-controller-manager

cat <<EOF >> ${BASE_K8S_PATH}/kube-controller-manager/kubeconfig
---
apiVersion: v1
clusters:
- cluster:
    certificate-authority: ${BASE_K8S_PATH}/pki/ca/kubernetes-ca.pem
    server: https://127.0.0.1:${KUBE_APISERVER_PORT}
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    namespace: default
    user: kube-controller-manager
  name: kube-controller-manager@kubernetes
current-context: kube-controller-manager@kubernetes
kind: Config
preferences: {}
users:
- name: kube-controller-manager
  user:
    client-certificate: ${BASE_K8S_PATH}/pki/certs/kube-controller-manager/kube-controller-manager-client.pem
    client-key: ${BASE_K8S_PATH}/pki/certs/kube-controller-manager/kube-controller-manager-client-key.pem
EOF

## RUN ON EACH MASTER.
mkdir -p ${BASE_K8S_PATH}/kubelet

cat <<EOF >> ${BASE_K8S_PATH}/kubelet/kubeconfig
---
apiVersion: v1
clusters:
- cluster:
    certificate-authority: ${BASE_K8S_PATH}/pki/ca/kubernetes-ca.pem
    server: https://127.0.0.1:${KUBE_APISERVER_PORT}
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    namespace: default
    user: kubelet
  name: kubelet@kubernetes
current-context: kubelet@kubernetes
kind: Config
preferences: {}
users:
- name: kubelet
  user:
    client-certificate: ${BASE_K8S_PATH}/pki/certs/kubelet/kubelet-client.pem
    client-key: ${BASE_K8S_PATH}/pki/certs/kubelet/kubelet-client-key.pem
EOF

## RUN ON EACH MASTER.
export ADVERTISE_ADDRESS=$(ip route get 1.1.1.1 | grep -oP 'src \K\S+')

cat <<EOF > /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: ${ADVERTISE_ADDRESS}:${KUBE_APISERVER_PORT}
  creationTimestamp: null
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --advertise-address=${ADVERTISE_ADDRESS}
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --bind-address=0.0.0.0
    - --client-ca-file=/etc/kubernetes/pki/ca/kubernetes-ca.pem
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/ca/etcd-ca.pem
    - --etcd-certfile=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client.pem
    - --etcd-keyfile=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client-key.pem
    - --etcd-servers=${ETCD_SERVERS}
    - --kubelet-client-certificate=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-kubelet-client.pem
    - --kubelet-client-key=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-kubelet-client-key.pem
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/certs/kube-apiserver/front-proxy-client.pem
    - --proxy-client-key-file=/etc/kubernetes/pki/certs/kube-apiserver/front-proxy-client-key.pem
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/ca/front-proxy-ca.pem
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=${KUBE_APISERVER_PORT}
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-sa.pub
    - --service-account-signing-key-file=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-sa.pem
    - --service-cluster-ip-range=${SERVICE_CIDR}
    - --tls-cert-file=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver.pem
    - --tls-private-key-file=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-key.pem
    image: k8s.gcr.io/kube-apiserver:${KUBERNETES_VERSION}
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /livez
        port: ${KUBE_APISERVER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-apiserver
    readinessProbe:
      failureThreshold: 3
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /readyz
        port: ${KUBE_APISERVER_PORT}
        scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 250m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /livez
        port: ${KUBE_APISERVER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
      readOnly: true
    - mountPath: /var/log/kubernetes/audit/
      name: k8s-audit
    - mountPath: /etc/kubernetes/pki/ca
      name: k8s-ca
      readOnly: true
    - mountPath: /etc/kubernetes/pki/certs
      name: k8s-certs
      readOnly: true
    - mountPath: /etc/kubernetes/kube-apiserver
      name: k8s-kube-apiserver-configs
      readOnly: true
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
      readOnly: true
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/ca-certificates
      type: DirectoryOrCreate
    name: etc-ca-certificates
  - hostPath:
      path: /var/log/kubernetes/audit/
      type: DirectoryOrCreate
    name: k8s-audit
  - hostPath:
      path: /etc/kubernetes/pki/ca
      type: DirectoryOrCreate
    name: k8s-ca
  - hostPath:
      path: /etc/kubernetes/pki/certs
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /etc/kubernetes/kube-apiserver
      type: DirectoryOrCreate
    name: k8s-kube-apiserver-configs
  - hostPath:
      path: /usr/local/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-local-share-ca-certificates
  - hostPath:
      path: /usr/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-share-ca-certificates
status: {}
EOF

** Please note that the variable ADVERTISE_ADDRESS requires the Internet, if it is not there, just specify the IP ADDRESS of the node.

## RUN ON EACH MASTER.
cat <<EOF > /etc/kubernetes/manifests/kube-controller-manager.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-controller-manager
    tier: control-plane
  name: kube-controller-manager
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-controller-manager
    - --authentication-kubeconfig=/etc/kubernetes/kube-controller-manager/kubeconfig
    - --authorization-always-allow-paths=/healthz,/metrics
    - --authorization-kubeconfig=/etc/kubernetes/kube-controller-manager/kubeconfig
    - --bind-address=${ADVERTISE_ADDRESS}
    - --client-ca-file=/etc/kubernetes/pki/ca/kubernetes-ca.pem
    - --cluster-cidr=${SERVICE_CIDR}
    - --cluster-name=kubernetes
    - --cluster-signing-cert-file=/etc/kubernetes/pki/ca/kubernetes-ca.pem
    - --cluster-signing-key-file=
    - --controllers=*,bootstrapsigner,tokencleaner
    - --kubeconfig=/etc/kubernetes/kube-controller-manager/kubeconfig
    - --leader-elect=true
    - --requestheader-client-ca-file=/etc/kubernetes/pki/ca/front-proxy-ca.pem
    - --root-ca-file=/etc/kubernetes/pki/ca/kubernetes-ca.pem
    - --secure-port=${KUBE_CONTROLLER_MANAGER_PORT}
    - --service-account-private-key-file=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-sa.pem
    - --tls-cert-file=/etc/kubernetes/pki/certs/kube-controller-manager/kube-controller-manager-server.pem
    - --tls-private-key-file=/etc/kubernetes/pki/certs/kube-controller-manager/kube-controller-manager-server-key.pem
    - --use-service-account-credentials=true
    image: k8s.gcr.io/kube-controller-manager:${KUBERNETES_VERSION}
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /healthz
        port: ${KUBE_CONTROLLER_MANAGER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-controller-manager
    resources:
      requests:
        cpu: 200m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /healthz
        port: ${KUBE_CONTROLLER_MANAGER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
    - mountPath: /etc/ca-certificates
      name: etc-ca-certificates
      readOnly: true
    - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
      name: flexvolume-dir
    - mountPath: /etc/kubernetes/pki/ca
      name: k8s-ca
      readOnly: true
    - mountPath: /etc/kubernetes/pki/certs
      name: k8s-certs
      readOnly: true
    - mountPath: /etc/kubernetes/kube-controller-manager
      name: k8s-kube-controller-manager-configs
      readOnly: true
    - mountPath: /usr/local/share/ca-certificates
      name: usr-local-share-ca-certificates
      readOnly: true
    - mountPath: /usr/share/ca-certificates
      name: usr-share-ca-certificates
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
    name: ca-certs
  - hostPath:
      path: /etc/ca-certificates
      type: DirectoryOrCreate
    name: etc-ca-certificates
  - hostPath:
      path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
      type: DirectoryOrCreate
    name: flexvolume-dir
  - hostPath:
      path: /etc/kubernetes/pki/ca
      type: DirectoryOrCreate
    name: k8s-ca
  - hostPath:
      path: /etc/kubernetes/pki/certs
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /etc/kubernetes/kube-controller-manager
      type: DirectoryOrCreate
    name: k8s-kube-controller-manager-configs
  - hostPath:
      path: /usr/local/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-local-share-ca-certificates
  - hostPath:
      path: /usr/share/ca-certificates
      type: DirectoryOrCreate
    name: usr-share-ca-certificates
status: {}
EOF

## RUN ON EACH MASTER.
cat <<EOF > /etc/kubernetes/manifests/kube-scheduler.yaml
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --authentication-kubeconfig=/etc/kubernetes/kube-scheduler/kubeconfig
    - --authorization-kubeconfig=/etc/kubernetes/kube-scheduler/kubeconfig
    - --bind-address=${ADVERTISE_ADDRESS}
    - --kubeconfig=/etc/kubernetes/kube-scheduler/kubeconfig
    - --leader-elect=true
    - --secure-port=${KUBE_SCHEDULER_PORT}
    - --tls-cert-file=/etc/kubernetes/pki/certs/kube-scheduler/kube-scheduler-server.pem
    - --tls-private-key-file=/etc/kubernetes/pki/certs/kube-scheduler/kube-scheduler-server-key.pem
    image: k8s.gcr.io/kube-scheduler:${KUBERNETES_VERSION}
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /healthz
        port: ${KUBE_SCHEDULER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    name: kube-scheduler
    resources:
      requests:
        cpu: 100m
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: ${ADVERTISE_ADDRESS}
        path: /healthz
        port: ${KUBE_SCHEDULER_PORT}
        scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    volumeMounts:
    - mountPath: /etc/kubernetes/pki/ca
      name: k8s-ca
      readOnly: true
    - mountPath: /etc/kubernetes/pki/certs
      name: k8s-certs
      readOnly: true
    - mountPath: /etc/kubernetes/kube-scheduler
      name: k8s-kube-scheduler-configs
      readOnly: true
  hostNetwork: true
  priorityClassName: system-node-critical
  securityContext:
    seccompProfile:
      type: RuntimeDefault
  volumes:
  - hostPath:
      path: /etc/kubernetes/pki/ca
      type: DirectoryOrCreate
    name: k8s-ca
  - hostPath:
      path: /etc/kubernetes/pki/certs
      type: DirectoryOrCreate
    name: k8s-certs
  - hostPath:
      path: /etc/kubernetes/kube-scheduler
      type: DirectoryOrCreate
    name: k8s-kube-scheduler-configs
status: {}
EOF

## RUN ON EACH MASTER.
cat <<EOF > /etc/kubernetes/manifests/etcd.yaml
---
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: etcd
    tier: control-plane
  name: etcd
  namespace: kube-system
spec:
  containers:
  - name: etcd
    command:
      - etcd
    args:
      - --name=${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}
      - --initial-cluster=${ETCD_INITIAL_CLUSTER}
      - --initial-advertise-peer-urls=https://${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}:${ETCD_PEER_PORT}
      - --advertise-client-urls=https://${MASTER_NAME}.${BASE_CLUSTER_DOMAIN}:${ETCD_SERVER_PORT}
      - --peer-trusted-ca-file=/etc/kubernetes/pki/ca/etcd-ca.pem
      - --trusted-ca-file=/etc/kubernetes/pki/ca/etcd-ca.pem
      - --peer-cert-file=/etc/kubernetes/pki/certs/etcd/etcd-peer.pem
      - --peer-key-file=/etc/kubernetes/pki/certs/etcd/etcd-peer-key.pem
      - --cert-file=/etc/kubernetes/pki/certs/etcd/etcd-server.pem
      - --key-file=/etc/kubernetes/pki/certs/etcd/etcd-server-key.pem
      - --listen-client-urls=https://0.0.0.0:${ETCD_SERVER_PORT}
      - --listen-peer-urls=https://0.0.0.0:${ETCD_PEER_PORT}
      - --listen-metrics-urls=http://0.0.0.0:${ETCD_METRICS_PORT}
      - --initial-cluster-token=etcd
      - --initial-cluster-state=new
      - --data-dir=/var/lib/etcd
      - --strict-reconfig-check=true
      - --peer-client-cert-auth=true
      - --peer-auto-tls=true
      - --client-cert-auth=true
      - --snapshot-count=10000
      - --heartbeat-interval=250
      - --election-timeout=1500
      - --quota-backend-bytes=0
      - --max-snapshots=10
      - --max-wals=10
      - --discovery-fallback=proxy
      - --auto-compaction-retention=8
      - --force-new-cluster=false
      - --enable-v2=false
      - --proxy=off
      - --proxy-failure-wait=5000
      - --proxy-refresh-interval=30000
      - --proxy-dial-timeout=1000
      - --proxy-write-timeout=5000
      - --proxy-read-timeout=0
      - --metrics=extensive
      - --logger=zap
    image: k8s.gcr.io/etcd:${ETCD_VERSION}
    imagePullPolicy: IfNotPresent
    livenessProbe:
      failureThreshold: 8
      httpGet:
        host: 127.0.0.1
        path: /health
        port: ${ETCD_METRICS_PORT}
        scheme: HTTP
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
    resources:
      requests:
        cpu: 100m
        memory: 100Mi
    startupProbe:
      failureThreshold: 24
      httpGet:
        host: 127.0.0.1
        path: /health
        port: ${ETCD_METRICS_PORT}
        scheme: HTTP
    volumeMounts:
    - mountPath: /var/lib/etcd
      name: etcd-data
    - mountPath: /etc/kubernetes/pki/certs/etcd
      name: etcd-certs
    - mountPath: /etc/kubernetes/pki/ca
      name: ca
  hostNetwork: true
  priorityClassName: system-node-critical
  securityContext:
      null
  volumes:
  - hostPath:
      path: /etc/kubernetes/pki/certs/etcd
      type: DirectoryOrCreate
    name: etcd-certs
  - hostPath:
      path: /etc/kubernetes/pki/ca
      type: DirectoryOrCreate
    name: ca
  - hostPath:
      path: /var/lib/etcd
      type: DirectoryOrCreate
    name: etcd-data
status: {}
EOF

## RUN ON EACH MASTER.
systemctl daemon-reload
systemctl enable  key-keeper.service
systemctl start   key-keeper.service
systemctl enable  kubelet.service
systemctl start   kubelet.service
systemctl enable  containerd.service
systemctl start   containerd.service
systemctl enable  systemd-resolved.service
systemctl start   systemd-resolved.service

## RUN ON EACH MASTER.
export ETCDCTL_CERT=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client.pem \
export ETCDCTL_KEY=/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client-key.pem \
export ETCDCTL_CACERT=/etc/kubernetes/pki/ca/etcd-ca.pem \

etcd_endpoints () {
export ENDPOINTS=$(echo $(ENDPOINTS=127.0.0.1:${ETCD_SERVER_PORT}
etcdctl \
--endpoints=$ENDPOINTS \
member list |
awk '{print $5}' |
sed "s/,//") | sed "s/ /,/g")
}

etcd_endpoints

estat () {
etcdctl \
--write-out=table \
--endpoints=$ENDPOINTS \
endpoint status
}

estat

It is useful to add this piece to bashrc, for convenient work, status check or etcd debugging.

cat <<EOF | kubectl apply -f -
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: custom:kube-apiserver-kubelet-client
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: User
  apiGroup: rbac.authorization.k8s.io
  name: custom:kube-apiserver-kubelet-client
EOF

** Please note that the certificate is issued with CN=custom:kube-apiserver-kubelet-client (if customization of the name is required, you need to edit in the terraform module)