apiVersion: v1
metadata:
  name: my-lamp-site-data
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: nfs-client
---
apiVersion: v1
kind: Pod
metadata:
  name: my-lamp-site
spec:
    containers:
    - name: mysql
      image: mysql
      env:
      - name: MYSQL_ROOT_PASSWORD
        value: "rootpasswd"
      volumeMounts:
      - mountPath: /var/lib/mysql
        name: site-data
        subPath: mysql
    - name: php
      image: php:7.0-apache
      volumeMounts:
      - mountPath: /var/www/html
        name: site-data
        subPath: html
    volumes:
    - name: site-data
      persistentVolumeClaim:
        claimName: my-lamp-site-data

[root@paas-m-k8s-master-1 ~]# kc exec -it my-lamp-site -c php  -- /bin/sh
# pwd
/var/www/html
# touch a.txt
# ls
a.txt
# cat > a.txt << EOF
> asdhkljfga
> EOF

[root@paas-m-k8s-nfs default-my-lamp-site-data-pvc-33ac110d-d2b3-4b91-be84-b481f1a4bd0f]# ll
总用量 4
drwxrwxrwx 2 root    root   19 12月 20 10:49 html
drwxrwxrwx 6 polkitd root 4096 12月 20 10:52 mysql
[root@paas-m-k8s-nfs default-my-lamp-site-data-pvc-33ac110d-d2b3-4b91-be84-b481f1a4bd0f]# ls html/
a.txt
[root@paas-m-k8s-nfs default-my-lamp-site-data-pvc-33ac110d-d2b3-4b91-be84-b481f1a4bd0f]# ls mysql/
auto.cnf       binlog.000002  b.txt       ca.pem   ...

apiVersion: v1
items:
- allowVolumeExpansion: true
  apiVersion: storage.k8s.io/v1
  kind: StorageClass
  metadata:
    annotations:
      storageclass.kubernetes.io/is-default-class: "true"
      storageclass.kubesphere.io/support-snapshot: "false"
      storageclass.kubesphere.io/supported_access_modes: '["ReadWriteOnce","ReadOnlyMany","ReadWriteMany"]'
    labels:
      app: nfs-provisioner
      chart: nfs-client-provisioner-1.1.2
      heritage: Tiller
      release: nfs-client-retain
    name: nfs-client-retain
  mountOptions:
  - nfsvers=3
  parameters:
    archiveOnDelete: "false"
  provisioner: cluster.local/nfs-client-nfs-client-provisioner
  reclaimPolicy: Retain
  volumeBindingMode: Immediate
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

# kubectl apply -f storageClass-retain.yaml
storageclass.storage.k8s.io/nfs-client-retain created

[root@paas-m-k8s-master-1 yaml]# kubectl get sc
NAME                           PROVISIONER                                       RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client                     cluster.local/nfs-client-nfs-client-provisioner   Delete          Immediate           true                   2y160d
nfs-client-reatain (default)   cluster.local/nfs-client-nfs-client-provisioner   Retain          Immediate           true                   106s
rook-ceph-block (default)      rook-ceph.rbd.csi.ceph.com                        Delete          Immediate           true                   2y145d

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: my-lamp-site-data
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: nfs-client-retain

apiVersion: projectcalico.org/v3
kind: IPPool
metadata:
  name: redis-ippool
spec:
  blockSize: 26
  cidr: 10.10.0.0/20
  ipipMode: Always
  natOutgoing: true

[root@paas-m-k8s-master-1 calico]# calicoctl get ippool
NAME                  CIDR            SELECTOR
default-ipv4-ippool   100.64.0.0/10   all()
redis-ippool          10.10.0.0/20    all()

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: redis-pvc-0
  namespace: redis-test
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 2Gi
  storageClassName: nfs-client
  volumeMode: Filesystem
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: redis-test
  name: redis-cluster-0
  labels:
    app: redis-cluster-0
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-cluster-0
  template:
    metadata:
      labels:
        app: redis-cluster-0
      annotations:
        "cni.projectcalico.org/ipAddrs": "[\"10.10.0.1\"]"
    spec:
      volumes:
        - name: redis-data
          persistentVolumeClaim:
            claimName: redis-pvc-0
        - name: config
          configMap:
            name: redis-config
      containers:
        - name: redis
          image: redis:5.0.5-alpine
          command:
            - "sh"
            - "-c"
            - "redis-server /data/conf/redis.conf"
          ports:
            - containerPort: 6379
              name: client
            - containerPort: 16379
              name: gossport
          resources:
            limits:
              cpu: "1"
          volumeMounts:
            - name: redis-data
              mountPath: /data/data
            - name: config
              mountPath: /data/conf

kc apply -f redis-deploy-0.yaml
kc apply -f redis-deploy-1.yaml
kc apply -f redis-deploy-2.yaml
kc apply -f redis-deploy-3.yaml
kc apply -f redis-deploy-4.yaml
kc apply -f redis-deploy-5.yaml

kubectl -n redis-test exec -it redis-cluster-0-77dbcddbb8-w29hm -- redis-cli --cluster create --cluster-replicas 1 -a 123456 10.10.0.1:6379 10.10.0.2:6379 10.10.0.3:6379 10.10.0.4:6379 10.10.0.5:6379 10.10.0.6:6379 

blockSize: 26
cidr: 10.10.0.0/20

blockSize: 21
cidr: 11.11.0.0/20

mkdir /nfs

yum -y install nfs-utils
设置开机启动
systemctl enable rpcbind.service
systemctl enable nfs-server.service

vim /etc/exports
添加如下内容
/nfs 1x.xxx.1.0/24(rw,sync,no_root_squash,no_wdelay)

systemctl start rpcbind
systemctl start nfs

systemctl start rpcbind

mount -t nfs 1x.xxx.1.164:/nfs /nfs

showmount -e 1x.xxx.1.164

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services", "endpoints"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-provisioner
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-provisioner
  apiGroup: rbac.authorization.k8s.io

kubectl apply -f rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-provisioner
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-provisioner
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-provisioner
      release: nfs-client
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-provisioner
        release: nfs-client
    spec:
      serviceAccount: nfs-provisioner
      containers:
        - name: nfs-provisioner
          image: kubesphere/nfs-client-provisioner:v3.1.0-k8s1.11
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: cluster.local/nfs-client-nfs-client-provisioner
            - name: NFS_SERVER
              value: 1x.xxx.1.164
            - name: NFS_PATH
              value: /nfs
      volumes:
        - name: nfs-client-root
          nfs:
            server: 1x.xxx.1.164
            path: /nfs

kubectl apply -f nfs-provisioner.yaml

apiVersion: v1
items:
- allowVolumeExpansion: true
  apiVersion: storage.k8s.io/v1
  kind: StorageClass
  metadata:
    annotations:
      storageclass.kubernetes.io/is-default-class: "true"
      storageclass.kubesphere.io/support-snapshot: "false"
      storageclass.kubesphere.io/supported_access_modes: '["ReadWriteOnce","ReadOnlyMany","ReadWriteMany"]'
    labels:
      app: nfs-provisioner
      chart: nfs-client-provisioner-1.1.2
      heritage: Tiller
      release: nfs-client
    name: nfs-client
  mountOptions:
  - nfsvers=3
  parameters:
    archiveOnDelete: "false"
  provisioner: cluster.local/nfs-client-nfs-client-provisioner
  reclaimPolicy: Delete
  volumeBindingMode: Immediate
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

kubectl apply -f  storageClass.yaml

[root@idp-restore-master01 sc]# kubectl get storageClass
NAME                   PROVISIONER                                       RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client (default)   cluster.local/nfs-client-nfs-client-provisioner   Delete          Immediate           true                   9s

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: my-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: nfs-client

kubectl apply -f my-pvc.yaml

kubectl get pvc

kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: willdockerhub/busybox:1.24
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1"
    volumeMounts:
      - name: nfs-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: my-pvc            

kubectl apply -f test-pod.yaml

Events:
  Type     Reason       Age                      From                           Message
  ----     ------       ----                     ----                           -------
  Warning  FailedMount  7m31s (x134 over 3h41m)  kubelet, idp-restore-master02  (combined from similar events): MountVolume.SetUp failed for volume "pvc-e88e0ca4-0047-47fa-ad2c-17499d5b44c3" : mount failed: exit status 32
  。。。
mount: wrong fs type, bad option, bad superblock on 1x.xxx.1.164:/nfs/kubesphere-system-data-redis-ha-server-0-pvc-e88e0ca4-0047-47fa-ad2c-17499d5b44c3,

sealos clean --all=true

vim /etc/hostname

systemctl stop firewalld
systemctl disable firewalld

setenforce 0
sed -i 's/^SELINUX=.*$/SELINUX=disable/' /etc/selinux/config

swapoff -a
echo "vm.swappiness=0" >> /etc/sysctl.conf
sysctl -p /etc/sysctl.conf
sed -i 's$/dev/mapper/centos-swap$#/dev/mapper/centos-swap$g' /etc/fstab
free -m

ulimit -n
1024

临时生效设置如下：
# ulimit -SHn 65535

永久生效设置，添加如下两行
# vim /etc/security/limits.conf
* soft nofile 65535 
* hard nofile 65535

先查看一下系统是否安装ntp，如下命令：
[root@k8s-master21 ~]# rpm -qa ntp

如果没有安装ntp，则如下安装：
[root@k8s-master21 ~]# yum install ntp -y

先设置好时区，如下：
[root@k8s-master21 ~]# ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
[root@k8s-master21 ~]# echo "Asia/Shanghai" > /etc/timezone

再设置阿里云的时间同步服务器，如下：
[root@k8s-master21 ~]# ntpdate time2.aliyun.com
26 Dec 15:35:57 ntpdate[7043]: step time server 203.107.6.88 offset -8.227227 sec

再将时间同步添加到系统定时任务中，如下命令后，添加 */5 * * * * ntpdate time2.aliyun.com 保存退出即可：
[root@k8s-master21 ~]# crontab -e
*/5 * * * * ntpdate time2.aliyun.com

最后，将时间同步添加到开机自启动中，打开/etc/rc.local 添加 ntpdate time2.aliyun.com 保存退出即可：
[root@k8s-master21 ~]# vim /etc/rc.local
ntpdate time2.aliyun.com

cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash  /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

yum remove docker \
			docker-client \
           docker-client-latest \
           docker-common \
           docker-latest \
           docker-latest-logrotate \
           docker-logrotate \
           docker-engine

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum install docker-ce

mkdir -p /etc/docker/

cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": ["https://6kx4zyno.mirror.aliyuncs.com"]
}
EOF

systemctl enable docker && systemctl daemon-reload &&  systemctl restart docker

cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg  https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubectl-1.22.8 kubelet-1.22.8 kubeadm-1.22.8

# 也可安装kubelet kubeadm kubectl 其他版本 但注意需要和docker版本匹配
# 查看 可安装 kubelet kubeadm kube版本
# yum list kubeadm kubelet kubectl --showduplicates|sort -r

cat >/etc/sysconfig/kubelet<<EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.2"
EOF

systemctl daemon-reload && systemctl enable --now kubelet

kubeadm config print init-defaults

vim /root/kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1x.xx.12.181
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  imagePullPolicy: IfNotPresent
  name: k8s-master1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 1x.xx.12.181:6443
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.22.0
networking:
  dnsDomain: cluster.local
  podSubnet: 192.168.0.0/16
  serviceSubnet: 1x.xx.0.0/12
scheduler: {}

kubeadm config images pull --config /root/kubeadm-config.yaml 

kubeadm init --config /root/kubeadm-config.yaml  --upload-certs

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 1x.xx.12.181:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:30e24f2bf41b30864c9a2aff54aece349f3bdff2882b9b54145cd9c7976e8ef9 \
        --control-plane --certificate-key e1fec22ea86d49cf3f74bab846075cf234fb19f4de18243bcdd220537fa4285f

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 1x.xx.12.181:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:30e24f2bf41b30864c9a2aff54aece349f3bdff2882b9b54145cd9c7976e8ef9

cd /etc/kubernetes/manifests
ls 
etcd.yaml  kube-apiserver.yaml  kube-controller-manager.yaml  kube-scheduler.yaml

cat <<EOF >> /root/.bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source /root/.bashrc

[root@k8s-master1 ~]# kubectl get svc -A
NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes   ClusterIP   1x.xx.0.1    <none>        443/TCP                  19m
kube-system   kube-dns     ClusterIP   1x.xx.0.10   <none>        53/UDP,53/TCP,9153/TCP   18m

[root@k8s-master1 ~]# kubectl get pod -A -owide
NAMESPACE     NAME                                  READY   STATUS    RESTARTS       AGE     IP              NODE          NOMINATED NODE   READINESS GATES
kube-system   coredns-7d89d9b6b8-6xpvs              0/1     Pending   0              49m     <none>          <none>        <none>           <none>
kube-system   coredns-7d89d9b6b8-d5q2h              0/1     Pending   0              49m     <none>          <none>        <none>           <none>
kube-system   etcd-k8s-master1                      1/1     Running   0              50m     1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-apiserver-k8s-master1            1/1     Running   1 (34m ago)    51m     1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-controller-manager-k8s-master1   1/1     Running   1 (108s ago)   3m10s   1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-proxy-kb9w9                      1/1     Running   0              49m     1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-scheduler-k8s-master1            1/1     Running   1 (109s ago)   3m10s   1x.xx.12.181   k8s-master1   <none>           <none>

kubeadm join 1x.xx.12.181:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:e8158944761027754f2ff819e8d299af1095bbc12699a2f34b6d8d80a67d8b6f \
        --control-plane --certificate-key 707aa3ccb1d514ea0270683f7c0ed351f08976617cebe902f1771f6198b6b344

[root@k8s-master1 ~]# kubectl get nodes
NAME          STATUS     ROLES                  AGE     VERSION
k8s-master1   NotReady   control-plane,master   59m     v1.22.8
k8s-master2   NotReady   control-plane,master   2m38s   v1.22.8
k8s-master3   NotReady   control-plane,master   2m37s   v1.22.8

kubeadm join 1x.xx.12.181:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:e8158944761027754f2ff819e8d299af1095bbc12699a2f34b6d8d80a67d8b6f

错误：软件包：containerd.io-1.6.6-3.1.el7.x86_64 (docker-ce-stable)
          需要：container-selinux >= 2:2.74

yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm

[root@k8s-master1 ~]# kubectl get pod -A -owide
NAMESPACE     NAME                                  READY   STATUS             RESTARTS        AGE   IP              NODE          NOMINATED NODE   READINESS GATES
kube-system   coredns-7d89d9b6b8-6xpvs              0/1     Pending            0               28m   <none>          <none>        <none>           <none>
kube-system   coredns-7d89d9b6b8-d5q2h              0/1     Pending            0               28m   <none>          <none>        <none>           <none>
kube-system   etcd-k8s-master1                      1/1     Running            0               30m   1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-apiserver-k8s-master1            1/1     Running            1 (14m ago)     30m   1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-controller-manager-k8s-master1   0/1     CrashLoopBackOff   7 (4m31s ago)   30m   1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-proxy-kb9w9                      1/1     Running            0               28m   1x.xx.12.181   k8s-master1   <none>           <none>
kube-system   kube-scheduler-k8s-master1            0/1     CrashLoopBackOff   6 (4m27s ago)   30m   1x.xx.12.181   k8s-master1   <none>           <none>

Liveness probe failed: Get "https://127.0.0.1:10257/healthz": dial tcp 127.0.0.1:10257: connect: connection refused

vim /etc/kubernetes/manifests/kube-scheduler.yaml
vim /etc/kubernetes/manifests/kube-controller-manager.yaml

systemctl restart kubelet.service

docker ps -a
docker logs -f etcd的容器id

leader failed to send out heartbeat on time; took too long, leader is overloaded likely from slow disk

sar -d 1 10

平均时间:       DEV       tps  rd_sec/s  wr_sec/s  avgrq-sz  avgqu-sz     await     svctm     %util
平均时间:       vda     12.80      0.00    248.00     19.38      2.28    177.77     35.69     45.68
平均时间:       vdb      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
平均时间: centos-root     24.70      0.00    252.00     10.20      2.29     92.52     40.52    100.09

iostat -dxk 1 10

平均时间:       DEV       tps  rd_sec/s  wr_sec/s  avgrq-sz  avgqu-sz     await     svctm     %util
平均时间:       vda     51.10      0.00    554.40     10.85      0.03      0.65      0.70      3.56
平均时间:       vdb      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00

The connection to the server 1x.xx.12.181:6443 was refused - did you specify the right host or port?

systemctl status docker.service
systemctl status kubelet.service
systemctl status firewalld.service   #关闭状态

netstat -pnlt | grep 6443

journalctl -xeu kubelet

kubeadm reset -f
modprobe -r ipip
yum -y remove kubeadm* kubectl* kubelet* docker*
rm -rf ~/.kube/
rm -rf /etc/kubernetes/
rm -rf /etc/systemd/system/kubelet.service.d
rm -rf /etc/systemd/system/kubelet.service
rm -rf /usr/bin/kube*
rm -rf /var/lib/etcd
rm -rf /var/etcd

ENV GO111MODULE=on
ENV GOPROXY=https://goproxy.cn,direct

anjia0532/distroless.static:nonroot

make docker-build docker-push IMG=harbor-test.xxx.net/paas/demo-operator:1.0

make deploy IMG=harbor-test.xxx.net/paas/demo-operator:1.0

kustomize build config/default | kubectl apply -f -
namespace/demo-operator-system created
customresourcedefinition.apiextensions.k8s.io/demoes.tutorial.demo.com unchanged
serviceaccount/demo-operator-controller-manager created
role.rbac.authorization.k8s.io/demo-operator-leader-election-role created
clusterrole.rbac.authorization.k8s.io/demo-operator-manager-role created
clusterrole.rbac.authorization.k8s.io/demo-operator-metrics-reader created
clusterrole.rbac.authorization.k8s.io/demo-operator-proxy-role created
rolebinding.rbac.authorization.k8s.io/demo-operator-leader-election-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/demo-operator-manager-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/demo-operator-proxy-rolebinding created
configmap/demo-operator-manager-config created
service/demo-operator-controller-manager-metrics-service created
deployment.apps/demo-operator-controller-manager created

kubectl apply -f config/samples/tutorial_v1_demo.yaml

.PHONY: deploy
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
	$(KUSTOMIZE) build config/default | kubectl apply -f -

cd config/manager && /usr/local/bin/kustomize edit set image controller=harbor-test.xxx.net/paas/demo-operator:1.0

cd ../.. && /usr/local/bin/kustomize build config/default > demo-operator.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    control-plane: controller-manager
  name: demo-operator-system
---
太长了，不贴了

kubectl apply -f demo-operator.yaml

kubectl apply -f demo-simple.yaml