• Документация по Kubernetes (на русском)

• youtube Введение в Kubernetes на примере Minikube
• youtube Kubernetes на Русском Языке 12 видео

• Ультимативный гайд по созданию CI/CD в GitLab с автодеплоем в Kubernetes на голом железе всего за 514$ в год ( ͡° ͜ʖ ͡°)
• Полноценный Kubernetes с нуля на Raspberry Pi

• 11 факапов PRO-уровня при внедрении Kubernetes и как их избежать

• A set of exercises that helped me prepare for the Certified Kubernetes Application Developer exam

• Наш опыт с Kubernetes в небольших проектах / Дмитрий Столяров (Флант)

• https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands

# curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl

# chmod +x kubectl

# mv kubectl /usr/local/bin/

gitlab-runner@server:~$ mkdir .kube/

gitlab-runner@server:~$ scp root@node1:.kube/config .kube/config

gitlab-runner@server:~$ cat .kube/config

...
    server: https://node1:6443
...

gitlab-runner@server:~$ kubectl get all -o wide --all-namespaces
gitlab-runner@server:~$ kubectl get all -o wide -A

gitlab-runner@server:~$ source <(kubectl completion bash)

gitlab-runner@server:~$ scp root@kube1:.kube/config .kube/config_kube1

gitlab-runner@server:~$ cat .kube/config_kube1

...
    .kube/config_kube1
...

gitlab-runner@server:~$ export KUBECONFIG=~/.kube/config_kube1

gitlab-runner@server:~$ kubectl get nodes

• How to Install Minikube on Ubuntu 20.04 LTS / 21.04
• Documentation/Get Started/minikube start

root@server:~# apt install -y curl wget apt-transport-https

root@server:~# wget https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64

root@server:~# mv minikube-linux-amd64 /usr/local/bin/minikube

root@server:~# chmod +x /usr/local/bin/minikube

• Технология Docker Предоставление прав непривилегированным пользователям

gitlab-runner@server:~$ ### minikube delete
gitlab-runner@server:~$ ### rm -rv .minikube/

gitlab-runner@server:~$ time minikube start --driver=docker --insecure-registry "server.corpX.un:5000"
real    29m8.320s
...

gitlab-runner@server:~$ minikube status

gitlab-runner@server:~$ minikube ip

gitlab-runner@server:~$ minikube addons list

gitlab-runner@server:~$ minikube addons configure registry-creds   #Не нужно для registry попубличных проектов
...
Do you want to enable Docker Registry? [y/n]: y
-- Enter docker registry server url: http://server.corpX.un:5000
-- Enter docker registry username: student
-- Enter docker registry password:
...

gitlab-runner@server:~$ minikube addons enable registry-creds

gitlab-runner@server:~$ minikube kubectl -- get pods -A

gitlab-runner@server:~$ alias kubectl='minikube kubectl --'

gitlab-runner@server:~$ kubectl get pods -A

или

• Инструмент командной строки kubectl

gitlab-runner@server:~$ ###minikube stop

gitlab-runner@server:~$ ###minikube start

• kubernetes.io Creating a cluster with kubeadm
• Как установить Kubernetes на Ubuntu 20.04 LTS
• How to Install Kubernetes Cluster on Ubuntu 22.04
• https://www.linuxtechi.com/install-kubernetes-cluster-on-debian/
• Установка Kubernetes

node1# ssh-keygen

node1# ssh-copy-id node2
node1# ssh-copy-id node3

node1# bash -c '
swapoff -a
ssh node2 swapoff -a
ssh node3 swapoff -a
'

node1# bash -c '
sed -i"" -e "/swap/s/^/#/" /etc/fstab
ssh node2 sed -i"" -e "/swap/s/^/#/" /etc/fstab
ssh node3 sed -i"" -e "/swap/s/^/#/" /etc/fstab
'

node1# bash -c '
http_proxy=http://proxy.isp.un:3128/ apt -y install apt-transport-https curl
ssh node2 http_proxy=http://proxy.isp.un:3128/ apt -y install apt-transport-https curl
ssh node3 http_proxy=http://proxy.isp.un:3128/ apt -y install apt-transport-https curl
'

node1# bash -c '
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add
ssh node2 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add"
ssh node3 "curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add"
'

node1# bash -c '
apt-add-repository "deb http://apt.kubernetes.io/ kubernetes-xenial main"
ssh node2 apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\"
ssh node3 apt-add-repository \"deb http://apt.kubernetes.io/ kubernetes-xenial main\"
'

node1# bash -c '
http_proxy=http://proxy.isp.un:3128/ apt -y install kubeadm kubelet kubectl kubernetes-cni
ssh node2 http_proxy=http://proxy.isp.un:3128/ apt -y install kubeadm kubelet kubectl kubernetes-cni
ssh node3 http_proxy=http://proxy.isp.un:3128/ apt -y install kubeadm kubelet kubectl kubernetes-cni
'

root@node1:~# kubeadm init --pod-network-cidr=10.244.0.0/16 --apiserver-advertise-address=192.168.X.201

root@node1:~# mkdir -p $HOME/.kube

root@node1:~# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

root@node1:~# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

root@node1:~# kubectl get pod -o wide --all-namespaces

root@node1:~# kubectl get --raw='/readyz?verbose'

• Может понадобиться в случае возникновения ошибки [ERROR CRI]: container runtime is not running

node1# bash -c '
rm /etc/containerd/config.toml
systemctl restart containerd
ssh node2 rm /etc/containerd/config.toml
ssh node2 systemctl restart containerd
ssh node3 rm /etc/containerd/config.toml
ssh node3 systemctl restart containerd
'

root@node2_3:~# curl -k https://node1:6443/livez?verbose

• [ERROR CRI]: container runtime is not running

root@node2_3:~# kubeadm join 192.168.X.201:6443 --token NNNNNNNNNNNNNNNNNNNN \
        --discovery-token-ca-cert-hash sha256:NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN

root@node1:~# kubectl cluster-info

root@node1:~# kubectl get nodes -o wide

• Unable to pull image from insecure registry, http: server gave HTTP response to HTTPS client #4938
• Containerd cannot pull image from insecure registry #3847

• How to Configure Private Registry for Kubernetes cluster running with containerd
• containerd/docs/PLUGINS.md migrate config v1 to v2

• Docker Insecure Private Registry

node1# scp server:/etc/docker/daemon.json /etc/docker/daemon.json

node1# bash -c '
scp /etc/docker/daemon.json node2:/etc/docker/daemon.json
scp /etc/docker/daemon.json node3:/etc/docker/daemon.json
service docker restart
ssh node2 service docker restart
ssh node3 service docker restart
'

# don't work in cri-tools 1.25, need public project in GitLab (Settings->General->Visibility, project features, permissions)
### node1# docker login http://server.corpX.un:5000

### node1# bash -c '
ssh node2 mkdir -p .docker
ssh node3 mkdir -p .docker
scp ~/.docker/config.json node2:.docker/config.json
scp ~/.docker/config.json node3:.docker/config.json
'

root@node1:~# mkdir /etc/containerd/

root@node1:~# cat /etc/containerd/config.toml

version = 2

[plugins."io.containerd.grpc.v1.cri".registry]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."server.corpX.un:5000"]
      endpoint = ["http://server.corpX.un:5000"]
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."server.corpX.un:5000".tls]
      insecure_skip_verify = true
# don't work in cri-tools 1.25, need public project
#[plugins."io.containerd.grpc.v1.cri".registry.configs."server.corpX.un:5000".auth]
#      auth = "c3R1ZGVudDpwYXNzd29yZA=="

node1# bash -c '
ssh node2 mkdir /etc/containerd/
ssh node3 mkdir /etc/containerd/
scp /etc/containerd/config.toml node2:/etc/containerd/config.toml
scp /etc/containerd/config.toml node3:/etc/containerd/config.toml
systemctl restart containerd
ssh node2 systemctl restart containerd
ssh node3 systemctl restart containerd
'

root@nodeN:~# containerd config dump | less

Проверка

root@nodeN:~# crictl -r unix:///run/containerd/containerd.sock pull server.corpX.un:5000/student/webd

• https://github.com/kubernetes-sigs/kubespray
• Самое подробное руководство по установке высокодоступного (почти ಠ ͜ʖ ಠ ) Kubernetes-кластера
• Bare-metal kubernetes-кластер на своём локальном компьютере
• Kubernetes — установка через Kubespray
• Installing Kubernetes on Ubuntu Server with Kubespray

kube1# ssh-keygen

kube1# ssh-copy-id kube1;ssh-copy-id kube2;ssh-copy-id kube3;ssh-copy-id kube4;

kube1# apt update

kube1# apt install python3-pip -y

kube1# git clone https://github.com/kubernetes-sigs/kubespray

kube1# cd kubespray/

~/kubespray# grep -r containerd_insecure_registries .
~/kubespray# git log

~/kubespray# git branch -r
~/kubespray# ### git checkout origin/release-2.22

~/kubespray# git tag -l
~/kubespray# ### git checkout tags/v2.22.1

~/kubespray# git checkout 4c37399c7582ea2bfb5202c3dde3223f9c43bf59

~/kubespray# ### git checkout master

• Может потребоваться Виртуальная среда Python

~/kubespray# time pip3 install -r requirements.txt
real    1m48.202s

~/kubespray# cp -rfp inventory/sample inventory/mycluster

~/kubespray# declare -a IPS=(kube1,192.168.X.221 kube2,192.168.X.222 kube3,192.168.X.223)

~/kubespray# CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]}

~/kubespray# time ansible-playbook -i inventory/mycluster/hosts.yaml cluster.yml
real    45m31.796s

kube1# less ~/.kube/config

~/kubespray# ###time ansible-playbook -i inventory/mycluster/hosts.yaml reset.yml
real    7m31.796s

~/kubespray# cat inventory/mycluster/hosts.yaml

...
    node4:
      ansible_host: 192.168.X.204
      ip: 192.168.X.204
      access_ip: 192.168.X.204
...
    kube_node:
...
        node4:
...

~/kubespray# ansible-playbook -i inventory/mycluster/hosts.yaml scale.yml

$ kubectl get nodes -o wide

~/kubespray# cat inventory/mycluster/group_vars/all/containerd.yml

...
containerd_insecure_registries:
  "server.corpX.un:5000": "http://server.corpX.un:5000"
containerd_registry_auth:
  - registry: server.corpX.un:5000
    username: student
    password: Pa$$w0rd
...

~/kubespray# time ansible-playbook -i inventory/mycluster/hosts.yaml cluster.yml
user    46m37.151s

# less /etc/containerd/config.toml

~/kubespray# cat inventory/mycluster/group_vars/k8s_cluster/addons.yml

...
helm_enabled: true
...
ingress_nginx_enabled: true
ingress_nginx_host_network: true
...

• Define a Command and Arguments for a Container
• kubectl для пользователей Docker
• Run a Stateless Application Using a Deployment

$ kubectl api-resources

$ kubectl run my-debian --image=debian -- "sleep" "3600"

$ kubectl get all

kubeN# crictl ps

$ kubectl delete pod my-debian

$ kubectl create deployment my-debian --image=debian -- "sleep" "3600"

$ kubectl get deployments

• Настройка автодополнения

$ kubectl attach my-debian-NNNNNNNNN-NNNNN

$ kubectl exec -ti my-debian-NNNNNNNNN-NNNNN -- bash
Ctrl-D

$ kubectl get deployment my-debian -o yaml

• Переменные окружения

$ kubectl edit deployment my-debian

$ kubectl get pods -o wide

$ kubectl delete deployment my-debian

• Kubernetes Documentation Reference Glossary/Manifest

$ cat my-debian-deployment.yaml

apiVersion: apps/v1
kind: ReplicaSet
#kind: Deployment
metadata:
  name: my-debian
spec:
  selector:
    matchLabels:
      app: my-debian
  replicas: 2
  template:
    metadata:
      labels:
        app: my-debian
    spec:
      containers:
      - name: my-debian
        image: debian
        command: ["/bin/sh"]
        args: ["-c", "while true; do echo hello; sleep 3;done"]
      restartPolicy: Always

$ kubectl apply -f my-debian-deployment.yaml
...
$ kubectl delete -f my-debian-deployment.yaml

• How to use an NFS volume

$ kubectl create namespace my-ns

$ kubectl get namespaces

$ ### kubectl create deployment my-webd --image=server.corpX.un:5000/student/webd:latest --replicas=2 -n my-ns

$ ### kubectl delete deployment my-webd -n my-ns

$ cd webd/

$ cat my-webd-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-webd
spec:
  selector:
    matchLabels:
      app: my-webd
  replicas: 2
  template:
    metadata:
      labels:
        app: my-webd
    spec:
      containers:
      - name: my-webd

#        image: server.corpX.un:5000/student/webd
#        image: server.corpX.un:5000/student/webd:ver1.N

#        livenessProbe:
#          httpGet:
#            port: 80

#        volumeMounts:
#        - name: nfs-volume
#          mountPath: /var/www
#      volumes:
#      - name: nfs-volume
#        nfs:
#          server: server.corpX.un
#          path: /var/www

$ kubectl apply -f my-webd-deployment.yaml -n my-ns

$ kubectl get all -n my-ns -o wide 

$ kubectl describe -n my-ns pod/my-webd-NNNNNNNNNN-NNNNN

$ kubectl scale deployment my-webd --replicas=3 -n my-ns

$ kubectl delete pod/my-webd-NNNNNNNNNN-NNNNN -n my-ns

• Kubernetes Documentation Concepts Services, Load Balancing, and Networking Service

• How do I get logs from all pods of a Kubernetes replication controller?

$ ### kubectl expose deployment my-webd --type=NodePort --port=80 -n my-ns

$ ### kubectl delete svc my-webd -n my-ns

$ cat my-webd-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: my-webd
spec:
#  type: NodePort
  selector:
    app: my-webd
  ports:
  - protocol: TCP
    port: 80
#    nodePort: 30111

$ kubectl apply -f my-webd-service.yaml -n my-ns

$ kubectl get svc my-webd -n my-ns
NAME              TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
my-webd-svc   NodePort   10.102.135.146   <none>        80:NNNNN/TCP   18h

$ kubectl describe svc my-webd -n my-ns

$ curl http://node1,2,3:NNNNN
на "самодельном kubeadm" кластере работает не стабильно



$ minikube service list

$ minikube service my-webd -n my-ns --url
http://192.168.49.2:NNNNN

$ curl $(minikube service my-webd -n my-ns --url)


$ kubectl logs -l app=my-webd -n my-ns 
(доступны опции -f, --tail=2000, --previous)

$ kubectl get all -n my-ns

$ kubectl delete -n my-ns -f my-webd-deployment.yaml,my-webd-service.yaml

или

$ kubectl delete namespace my-ns

• NGINX ingress controller quick-start

• Set up Ingress on Minikube with the NGINX Ingress Controller
• kubectl port-forward examples in Kubernetes

server# cat /etc/bind/corpX.un

...
webd A 192.168.49.2

gitlab-runner@server:~$ minikube addons enable ingress

• https://github.com/kubernetes/ingress-nginx/tags Версии
• Nginx Ingress Controller - Failed Calling Webhook

server# cat /etc/bind/corpX.un

...
webd            A       192.168.X.201
gowebd          CNAME   webd

$ curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.1/deploy/static/provider/cloud/deploy.yaml

$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.1/deploy/static/provider/cloud/deploy.yaml

$ kubectl get all -n ingress-nginx -o wide
...
pod/ingress-nginx-controller...  ... Running ...
...

$ kubectl delete -A ValidatingWebhookConfiguration ingress-nginx-admission   # Понадобилось см. ссылку выше

root@node1:~# kubectl port-forward --namespace=ingress-nginx --address 0.0.0.0 service/ingress-nginx-controller 80:80

$ kubectl delete -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.1/deploy/static/provider/cloud/deploy.yaml

• Empty ADDRESS kubernetes ingress

server# cat /etc/bind/corpX.un

...
webd            A       192.168.X.202
                A       192.168.X.203
gowebd          CNAME   webd

$ curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.3.1/deploy/static/provider/baremetal/deploy.yaml | tee ingress-nginx.controller-v1.3.1.baremetal.yaml

$ cat ingress-nginx.controller-v1.3.1.baremetal.yaml

...
kind: Deployment
...
spec:
...
  replicas: 3    ### insert this (equial count of worker nodes)
  template:
...
      terminationGracePeriodSeconds: 300
      hostNetwork: true                    ###insert this
      volumes:
...

$ kubectl apply -f ingress-nginx.controller-v1.3.1.baremetal.yaml

$ ### kubectl delete -f ingress-nginx.controller-v1.3.1.baremetal.yaml

master-1:~$ kubectl exec -n ingress-nginx pods/ingress-nginx-controller-<TAB> -- cat /etc/nginx/nginx.conf | tee nginx.conf

master-1:~$ kubectl edit -n ingress-nginx configmaps ingress-nginx-controller

...
data:
  use-forwarded-headers: "true"
...

gitlab-runner@server:~/webd$ ### kubectl create ingress my-ingress --class=nginx --rule="webd.corpX.un/*=my-webd:80" -n my-ns

gitlab-runner@server:~$ cat my-ingress.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: my-ingress
spec:
  ingressClassName: nginx
#  tls:
#  - hosts:
#    - gowebd.corpX.un
#    secretName: gowebd-tls
  rules:
  - host: webd.corpX.un
    http:
      paths:
      - backend:
          service:
            name: my-webd
            port:
              number: 80
        path: /
        pathType: Prefix
  - host: gowebd.corpX.un
    http:
      paths:
      - backend:
          service:
            name: my-gowebd
            port:
              number: 80
        path: /
        pathType: Prefix

$ kubectl apply -f my-ingress.yaml -n my-ns


$ kubectl get ingress -n my-ns
NAME      CLASS   HOSTS                             ADDRESS                         PORTS   AGE
my-webd   nginx   webd.corpX.un,gowebd.corpX.un   192.168.X.202,192.168.X.203   80      14m


$ curl webd.corpX.un
$ curl gowebd.corpX.un

$ ### kubectl delete ingress my-ingress -n my-ns

• How To Configure Ingress TLS/SSL Certificates in Kubernetes

$ kubectl create secret tls gowebd-tls --key gowebd.key --cert gowebd.crt -n my-ns
    
$ kubectl get secrets -n my-ns

$ kubectl get secret/gowebd-tls -o yaml -n my-ns

$ curl https://gowebd.corpX.un -kv

$ ###kubectl delete secret/gowebd-tls -n my-ns

root@node1:~# # ssh node2 mkdir /disk2

root@node1:~# ssh node2 touch /disk2/disk2_node2

root@node1:~# kubectl label nodes node2 disk2=yes

root@node1:~# kubectl get nodes --show-labels

root@node1:~# ###kubectl label nodes node2 disk2-

root@node1:~# cat my-debian-deployment.yaml

...
        args: ["-c", "while true; do echo hello; sleep 3;done"]

        volumeMounts:
          - name: my-disk2-volume
            mountPath: /data

#        volumeMounts:
#          - name: data
#            mountPath: /data

      volumes:
        - name: my-disk2-volume
          hostPath:
            path: /disk2/
      nodeSelector:
        disk2: "yes"

#      volumes:
#      - name: data
#        persistentVolumeClaim:
#          claimName: my-ha-pvc-sz64m

root@node1:~# kubectl apply -f my-debian-deployment.yaml

root@node1:~# kubectl get all -o wide

• Несколько Claim на один Persistent Volumes?
• Хранилища данных (Persistent Volumes) в Kubernetes
• Limit persistent volume claim content folder size using hostPath

root@node1:~# cat my-ha-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: my-pv-node2-sz-128m-num-001
  labels:
    type: local
spec:
  storageClassName: my-ha-sc
  capacity:
    storage: 128Mi
  accessModes:
    - ReadWriteMany
  hostPath:
    path: /disk2
  persistentVolumeReclaimPolicy: Retain
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - node2

root@node1:~# kubectl apply -f my-ha-pv.yaml

root@node1:~# kubectl get persistentvolume
...
root@node1:~# cat my-ha-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: my-ha-pvc-sz64m
spec:
  storageClassName: my-ha-sc
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 64Mi

root@node1:~# kubectl apply -f my-ha-pvc.yaml

root@node1:~# kubectl get persistentvolumeclaims
...

• Kubernetes ConfigMap: Creating, Viewing, Consuming & Managing
• How to enable SSH connections into a Kubernetes pod

root@node1:~# cat sshd_config

PermitRootLogin yes
PasswordAuthentication no
ChallengeResponseAuthentication no
UsePAM no

root@node1:~# kubectl create configmap ssh-config --from-file=sshd_config --dry-run=client -o yaml
...

server:~# cat .ssh/id_rsa.pub
...

root@node1:~# cat my-openssh-server-deployment.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: ssh-config
data:
  sshd_config: |
    PermitRootLogin yes
    PasswordAuthentication no
    ChallengeResponseAuthentication no
    UsePAM no
  authorized_keys: |
    ssh-rsa AAAAB.....C0zOcZ68= root@server.corpX.un
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-openssh-server
spec:
  selector:
    matchLabels:
      app: my-openssh-server
  template:
    metadata:
      labels:
        app: my-openssh-server
    spec:
      containers:
      - name: my-openssh-server
        image: linuxserver/openssh-server
        command: ["/bin/sh"]
        args: ["-c", "/usr/bin/ssh-keygen -A; usermod -p '*' root; /usr/sbin/sshd.pam -D"]
        ports:
        - containerPort: 22
        volumeMounts:
        - name: ssh-volume
          subPath: sshd_config
          mountPath: /etc/ssh/sshd_config
        - name: ssh-volume
          subPath: authorized_keys
          mountPath: /root/.ssh/authorized_keys
      volumes:
      - name: ssh-volume
        configMap:
          name: ssh-config
---
apiVersion: v1
kind: Service
metadata:
  name: my-openssh-server
spec:
  type: NodePort
  ports:
  - port: 22
    nodePort: 32222
  selector:
    app: my-openssh-server

root@node1:~# kubectl apply -f my-openssh-server-deployment.yaml

root@node1:~# iptables-save | grep 32222

root@node1:~# ###kubectl exec -ti my-openssh-server-NNNNNNNN-NNNNN -- bash

server:~# ssh -p 32222 nodeN
Welcome to OpenSSH Server
my-openssh-server-NNNNNNNN-NNNNN:~#

• Kubernetes multi-container pods and container communication
• kubectl exec into container of a multi container pod
• How to SSH Into a Kubernetes Pod From Outside the Cluster
• Is it possible to expose 2 ports in Kubernetes pod?

gitlab-runner@gate:~/webd$ cat my-webd-ssh-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-webd-ssh
  namespace: my-ns
spec:
  selector:
    matchLabels:
      app: my-webd-ssh
  replicas: 1
  template:
    metadata:
      labels:
        app: my-webd-ssh
    spec:
      containers:
      - name: my-webd
        image: server.corpX.un:5000/student/webd:latest
        volumeMounts:
        - name: html
          mountPath: /var/www
      - name: my-ssh
        image: atmoz/sftp
        args: ["user3:password3:10003"]
        volumeMounts:
        - name: html
          mountPath: /home/user3/www
      volumes:
      - name: html
        emptyDir: {}

...
$ kubectl describe pod my-webd-NNNNNNNNNN-NNNNN -n my-ns

$ kubectl exec -ti -n my-ns my-webd-NNNNNNNNNN-NNNNN -c my-ssh -- bash

$ ### kubectl expose deployment my-webd-ssh --type=NodePort --port=80,22 -n my-ns

$ cat my-webd-ssh-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: my-webd-ssh
  namespace: my-ns
spec:
  type: NodePort
  selector:
    app: my-webd-ssh
  ports:
  - name: http
    protocol: TCP
    port: 80
    targetPort: 80
  - name: ssh
    protocol: TCP
    port: 22
    targetPort: 22

• Installing Helm

server# wget https://get.helm.sh/helm-v3.9.0-linux-amd64.tar.gz

# tar -zxvf helm-v3.9.0-linux-amd64.tar.gz

# mv linux-amd64/helm /usr/local/bin/helm

• NGINX Ingress Controller Installation Guide
• stackoverflow How to install nginx-ingress with hostNetwork on bare-metal?
• devpress.csdn.net How to install nginx-ingress with hostNetwork on bare-metal?

$ helm upgrade ingress-nginx --install ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace

пока не заработало --set controller.hostNetwork=true,controller.service.type="",controller.kind=DaemonSet

$ helm list --namespace ingress-nginx
$ helm list -A

$ kubectl get all -n ingress-nginx -o wide

$ ### helm delete ingress-nginx --namespace ingress-nginx

• How to make a Helm chart in 10 minutes
• Helm upgrade with same chart version, but different Docker image tag
• how set field app-version in helm3 chart?

gitlab-runner@server:~/gowebd-k8s$ helm create webd-chart

$ less webd-chart/templates/deployment.yaml

$ cat webd-chart/Chart.yaml

...
description: A Helm chart WebD for Kubernetes
...
version: 0.1.1
...
appVersion: "latest"
#appVersion: ver1.7   #for vanilla argocd

$ cat webd-chart/values.yaml

...
replicaCount: 2

image:
  repository: server.corpX.un:5000/student/webd
  pullPolicy: Always
...
serviceAccount:
  create: false
...
service:
#  type: NodePort
...
ingress:
  enabled: true
  className: "nginx"
...
  hosts:
    - host: webd.corpX.un
...
#  tls: []
#  tls:
#    - secretName: gowebd-tls
#      hosts:
#        - gowebd.corpX.un
...

$ less webd-chart/templates/deployment.yaml

...
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
...
#          livenessProbe:
#            httpGet:
#              path: /
#              port: http
#          readinessProbe:
#            httpGet:
#              path: /
#              port: http
...

$ helm template my-webd webd-chart/ | less

$ helm install my-webd webd-chart/ -n my-ns --create-namespace --wait

$ kubectl describe events -n my-ns | less

$ export HELM_NAMESPACE=my-ns

$ helm list

$ ### helm upgrade my-webd webd-chart/ --set=image.tag=ver1.10

$ helm history my-webd

$ helm rollback my-webd 1

$ helm uninstall my-webd

• Helm charts in the Package Registry Gitlab
• helm cm-push plugin
• How to make and share your own Helm package
• Gitlab Personal access tokens
• Подключение через API - Role: Mainteiner, api, read_registry, write_registry

gitlab-runner@server:~/gowebd-k8s$ helm repo add --username student --password NNNNN-NNNNNNNNNNNNNNNNNNN webd http://server.corpX.un/api/v4/projects/N/packages/helm/stable
"webd" has been added to your repositories

gitlab-runner@server:~/gowebd-k8s$ ### helm repo remove webd

gitlab-runner@server:~/gowebd-k8s$ helm repo list

gitlab-runner@server:~/gowebd-k8s$ helm package webd-chart

gitlab-runner@server:~/gowebd-k8s$ tar -tf webd-chart-0.1.1.tgz

gitlab-runner@server:~/gowebd-k8s$ helm plugin install https://github.com/chartmuseum/helm-push

gitlab-runner@server:~/gowebd-k8s$ helm cm-push webd-chart-0.1.1.tgz webd

gitlab-runner@server:~/gowebd-k8s$ rm webd-chart-0.1.1.tgz

kube1:~# helm repo add webd http://server.corpX.un/api/v4/projects/N/packages/helm/stable

kube1:~# helm repo update

kube1:~# helm search repo webd

kube1:~# helm repo update webd

kube1:~# helm install my-webd webd/webd-chart

kube1:~# ###helm uninstall my-webd

helm repo add gitlab https://charts.gitlab.io

helm search repo -l gitlab/gitlab-runner

helm show values gitlab/gitlab-runner | tee values.yaml

gitlab-runner@server:~$ diff values.yaml

...
gitlabUrl: http://server.corpX.un/
...
runnerRegistrationToken: "NNNNNNNNNNNNNNNNNNNNNNNN"
...
148,149c142
<   create: true
---
>   create: false
325d317
<         privileged = true
432c424
<   allowPrivilegeEscalation: true
---
>   allowPrivilegeEscalation: false
435c427
<   privileged: true
---
>   privileged: false

gitlab-runner@server:~$ helm upgrade -i gitlab-runner gitlab/gitlab-runner -f values.yaml -n gitlab-runner --create-namespace

gitlab-runner@server:~$ kubectl get all -n gitlab-runner

$ helm search hub -o json wordpress | jq '.' | less

$ helm repo add bitnami https://charts.bitnami.com/bitnami

$ helm show values bitnami/wordpress

student@node2:~$ sudo apt install conntrack

https://computingforgeeks.com/install-mirantis-cri-dockerd-as-docker-engine-shim-for-kubernetes/
...

wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.2/crictl-v1.24.2-linux-amd64.tar.gz
...

student@node2:~$ minikube start --driver=none --insecure-registry "server.corpX.un:5000"

student@node1:~$ minikube dashboard &
...
Opening http://127.0.0.1:NNNNN/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/ in your default browser
...
/home/mobaxterm> ssh -L NNNNN:localhost:NNNNN student@192.168.X.10
Теперь, та же ссылка работает на win host системе

• Если не minikube, то достаточно только копию .kube/config
• см. Настройка GitLab Runner

student@node1:~$ tar -cvzf kube-config.tar.gz .kube/config .minikube/ca.crt .minikube/profiles/minikube

gitlab-runner@server:~$ scp student@node1:kube-config.tar.gz .

gitlab-runner@server:~$ tar -xvf kube-config.tar.gz

gitlab-runner@server:~$ cat .kube/config

...
    certificate-authority: /home/gitlab-runner/.minikube/ca.crt
...
    client-certificate: /home/gitlab-runner/.minikube/profiles/minikube/client.crt
    client-key: /home/gitlab-runner/.minikube/profiles/minikube/client.key
...

• What's the difference between Docker Compose and Kubernetes?
• Docker Compose to Kubernetes: Step-by-Step Migration
• Translate a Docker Compose File to Kubernetes Resources

root@gate:~# curl -L https://github.com/kubernetes/kompose/releases/download/v1.26.0/kompose-linux-amd64 -o kompose
root@gate:~# chmod +x kompose
root@gate:~# sudo mv ./kompose /usr/local/bin/kompose

• docker-compose

gitlab-runner@gate:~/webd$ kompose convert
gitlab-runner@gate:~/webd$ ls *yaml
gitlab-runner@gate:~/webd$ kubectl apply -f sftp-deployment.yaml,vol1-persistentvolumeclaim.yaml,webd-service.yaml,sftp-service.yaml,webd-deployment.yaml
gitlab-runner@gate:~/webd$ kubectl get all