2.3
MinIO 对象存储部署
Velero 依赖对象存储保存备份数据，这里部署 MinIO 替代公有云对象存储
minio.yaml
spec:
nodeSelector:
kubernetes.io/hostname: k8smaster1 //替换成自己的主机节点名称
image: nexus.cmss.com:8086/minio/mc:latest // mc替换成minio-client
启动报错需要修改目录权限：
chmod 755 /data/minio
# kubectl apply -f minio.yaml
# kubectl get all -n velero
NAME READY STATUS RESTARTS AGE
pod/minio-6d4696f57f-g64q9 1/1 Running 0 2d3h
pod/restic-k9sw9 1/1 Running 0 2d2h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/minio NodePort 10.1.241.109 <none> 9000:30080/TCP,9001:30081/TCP 2d4h
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/minio 1/1 1 1 2d4h
NAME DESIRED CURRENT READY AGE
replicaset.apps/minio-6d4696f57f 1 1 1 2d3h
NAME COMPLETIONS DURATION AGE
job.batch/minio-setup 0/1 2d4h 2d4h
http://10.1.6.104:30081/login
Velero 客户端
wget https://github.com/vmware-tanzu/velero/releases/download/v1.8.1/velero-v1.8.1-linux-amd64.tar.gz
tar -zxvf velero-v1.8.1-linux-amd64.tar.gz && cd velero-v1.8.1-linux-amd64
tree .
.
├── LICENSE
├── examples
│ ├── README.md
│ ├── minio
│ │ └── 00-minio-deployment.yaml
│ └── nginx-app
│ ├── README.md
│ ├── base.yaml
│ └── with-pv.yaml
└── velero
➜~ cp velero /usr/local/bin && chmod +x /usr/local/bin/velero
➜~ velero version
Client:
Version: v1.8.1
Git commit: 18ee078dffd9345df610e0ca9f61b31124e93f50
Server:
Version: v1.8.1
Velero 服务端
密钥准备
# 秘钥文件credentials-velero
# cat /root/wnh/k8s/velero/credentials-velero
# 秘钥文件credentials-velero
[default]
aws_access_key_id=admin
aws_secret_access_key=minio123
客户端安装
velero install --provider aws --bucket velero --image velero/velero:v1.8.1 --plugins velero/velero-plugin-for-aws:v1.4.1 --namespace velero --secret-file /root/wnh/k8s/velero/credentials-velero --use-volume-snapshots=false --default-volumes-to-restic=true --kubeconfig=/root/.kube/config --use-restic --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://10.1.6.104:30080
# kubectl get all -n velero
NAME READY STATUS RESTARTS AGE
pod/minio-6d4696f57f-g64q9 1/1 Running 0 2d3h
pod/restic-k9sw9 1/1 Running 0 2d3h
pod/velero-55796c7c76-v9nfg 1/1 Running 0 2d3h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/minio NodePort 10.1.241.109 <none> 9000:30080/TCP,9001:30081/TCP 2d4h
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/restic 1 1 1 1 1 <none> 2d3h
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/minio 1/1 1 1 2d4h
deployment.apps/velero 1/1 1 1 2d3h
NAME DESIRED CURRENT READY AGE
replicaset.apps/minio-57c84f9cf8 0 0 0 2d4h
replicaset.apps/minio-5d66fd5444 0 0 0 2d3h
replicaset.apps/minio-6c866cffdd 0 0 0 2d3h
replicaset.apps/minio-6d4696f57f 1 1 1 2d3h
replicaset.apps/minio-7568b7d594 0 0 0 2d4h
replicaset.apps/minio-75964b589d 0 0 0 2d3h
replicaset.apps/minio-9d5556dfc 0 0 0 2d3h
replicaset.apps/velero-55796c7c76 1 1 1 2d3h
NAME COMPLETIONS DURATION AGE
job.batch/minio-setup 0/1 2d4h 2d4h
部署多集群查询服务Clusterpedia
本次测试服务使用的是一个多集群查询服务 Clusterpedia（包含使用本地存储的 MySQL+3 个 deployment 等资源）
Clusterpedia中文官网
Clusterpedia 的安装分为两个部分：
安装存储组件
安装 Clusterpedia
//拉取项目：
git clone https://github.com/clusterpedia-io/clusterpedia.git
cd clusterpedia
git checkout v0.7.0
//安装存储组件
cd ./deploy/internalstorage/mysql
export STORAGE_NODE_NAME=<节点名称>
sed "s|__NODE_NAME__|$STORAGE_NODE_NAME|g" `grep __NODE_NAME__ -rl ./templates` > clusterpedia_internalstorage_pv.yaml
kubectl apply -f .
//安装 Clusterpedia
# 跳回 Clusterpedia 项目根目录
cd ../../../
kubectl apply -f ./deploy
# kubectl get all -n clusterpedia-system
NAME READY STATUS RESTARTS AGE
pod/clusterpedia-apiserver-5bd88d6488-nw8p5 1/1 Running 0 27h
pod/clusterpedia-clustersynchro-manager-57f7b7bd4-b9pxt 1/1 Running 0 27h
pod/clusterpedia-controller-manager-846c98df9d-v5v8m 1/1 Running 0 28h
pod/clusterpedia-internalstorage-mysql-7f675744f9-bsqqc 1/1 Running 0 28h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/clusterpedia-apiserver ClusterIP 10.1.85.124 <none> 443/TCP 28h
service/clusterpedia-internalstorage-mysql ClusterIP 10.1.93.37 <none> 3306/TCP 28h
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/clusterpedia-apiserver 1/1 1 1 28h
deployment.apps/clusterpedia-clustersynchro-manager 1/1 1 1 28h
deployment.apps/clusterpedia-controller-manager 1/1 1 1 28h
deployment.apps/clusterpedia-internalstorage-mysql 1/1 1 1 28h
NAME DESIRED CURRENT READY AGE
replicaset.apps/clusterpedia-apiserver-5bd88d6488 1 1 1 28h
replicaset.apps/clusterpedia-clustersynchro-manager-57f7b7bd4 1 1 1 28h
replicaset.apps/clusterpedia-controller-manager-846c98df9d 1 1 1 28h
replicaset.apps/clusterpedia-internalstorage-mysql-7f675744f9 1 1 1 28h
接入集群
// 可参考 https://clusterpedia.io/zh-cn/docs/usage/import-clusters/
# cat pediacluster-master.yaml
apiVersion: cluster.clusterpedia.io/v1alpha2
kind: PediaCluster
metadata:
name: master
spec:
apiserver: "https://10.1.6.104:6443"
kubeconfig: YXBpVmVyc2lvbjogdjEKY2x1c3RlcnM6Ci0gY2x1c3RlcjoKICAgIGNlcnRpZmljYXRlLWF1dGhvcml0eS1kYXRhOiBMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNdmFrTkRRV1ZoWjBGM1NVSkJaMGxDUVVSQlRrSm5hM0ZvYTJsSE9YY3dRa0ZSYzBaQlJFRldUVkpOZDBWUldVUldVVkZFUlhkd2NtUlhTbXdLWTIwMWJHUkhWbnBOUWpSWVJGUkplazFVUVhoTlZFRjZUV3BqTUUweGIxaEVWRTE2VFZSQmQwOUVRWHBOYW1Nd1RURnZkMFpVUlZSTlFrVkhRVEZWUlFwQmVFMUxZVE5XYVZwWVNuVmFXRkpzWTNwRFEwRlRTWGRFVVZsS1MyOWFTV2gyWTA1QlVVVkNRbEZCUkdkblJWQkJSRU5EUVZGdlEyZG5SVUpCVEdocUNuWXlORE55UjFOWk1VRkJWMDFNZVhGTU5rdHNSV0YzZWl0V05sRndkSHBPTWtKSFpFTkRRbUl5Tld4NWRVWjFPVVF6WWxKQ01UWkJVbXRZZHpoUFluWUtiamN6YjBsbFFVbHNOVXR6ZFZkR1owbDZXRTV6V1RKVVZHeDRZWGt4UkhWSVFtSlFWRGRsT0VWVFRVUkJWV2NyYVZFcmNFdHpjU3RaZERsSWJWZFFPUXBMUnpZemFsUndjVEZRTlZvd2EwczVlRzVqTlZORFpTOVFWak14VVRoalNVdFhRVEJOVVc1aVFUaDFiRmsyTmtjME9EZzBkMGxCV1cxM1EzVnhWVEpOQ2xwVE5FY3pZMEZCZDFOeVZtbG9OemdyTlZkNWJrMU5aME5KWld4ME0xTTFRbnBPVXpoYU1WUkpia1ZQZEVrMU9YaHRkMFJJUlVRNGJtMTRTME5hWVVZS2NsaDNTekJPWVRkVWVUQnZSeTloVlRCWFdHcHRWRXhpTVZObGIyWllUV2RsUmpKbllXbDBTVnBTTVVrMWVXRlRiREJDYkVOT1NrbHlUR1kwV2tSdEt3cEdWVmN5Y0dwcWEwRnBVSEV4U1dvclIyOU5RMEYzUlVGQllVNWFUVVpqZDBSbldVUldVakJRUVZGSUwwSkJVVVJCWjB0clRVRTRSMEV4VldSRmQwVkNDaTkzVVVaTlFVMUNRV1k0ZDBoUldVUldVakJQUWtKWlJVWk9jV3ROYzNOME0wWnRhVmR6YmtsVVZITm1jR0ZQV25wdmNqbE5RbFZIUVRGVlpFVlJVVThLVFVGNVEwTnRkREZaYlZaNVltMVdNRnBZVFhkRVVWbEtTMjlhU1doMlkwNUJVVVZNUWxGQlJHZG5SVUpCUnpGNlVGa3lZVWxMVXpKemRVMHJhVFJUV1FwNWMwMDVhV0ZJYTNseFZUVTRjbWRpWlcxTWVHczJVbEpyYVRKdmFXdGxlV3RIWnpKV2FpdHJXVXhCVUVwcmRHOWtNemxMT1hrMFpFVm9RMDF5UVVWdENtVTNXRlZ3Ym5sR2JXdEdaRmd6VTJacmVEZHpWbFpIZFVkSE1rcHFVMkZ5YUd4Q1QzSlpSMmxOTXl0Nk1WZFNlVEJOU2xnMFR6aGFUa0Y1Vlc5alluY0tSRFl5V0ZSelZEa3pUMFI2VkRSalRXVjJVbE5zWm1adk9HeERRa3c0ZWpGd1VWcFFlVlkzUjJKNU55OVlXbEZRZERkUlExUjBhQzgzWWtvM1duQjVhZ3B4VmtRcmRuTXZWeTlsY25ZeE1FdEtZM1I2YmxZeE1HY3pjbVoySzI4NGFFcDNkSGR5TDFGQ1RFNXJkR1pYYzBOb2RXWTNWbWxYUkRGdEsySjZOR1ZzQ25ReVZWZG5VMVptZEVka1UwczFWbVJLTUZJMldtVlljRko2VVdGRFNHbDViR0pTVVRKSE9EZEthVEJHVGsxTFdXVnNaSGQyZHpVemVVVldRaXN4TTNNS00yVkJQUW90TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09CiAgICBzZXJ2ZXI6IGh0dHBzOi8vMTAuMS42LjEwNDo2NDQzCiAgbmFtZToga3ViZXJuZXRlcwpjb250ZXh0czoKLSBjb250ZXh0OgogICAgY2x1c3Rlcjoga3ViZXJuZXRlcwogICAgdXNlcjoga3ViZXJuZXRlcy1hZG1pbgogIG5hbWU6IGt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcwpjdXJyZW50LWNvbnRleHQ6IGt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcwpraW5kOiBDb25maWcKcHJlZmVyZW5jZXM6IHt9CnVzZXJzOgotIG5hbWU6IGt1YmVybmV0ZXMtYWRtaW4KICB1c2VyOgogICAgY2xpZW50LWNlcnRpZmljYXRlLWRhdGE6IExTMHRMUzFDUlVkSlRpQkRSVkpVU1VaSlEwRlVSUzB0TFMwdENrMUpTVVJKVkVORFFXZHRaMEYzU1VKQlowbEpUWGhCSzFwNlQxWTBZVTEzUkZGWlNrdHZXa2xvZG1OT1FWRkZURUpSUVhkR1ZFVlVUVUpGUjBFeFZVVUtRWGhOUzJFelZtbGFXRXAxV2xoU2JHTjZRV1ZHZHpCNVRYcEZkMDFVUlhkTmVra3pUa1JPWVVaM01IbE9SRVYzVFZSQmQwMTZTVE5PUkdoaFRVUlJlQXBHZWtGV1FtZE9Wa0pCYjFSRWJrNDFZek5TYkdKVWNIUlpXRTR3V2xoS2VrMVNhM2RHZDFsRVZsRlJSRVY0UW5Ka1YwcHNZMjAxYkdSSFZucE1WMFpyQ21KWGJIVk5TVWxDU1dwQlRrSm5hM0ZvYTJsSE9YY3dRa0ZSUlVaQlFVOURRVkU0UVUxSlNVSkRaMHREUVZGRlFURkZOazlEZFRsSlRYaHJPRUpDTjBNS1FXcFBRMFp5UjA5cFNUUjZNV0ZCYzB4aFlrcGlUR2hKUlhKa1dURldhbk5ZVVVSR2VDdFljRWsyUVZkb1ExaElRVms0Um05aFFXSkRUVVpZZDBSbWJBcFlWQzh5UTFGaGNUY3pOMEVyYWtFemRrWm5VQ3MyV0dOT1IyVk9kbEZRVkZGRWRrTkJSVmRSZGxacFdFcHdWREJOZDJGcU1YRnhSemRxYlVaWGFIRjJDbTlSTWxaTldtOHdSVFZoYVdsMU9WaENjU3QxYWs1WFZHMW1aazlvV0VOdlNFOWpaWGd6VlZCa2VuZ3ZiMVJzY0dWSlVXRlhiMDVNWkdKeWRWQmtjMlFLUXpCV1RpOUVOVGd4T1Rsd05Ga3hVVmRIVTNkaVluUnhlV2wwZW5kelFYVTJhV0V2TkdKVU5IWjBLM2MwY3pCcGQyMHljbkpyU1hVdkswMU5jSGxxUWdwM1pXRmtZMjFyV1hWS01WZEJiR0pqWlU5alpIZHRlVlV3VUhKc2NIcFBjWGREUVVJNFR6bHRhVTgzTlM4NFUyWkhOV3RxVTNaTE1rOW5NblYwYjFsNkNqTkZWSEo2ZDBsRVFWRkJRbTh4V1hkV1JFRlBRbWRPVmtoUk9FSkJaamhGUWtGTlEwSmhRWGRGZDFsRVZsSXdiRUpCZDNkRFoxbEpTM2RaUWtKUlZVZ0tRWGRKZDBSQldVUldVakJVUVZGSUwwSkJTWGRCUkVGbVFtZE9Wa2hUVFVWSFJFRlhaMEpVWVhCRVRFeE1aSGhhYjJ4eVNubEZNRGRJTmxkcWJXTTJTd292VkVGT1FtZHJjV2hyYVVjNWR6QkNRVkZ6UmtGQlQwTkJVVVZCWWtRdlZHVmFSVGxHSzFCVlprUkJaRXhJYzFnd1QyOWxkemwwWlhkcVZGUkdVMHB0Q2xCRVprSjJXV05LTUZsblNuSkZUQ3R2YmtFNVJqSjVWa0o0VTJ4eFJtSldXSFF4WldkTFZXNHJVbGh2YVZaNmEyeGxZblk1TWtocmRsazBSbE54Y1RRS1puSlNTVWRxU1dOeWNHUjBTbUYxZWtwUFUzcENka0ZJS3pWdWVGTmpkbHBuUlhaNVJrRmhTVlZWU1c1aWRuQkNaVzFaUzJGQlpIcHdNRGR0T0U1dk5ncHRaVEJKT1d0UGRFRklWMGhaT1hFdmJ6RXdiVmR0VWxOUE16Qk5jemxrZDIxbFJYZEJXR3hvZWxvclZTOXNhbmhUU3pFdk16SnlXbGhZU21vM2JGZFRDbU5ZUTFabFVrRnZlR3QwTDFaWmN5dE5WVmg0Tldwa2RsUTJkREYwZEdkTFJHZEJabUZhVUhGWE1IbDJUVEZDTWtNd1VFeEdjVlZuZEV0dmEwZGFWRkVLZVN0a00xSndSMUZCVjBrNVNXTjFWV0o2YVZWRGFpdE9ZemxvY2tOTU9XOW5VbVozVWl0WGEyOTRVRWRwWXpCcGFXYzlQUW90TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09CiAgICBjbGllbnQta2V5LWRhdGE6IExTMHRMUzFDUlVkSlRpQlNVMEVnVUZKSlZrRlVSU0JMUlZrdExTMHRMUXBOU1VsRmIyZEpRa0ZCUzBOQlVVVkJNVVUyVDBOMU9VbE5lR3M0UWtJM1EwRnFUME5HY2tkUGFVazBlakZoUVhOTVlXSktZa3hvU1VWeVpGa3hWbXB6Q2xoUlJFWjRLMWh3U1RaQlYyaERXRWhCV1RoR2IyRkJZa05OUmxoM1JHWnNXRlF2TWtOUllYRTNNemRCSzJwQk0zWkdaMUFyTmxoalRrZGxUblpSVUZRS1VVUjJRMEZGVjFGMlZtbFlTbkJVTUUxM1lXb3hjWEZITjJwdFJsZG9jWFp2VVRKV1RWcHZNRVUxWVdscGRUbFlRbkVyZFdwT1YxUnRabVpQYUZoRGJ3cElUMk5sZUROVlVHUjZlQzl2Vkd4d1pVbFJZVmR2VGt4a1luSjFVR1J6WkVNd1ZrNHZSRFU0TVRrNWNEUlpNVkZYUjFOM1ltSjBjWGxwZEhwM2MwRjFDalpwWVM4MFlsUTBkblFyZHpSek1HbDNiVEp5Y210SmRTOHJUVTF3ZVdwQ2QyVmhaR050YTFsMVNqRlhRV3hpWTJWUFkyUjNiWGxWTUZCeWJIQjZUM0VLZDBOQlFqaFBPVzFwVHpjMUx6aFRaa2MxYTJwVGRrc3lUMmN5ZFhSdldYb3pSVlJ5ZW5kSlJFRlJRVUpCYjBsQ1FVaEljWFZMU3pkWk9WVjJVRUp1UmdveWVtNU9TVFJrUkhaSlVXWk5WV0l5UXprMWRTdFFUbWxMYVVsSmVWSjVieXQ1UzI5RlNHWkVUVWxoVTBaeFRGQnNiVzFMYzFjeldUazRhVW96WVdwb0NrWlRkMnMzYzBwSVJuQTRkRTU1VkVocVNrSTVMMEV4SzBsS2NuQXlXVGRNU2paWU1uRndSREIzYzFsdFlTdDNhWHB4YXpKaVpVUTBXRlJCYTBsNE5rY0tOVk5uV1c1eE5HWXpUVTlLT1hsMGRsbDRXRTlZUnpGeVEwTnFOMkZTYlVaSWJtWkVRM2hsYldoSEswcFhNMGRCT1VkU1ZuZDNORGd4V25WRGRrSnFRd3BPYjNKWVprSmxRVUppTlVKT1dVeEZWek1yWjFaUGIxQXdTbTFzU1dneVNsVTJja0ZXTWxwMFYxQmFVRkYxVFVWYVREZDBjSG8zUzNaRlRIbzBZVUp2Q2xWTlRYazVlRlpKWjBrNGFFVmlURzlTZEd3MmRETlRUV2xVUnpONVp6ZzFOSFkwVWtvd1owOUpiMGhLWWswd2JrRlZTRWxCT0VNeVVWUkRaREJJVG1rS1VUTkJObUpxUlVObldVVkJPVVl5YkVwbVdFNXlXamRxZVhOTFFVbE1NWFJYVTFKUmRHaGtjemQ0SzFBeU5VaHRVR2hwWjJRNVJXbG9hMHhLWkVSdFFRcE1kV3BKUW1wWVkzcHZhbEF6ZHpKa01VMTZaVlZrWkVsWGR6Qk1PWEU0YjBodmNETXJWMk5IYW5ObWFYRkRhMnBUY1N0alZUbHpNV2hGVERKeFdteGpDazVuTVRKWVV6QnBWMndyVVZsRVlrZGtWa3htY1RObVJUVTVPRTlyZG5GWlNEUmhObmg1VjA4NWJ6RkRla2QxT0hsTWFVbzNXR05EWjFsRlFUTnRiM01LUkdrNFNuZFVablZXTlZWcGFqQm1hRVY0VVdjM2VVTmxSbEY0TUhjemRtbGxUMDgzTDAxU1Uyc3lVVTFYSzJzME9XaHpVMHR5Y1VkaVJqaHBabW8xTVFvd2NHTkpZemxETDA5QmQwZE5jWE52V1hCMU5IUkNNM0JrY20xdlNFbHFNSHB4ZWtGVWRXTXpPWEUzT1hCRFdFSnFhR3hsYUNzclVtOWpPREpHTjBkT0NuUnJMMGRFTlc1Mk9UVkJRelUzYms5U1ZrcFBWSEV2Tkd4TlJGUnplRXRzVlZsYWNrdHRhME5uV1VGaEwwRkNVbkZqZDFadlVtTmpaVGd2TUZSYVlUSUtiMnRxTjFseGFsZ3lkemxwZFhORVNFZExZMWgwUWs1M05tY3daVGxZTmxCTU5IUjJSRU5IWlN0UlMycGhabWxGTWtRMmQyOVBjRkJwTm1oSWFrWTBUUXA1Vms5dlUyUlZlV3BUV1dSV1lsZGpLMHhzTVd4U05XRjRXalZDWVdwelFtbFJhMHRFZDB4a1NXWktjV1paU0RZd1RGUTRWbFpTT0VkdGFXTnRSMjFYQ2tOaFZ6QmxhRkpKWVRVM01YWmxkbmcyYkdKdmVuZExRbWRGT0hWMFYybElWRzh5TDJZNVkyMW1VblJPYkRndldFSnlTMkpvYVhkM1RuaEhlR1JVZEdNS2RVaDVVVEpIVlc4MVZUSkVhRzltTTNNd2QweEJMMmh3Um1weGFuVm5ZWHBTYjNOVk9HbzNaRFpSWWpRNFNGQkxlRTFOTjAxMllsWjFaMmRQY1dwck5ncFFNbnBZTTBwRGVWWkNTM2RpYlhoTmF6azRTbG95ZFZndlkxbEtja1UwVUhOSkt6RlhhakZyZHl0UlJIWnRMMjVyV1VaeFNFeEhOSGhMVjI5TFluTlJDbWNyTm1oQmIwZEJaQ3RyV1M5alJXSkROMFU1UWtobmRFcDNjazV1UjBKc01FWTFZVkpCVDNFMGFGZG5hamc0Y0ZGUlRXRXpXa2xqZDBScloyOXVSbG9LVDNWMVYxSkdNR2hNVUdaUmQzSXpiRFZJY0hvM2QxSXpSbWxXZW1ZeU5uQXJVVGxYUVU5TU0xVjZWVVJQYmtzMFNrNTFhblZaTkZSUlRubEdPWGxLYkFwaVJUTk9NRlI1ZUU5Nk0xZE9SbFYyUjJoa09HMXVNV3hPUldsNmQyZGxOMEZQT1ZkWGFtUkxOVkZDTW1GaE1rZEpSbk05Q2kwdExTMHRSVTVFSUZKVFFTQlFVa2xXUVZSRklFdEZXUzB0TFMwdENnPT0K
caData:
tokenData:
certData:
keyData:
syncResources:
- group: apps
resources:
- "*"
- group: ""
resources:
- "*"
查看数据
# kubectl exec -it clusterpedia-internalstorage-mysql-7f675744f9-bsqqc -n clusterpedia-system -- bash
Defaulted container "mysql" out of: mysql, restic-wait (init)
bash-4.4# mysql -uroot -p'dangerous0'
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor. Commands end with ; or \g.
Your MySQL connection id is 11
Server version: 8.1.0 MySQL Community Server - GPL
Copyright (c) 2000, 2023, Oracle and/or its affiliates.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| clusterpedia |
| information_schema |
| mysql |
| performance_schema |
| sys |
+--------------------+
5 rows in set (0.00 sec)
mysql> use clusterpedia;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> select count(1) from clusterpedia.resources;
+----------+
| count(1) |
+----------+
| 127 |
+----------+
1 row in set (0.00 sec)
mysql> select r.id, r.group, r.version, r.resource, r.kind, r.name, r.namespace from clusterpedia.resources r order by id limit 10;
+----+-------+---------+-------------------+------------------+------------------------------------+---------------------+
| id | group | version | resource | kind | name | namespace |
+----+-------+---------+-------------------+------------------+------------------------------------+---------------------+
| 1 | | v1 | secrets | Secret | cloud-credentials | velero |
| 2 | | v1 | endpoints | Endpoints | kubernetes | default |
| 3 | | v1 | events | Event | master.178cefddd0563c74 | default |
| 4 | | v1 | secrets | Secret | velero-restic-credentials | velero |
| 5 | | v1 | endpoints | Endpoints | kube-dns | kube-system |
| 6 | | v1 | persistentvolumes | PersistentVolume | clusterpedia-internalstorage-mysql | |
| 7 | | v1 | secrets | Secret | internalstorage-password | clusterpedia-system |
| 8 | | v1 | endpoints | Endpoints | minio | velero |
| 9 | | v1 | serviceaccounts | ServiceAccount | node-controller | kube-system |
| 10 | apps | v1 | replicasets | ReplicaSet | minio-7568b7d594 | velero |
+----+-------+---------+-------------------+------------------+------------------------------------+---------------------+
10 rows in set (0.00 sec)
mysql> select r.id, r.group, r.version, r.resource, r.kind, r.name, r.namespace from clusterpedia.resources r order by id;
+-----+-------+---------+------------------------+-----------------------+-----------------------------------------------------+---------------------+
| id | group | version | resource | kind | name | namespace |
+-----+-------+---------+------------------------+-----------------------+-----------------------------------------------------+---------------------+
| 1 | | v1 | secrets | Secret | cloud-credentials | velero |
| 2 | | v1 | endpoints | Endpoints | kubernetes | default |
| 3 | | v1 | events | Event | master.178cefddd0563c74 | default |
| 4 | | v1 | secrets | Secret | velero-restic-credentials | velero |
| 5 | | v1 | endpoints | Endpoints | kube-dns | kube-system |
| 6 | | v1 | persistentvolumes | PersistentVolume | clusterpedia-internalstorage-mysql | |
| 7 | | v1 | secrets | Secret | internalstorage-password | clusterpedia-system |
| 8 | | v1 | endpoints | Endpoints | minio | velero |
| 9 | | v1 | serviceaccounts | ServiceAccount | node-controller | kube-system |
| 10 | apps | v1 | replicasets | ReplicaSet | minio-7568b7d594 | velero |
| 11 | | v1 | persistentvolumeclaims | PersistentVolumeClaim | internalstorage-mysql | clusterpedia-system |
| 12 | | v1 | namespaces | Namespace | kube-public | |
| 13 | apps | v1 | controllerrevisions | ControllerRevision | restic-688ffc5bd | velero |
| 14 | apps | v1 | daemonsets | DaemonSet | kube-proxy | kube-system |
| 15 | | v1 | services | Service | kubernetes | default |
| 16 | | v1 | nodes | Node | master | |
| 17 | | v1 | configmaps | ConfigMap | extension-apiserver-authentication | kube-system |
| 18 | | v1 | endpoints | Endpoints | clusterpedia-internalstorage-mysql | clusterpedia-system |
| 19 | | v1 | namespaces | Namespace | kube-node-lease | |
| 20 | apps | v1 | replicasets | ReplicaSet | minio-57c84f9cf8 | velero |
| 21 | | v1 | serviceaccounts | ServiceAccount | ttl-after-finished-controller | kube-system |
| 22 | apps | v1 | controllerrevisions | ControllerRevision | kube-proxy-65bd75fd4 | kube-system |
| 23 | | v1 | services | Service | kube-dns | kube-system |
| 24 | | v1 | pods | Pod | coredns-5bbd96d687-xvhw5 | kube-system |
| 25 | apps | v1 | daemonsets | DaemonSet | calico-node | kube-system |
| 26 | | v1 | serviceaccounts | ServiceAccount | token-cleaner | kube-system |
| 27 | | v1 | services | Service | minio | velero |
| 28 | | v1 | namespaces | Namespace | default | |
| 29 | | v1 | endpoints | Endpoints | clusterpedia-apiserver | clusterpedia-system |
| 30 | apps | v1 | replicasets | ReplicaSet | clusterpedia-internalstorage-mysql-7f675744f9 | clusterpedia-system |
| 31 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | kube-system |
| 32 | | v1 | serviceaccounts | ServiceAccount | pvc-protection-controller | kube-system |
| 33 | apps | v1 | controllerrevisions | ControllerRevision | calico-node-dd8ff9d77 | kube-system |
| 34 | apps | v1 | daemonsets | DaemonSet | restic | velero |
| 35 | | v1 | namespaces | Namespace | velero | |
| 36 | | v1 | serviceaccounts | ServiceAccount | default | kube-node-lease |
| 37 | apps | v1 | replicasets | ReplicaSet | clusterpedia-apiserver-5bd88d6488 | clusterpedia-system |
| 38 | | v1 | pods | Pod | calico-node-gvdd9 | kube-system |
| 39 | | v1 | services | Service | clusterpedia-internalstorage-mysql | clusterpedia-system |
| 40 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | default |
| 41 | | v1 | namespaces | Namespace | clusterpedia-system | |
| 42 | | v1 | serviceaccounts | ServiceAccount | bootstrap-signer | kube-system |
| 43 | apps | v1 | replicasets | ReplicaSet | calico-kube-controllers-79568db7f8 | kube-system |
| 44 | | v1 | services | Service | clusterpedia-apiserver | clusterpedia-system |
| 45 | | v1 | namespaces | Namespace | kube-system | |
| 46 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | velero |
| 47 | | v1 | serviceaccounts | ServiceAccount | horizontal-pod-autoscaler | kube-system |
| 48 | | v1 | pods | Pod | velero-55796c7c76-v9nfg | velero |
| 49 | apps | v1 | replicasets | ReplicaSet | minio-5d66fd5444 | velero |
| 50 | | v1 | configmaps | ConfigMap | kubeadm-config | kube-system |
| 51 | | v1 | serviceaccounts | ServiceAccount | ephemeral-volume-controller | kube-system |
| 52 | apps | v1 | replicasets | ReplicaSet | minio-6c866cffdd | velero |
| 53 | | v1 | pods | Pod | etcd-master | kube-system |
| 54 | | v1 | configmaps | ConfigMap | kube-proxy | kube-system |
| 55 | | v1 | serviceaccounts | ServiceAccount | service-account-controller | kube-system |
| 56 | apps | v1 | replicasets | ReplicaSet | velero-55796c7c76 | velero |
| 57 | | v1 | pods | Pod | calico-kube-controllers-79568db7f8-zxx7p | kube-system |
| 58 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | kube-node-lease |
| 59 | | v1 | serviceaccounts | ServiceAccount | endpoint-controller | kube-system |
| 60 | apps | v1 | replicasets | ReplicaSet | clusterpedia-controller-manager-846c98df9d | clusterpedia-system |
| 61 | | v1 | serviceaccounts | ServiceAccount | service-controller | kube-system |
| 62 | | v1 | pods | Pod | minio-6d4696f57f-g64q9 | velero |
| 63 | | v1 | configmaps | ConfigMap | kubelet-config | kube-system |
| 64 | apps | v1 | replicasets | ReplicaSet | minio-6d4696f57f | velero |
| 65 | | v1 | serviceaccounts | ServiceAccount | resourcequota-controller | kube-system |
| 66 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | kube-public |
| 67 | | v1 | pods | Pod | restic-k9sw9 | velero |
| 68 | apps | v1 | replicasets | ReplicaSet | coredns-5bbd96d687 | kube-system |
| 69 | | v1 | serviceaccounts | ServiceAccount | expand-controller | kube-system |
| 70 | | v1 | configmaps | ConfigMap | calico-config | kube-system |
| 71 | | v1 | pods | Pod | kube-scheduler-master | kube-system |
| 72 | | v1 | serviceaccounts | ServiceAccount | namespace-controller | kube-system |
| 73 | apps | v1 | replicasets | ReplicaSet | minio-75964b589d | velero |
| 74 | | v1 | configmaps | ConfigMap | cluster-info | kube-public |
| 75 | | v1 | serviceaccounts | ServiceAccount | replicaset-controller | kube-system |
| 76 | apps | v1 | replicasets | ReplicaSet | minio-9d5556dfc | velero |
| 77 | | v1 | pods | Pod | kube-apiserver-master | kube-system |
| 78 | | v1 | configmaps | ConfigMap | coredns | kube-system |
| 79 | | v1 | serviceaccounts | ServiceAccount | attachdetach-controller | kube-system |
| 80 | apps | v1 | replicasets | ReplicaSet | clusterpedia-clustersynchro-manager-57f7b7bd4 | clusterpedia-system |
| 81 | | v1 | serviceaccounts | ServiceAccount | calico-node | kube-system |
| 82 | | v1 | pods | Pod | kube-proxy-qctcj | kube-system |
| 83 | | v1 | configmaps | ConfigMap | kube-root-ca.crt | clusterpedia-system |
| 84 | | v1 | serviceaccounts | ServiceAccount | default | velero |
| 85 | | v1 | pods | Pod | clusterpedia-controller-manager-846c98df9d-v5v8m | clusterpedia-system |
| 86 | | v1 | configmaps | ConfigMap | clusterpedia-internalstorage | clusterpedia-system |
| 87 | | v1 | serviceaccounts | ServiceAccount | certificate-controller | kube-system |
| 88 | | v1 | serviceaccounts | ServiceAccount | replication-controller | kube-system |
| 89 | | v1 | pods | Pod | kube-controller-manager-master | kube-system |
| 90 | | v1 | serviceaccounts | ServiceAccount | calico-kube-controllers | kube-system |
| 91 | | v1 | pods | Pod | coredns-5bbd96d687-5vcst | kube-system |
| 92 | | v1 | serviceaccounts | ServiceAccount | coredns | kube-system |
| 93 | | v1 | serviceaccounts | ServiceAccount | kube-proxy | kube-system |
| 94 | | v1 | pods | Pod | clusterpedia-internalstorage-mysql-7f675744f9-bsqqc | clusterpedia-system |
| 95 | | v1 | serviceaccounts | ServiceAccount | cronjob-controller | kube-system |
| 96 | | v1 | pods | Pod | clusterpedia-apiserver-5bd88d6488-rqcww | clusterpedia-system |
| 97 | | v1 | serviceaccounts | ServiceAccount | pv-protection-controller | kube-system |
| 98 | | v1 | pods | Pod | clusterpedia-clustersynchro-manager-57f7b7bd4-dgrzc | clusterpedia-system |
| 99 | | v1 | serviceaccounts | ServiceAccount | default | kube-system |
| 100 | | v1 | serviceaccounts | ServiceAccount | clusterpedia-apiserver | clusterpedia-system |
| 101 | | v1 | serviceaccounts | ServiceAccount | persistent-volume-binder | kube-system |
| 102 | | v1 | serviceaccounts | ServiceAccount | clusterrole-aggregation-controller | kube-system |
| 103 | | v1 | serviceaccounts | ServiceAccount | daemon-set-controller | kube-system |
| 104 | | v1 | serviceaccounts | ServiceAccount | job-controller | kube-system |
| 105 | | v1 | serviceaccounts | ServiceAccount | root-ca-cert-publisher | kube-system |
| 106 | | v1 | serviceaccounts | ServiceAccount | endpointslicemirroring-controller | kube-system |
| 107 | | v1 | serviceaccounts | ServiceAccount | statefulset-controller | kube-system |
| 108 | | v1 | serviceaccounts | ServiceAccount | ttl-controller | kube-system |
| 109 | | v1 | serviceaccounts | ServiceAccount | velero | velero |
| 110 | | v1 | serviceaccounts | ServiceAccount | clusterpedia-controller-manager | clusterpedia-system |
| 111 | apps | v1 | deployments | Deployment | coredns | kube-system |
| 112 | | v1 | serviceaccounts | ServiceAccount | deployment-controller | kube-system |
| 113 | apps | v1 | deployments | Deployment | calico-kube-controllers | kube-system |
| 114 | | v1 | serviceaccounts | ServiceAccount | disruption-controller | kube-system |
| 115 | apps | v1 | deployments | Deployment | minio | velero |
| 116 | | v1 | serviceaccounts | ServiceAccount | generic-garbage-collector | kube-system |
| 117 | | v1 | serviceaccounts | ServiceAccount | default | kube-public |
| 118 | apps | v1 | deployments | Deployment | velero | velero |
| 119 | | v1 | serviceaccounts | ServiceAccount | default | default |
| 120 | apps | v1 | deployments | Deployment | clusterpedia-internalstorage-mysql | clusterpedia-system |
| 121 | | v1 | serviceaccounts | ServiceAccount | default | clusterpedia-system |
| 122 | apps | v1 | deployments | Deployment | clusterpedia-apiserver | clusterpedia-system |
| 123 | | v1 | serviceaccounts | ServiceAccount | endpointslice-controller | kube-system |
| 124 | apps | v1 | deployments | Deployment | clusterpedia-clustersynchro-manager | clusterpedia-system |
| 125 | | v1 | serviceaccounts | ServiceAccount | pod-garbage-collector | kube-system |
| 126 | apps | v1 | deployments | Deployment | clusterpedia-controller-manager | clusterpedia-system |
| 127 | | v1 | serviceaccounts | ServiceAccount | clusterpedia-clustersynchro-manager | clusterpedia-system |
+-----+-------+---------+------------------------+-----------------------+-----------------------------------------------------+---------------------+
127 rows in set (0.00 sec)
mysql>
创建查看备份
# velero get backup
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
my-backup-1009 Completed 0 0 2023-10-12 14:07:22 +0800 CST 25d default <none>
# velero backup create my-backup-1016 --include-namespaces clusterpedia-system --default-volumes-to-restic -n velero
Backup request "my-backup-1016" submitted successfully.
Run `velero backup describe my-backup-1016` or `velero backup logs my-backup-1016` for more details.
# velero get backup
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
my-backup-1009 Completed 0 0 2023-10-12 14:07:22 +0800 CST 25d default <none>
my-backup-1016 Completed 0 0 2023-10-16 16:11:58 +0800 CST 29d default <none>
删除clusterpedia服务
# delete compontents
kubectl delete -f ./deploy/clusterpedia_apiserver_apiservice.yaml
kubectl delete -f ./deploy/clusterpedia_apiserver_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_clustersynchro_manager_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_controller_manager_deployment.yaml
kubectl delete -f ./deploy/clusterpedia_apiserver_rbac.yaml
# delete crds
kubectl delete -f ./deploy/cluster.clusterpedia.io_clustersyncresources.yaml
kubectl delete -f ./deploy/cluster.clusterpedia.io_pediaclusers.yaml
kubectl delete -f ./deploy/policy.clusterpedia.io_clusterimportpolicies.yaml
kubectl delete -f ./deploy/policy.clusterpedia.io_pediaclusterlifecycles.yaml
# 卸载存储组件
kubectl delete -f ./deploy/internalstorage/mysql
# 清理 Local PV 以及数据
kubectl delete pv clusterpedia-internalstorage-mysql
# 登录数据所在节点，清理数据
rm /var/local/clusterpedia/internalstorage/mysql
备份还原
velero restore create --from-backup my-backup-1009
Restore request "my-backup-1009-20221009153041" submitted successfully.
Run `velero restore describe my-backup-1009-20221009153041` or `velero restore logs my-backup-1009-20221009153041` for more details.
velero get restore
NAME BACKUP STATUS STARTED COMPLETED ERRORS WARNINGS CREATED SELECTOR
my-backup-1009-20221009153041 my-backup-1009 InProgress 2022-10-09 15:30:41 +0800 +08 <nil> 0 0 2022-10-09 15:30:41 +0800 +08 <none>
velero get restore
NAME BACKUP STATUS STARTED COMPLETED ERRORS WARNINGS CREATED SELECTOR
my-backup-1009-20221009153041 my-backup-1009 Completed 2022-10-09 15:30:41 +0800 +08 2022-10-09 15:35:28 +0800 +08 0 0 2022-10-09 15:30:41 +0800 +08 <none>
验证服务
验证数据
可以看到我们的创建的 Clusterpedia 服务，数据依然存在，证明已经完全恢复了。 只要将每个 Velero 实例指向相同的对象存储（MinIO），Velero 就能将资源从一个群集迁移到另一个群集。此外还支持定时备份，触发备份 Hooks 等操作，更多资料请查阅官方文档：https://velero.io/docs/
限制/注意事项
Velero 支持 BackupStorageLocation 的多个凭证，按需指定使用，但使用此功能需要插件支持所使用的对象存储服务
Velero 仅支持 VolumeSnapshotLocations 的一组凭据。Velero 将始终使用安装时提供的凭据（存储在 cloud-credentials 密钥中）进行卷快照
卷快照仍然受到提供商允许您创建快照的位置的限制。例如，AWS 和 Azure 不允许您在与卷所在的区域不同的区域中创建卷快照。
每个 Velero 备份有一个 BackupStorageLocation，一个 VolumeSnapshotLocation。不可能同时将单个 Velero 备份发送到多个备份存储位置，或者同时将单个卷快照发送到多个位置。但是可以设置多个存储位置不同的计划备份。
不支持跨提供商快照。如果您的集群具有多种类型的卷，例如 EBS 和 Portworx，但您只有为 EBS 配置的 VolumeSnapshotLocation，则 Velero 将仅对 EBS 卷进行快照。
Restic 数据存储在 Velero 主存储桶的前缀/子目录下，并将进入与用户在创建备份时选择的 BackupStorageLocation 对应的存储桶中。
Velero 的备份分为两部分——存储在对象存储中的元数据，以及持久卷数据的快照/备份。目前，Velero 本身并没有加密它们中的任何一个，而是依赖于对象和快照系统中的本机机制。一种特殊情况是 restic，它在文件系统级别备份持久卷数据并将其发送到 Velero 的对象存储。
Velero 对对象元数据的压缩是有限的，使用 Golang 的 tar 实现。在大多数情况下，Kubernetes 对象的大小限制为 1.5MB，但很多都没有达到这个要求，这意味着可能不需要压缩。请注意，restic 尚未实现压缩，但确实具有重复数据删除功能。
如果配置了多个 VolumeSnapshotLocation，则在创建备份时必须始终指定有效的 VolumeSnapshotLocation，即使使用 Restic 进行卷备份也是如此。可以配置 –default-volume-snapshot-locations 指定在创建备份时默认 VolumeSnapshotLocation。如果只有一个 VolumeSnapshotLocation，Velero 将自动使用该位置作为默认位置。
定时备份
// 每天一点备份，备份保存时间两天
[root@master-1 ~]# velero create schedule 103-all-app --schedule="0 1 * * *" --ttl 48h
Schedule "103-all-app" created successfully.
// 查看备份策略
[root@master-1 ~]# velero get schedule
NAME STATUS CREATED SCHEDULE BACKUP TTL LAST BACKUP SELECTOR
103-all-app Enabled 2024-05-21 17:36:59 +0800 CST 0 1 * * * 48h0m0s n/a <none>
// 备份成功查询
[root@master-1 ~]# velero get schedule
NAME STATUS CREATED SCHEDULE BACKUP TTL LAST BACKUP SELECTOR
103-all-app Enabled 2024-05-21 17:36:59 +0800 CST 0 1 * * * 48h0m0s 44m ago <none>
[root@master-1 ~]#
[root@master-1 ~]# velero get backup
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
103-all-app-20240522010039 Completed 0 0 2024-05-22 09:00:39 +0800 CST 1d default <none>
clusterpedia-system-bak240520 Completed 0 0 2024-05-20 22:28:15 +0800 CST 28d default <none>
k8s103cluster-bak240520 Completed 0 0 2024-05-20 16:25:06 +0800 CST 28d default <none>
kube-flannel-bak240520 Completed 0 0 2024-05-20 22:29:21 +0800 CST 28d default <none>
kube-system-bak240520 Completed 0 0 2024-05-20 22:28:50 +0800 CST 28d default <none>
kubernetes-dashboard-bak240520 Completed 0 0 2024-05-20 22:29:50 +0800 CST 28d default <none>
[root@master-1 ~]#
基于minio备份其他k8s集群测试
在其他k8s集群服务器只安装velero server
velero install --provider aws --bucket velero --image velero/velero:v1.8.1 --plugins velero/velero-plugin-for-aws:v1.4.1 --namespace velero --secret-file /root/wnh/k8s/velero/credentials-velero --use-volume-snapshots=false --default-volumes-to-restic=true --kubeconfig=/root/.kube/config --use-restic --backup-location-config region=minio,s3ForcePathStyle="true",s3Url=http://10.2.6.103:30080
// 验证测试通过103集群，执行备份104上面的tekton
# velero backup create from103-backup-cluster104 --include-namespaces tekton-pipelines -n velero --kubeconfig /root/wnh/from104test/admin.conf
[root@master-1 from104test]# velero get backup
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
103-all-app-20240522010039 Completed 0 0 2024-05-22 09:00:39 +0800 CST 1d default <none>
clusterpedia-system-bak240520 Completed 0 0 2024-05-20 22:28:15 +0800 CST 28d default <none>
from103-backup-cluster104 Completed 0 0 2024-05-22 17:27:06 +0800 CST 29d default <none>
k8s103cluster-bak240520 Completed 0 0 2024-05-20 16:25:06 +0800 CST 27d default <none>
kube-flannel-bak240520 Completed 0 0 2024-05-20 22:29:21 +0800 CST 28d default <none>
kube-system-bak240520 Completed 0 0 2024-05-20 22:28:50 +0800 CST 28d default <none>
kubernetes-dashboard-bak240520 Completed 0 0 2024-05-20 22:29:50 +0800 CST 28d default <none>
--include-namespaces *
表示包含所有namespace，默认default是*
[root@master-1 from104test]# velero backup describe from103-backup-all-cluster104 -n velero --kubeconfig /root/wnh/from104test/admin.conf
Name: from103-backup-all-cluster104
Namespace: velero
Labels: velero.io/storage-location=default
Annotations: velero.io/source-cluster-k8s-gitversion=v1.26.6
velero.io/source-cluster-k8s-major-version=1
velero.io/source-cluster-k8s-minor-version=26
Phase: InProgress
Errors: 0
Warnings: 0
Namespaces:
Included: *
Excluded: <none>
Resources:
Included: *
Excluded: <none>
Cluster-scoped: auto
Label selector: <none>
Storage Location: default
Velero-Native Snapshot PVs: auto
TTL: 720h0m0s
Hooks: <none>
Backup Format Version: 1.1.0
Started: 2024-05-22 17:49:42 +0800 CST
Completed: <n/a>
Expiration: 2024-06-21 17:49:42 +0800 CST
Estimated total items to be backed up: 1237
Items backed up so far: 45
Velero-Native Snapshots: <none included>
Restic Backups (specify --details for more information):
Completed: 91
Failed: 1
In Progress: 1
In Progress: 1
此时在备份103节点无法查看备份的数据状态，可以在备份的目标集群上查看备份的数据
[root@master velero]# velero get backup
NAME STATUS ERRORS WARNINGS CREATED EXPIRES STORAGE LOCATION SELECTOR
103-all-app-20240522010039 Completed 0 0 2024-05-22 09:00:39 +0800 CST 1d default <none>
clusterpedia-system-bak240520 Completed 0 0 2024-05-20 22:28:15 +0800 CST 28d default <none>
from103-backup-all-cluster104 InProgress 0 0 2024-05-22 17:49:42 +0800 CST 29d default <none>
[root@master-1 from104test]# velero backup describe from103-backup-all-cluster104 -n velero --kubeconfig /root/wnh/from104test/admin.conf --details
Name: from103-backup-all-cluster104
Namespace: velero
Labels: velero.io/storage-location=default
Annotations: velero.io/source-cluster-k8s-gitversion=v1.26.6
velero.io/source-cluster-k8s-major-version=1
velero.io/source-cluster-k8s-minor-version=26
Phase: InProgress
Errors: 0
Warnings: 0
Namespaces:
Included: *
Excluded: <none>
Resources:
Included: *
Excluded: <none>
Cluster-scoped: auto
Label selector: <none>
Storage Location: default
Velero-Native Snapshot PVs: auto
TTL: 720h0m0s
Hooks: <none>
Backup Format Version: 1.1.0
Started: 2024-05-22 17:49:42 +0800 CST
Completed: <n/a>
Expiration: 2024-06-21 17:49:42 +0800 CST
Estimated total items to be backed up: 1241
Items backed up so far: 56
Resource List: <backup resource list not found>
Velero-Native Snapshots: <none included>
Restic Backups:
Completed:
clusterpedia-system/clusterpedia-internalstorage-mysql-7f675744f9-bsqqc: data
default/details-v1-7745b6fcf4-lzgrh: credential-socket, istio-data, istio-envoy, workload-certs, workload-socket
default/gitlab-shared-secrets-1-3q5-zh8jr: credential-socket, istio-data, istio-envoy, ssh, workload-certs, workload-socket
default/gitlab-shared-secrets-1-7k1-7mr8d: credential-socket, istio-data, istio-envoy, ssh, workload-certs, workload-socket
default/gitlab-shared-secrets-1-gmm-jhcp9: credential-socket, istio-data, istio-envoy, ssh, workload-certs, workload-socket
default/gitlab-shared-secrets-1-jsi-fjmgf: credential-socket, istio-data, istio-envoy, ssh, workload-certs, workload-socket
default/loki-0: credential-socket, istio-data, istio-envoy, storage, tmp, workload-certs, workload-socket
default/productpage-v1-6f89b6c557-x4kvq: credential-socket, istio-data, istio-envoy, tmp, workload-certs, workload-socket
default/ratings-v1-77bdbf89bb-bxfvm: credential-socket, istio-data, istio-envoy, workload-certs, workload-socket
default/reviews-v1-667b5cc65d-9vlwh: credential-socket, istio-data, istio-envoy, tmp, wlp-output, workload-certs, workload-socket
default/reviews-v2-6f76498fc8-46bj5: credential-socket, istio-data, istio-envoy, tmp, wlp-output, workload-certs, workload-socket
default/reviews-v3-5d8667cc66-68pxm: credential-socket, istio-data, istio-envoy, tmp, wlp-output, workload-certs, workload-socket
gitlab/gitlab-gitaly-0: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, gitaly-secrets, repo-data
gitlab/gitlab-gitlab-exporter-f9bf968fd-xkxld: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, gitlab-exporter-secrets
gitlab/gitlab-gitlab-runner-75cccf78cc-6fdgm: etc-gitlab-runner
gitlab/gitlab-gitlab-shell-c6f75f6f5-469tn: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, shell-secrets
gitlab/gitlab-gitlab-shell-c6f75f6f5-rkf8m: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, shell-secrets
gitlab/gitlab-kas-568568cdd4-dtwc2: etc-pki-ca-trust-extracted-pem, etc-ssl-certs
gitlab/gitlab-kas-568568cdd4-n4tn5: etc-pki-ca-trust-extracted-pem, etc-ssl-certs
gitlab/gitlab-minio-7f7dbccc49-rxzl8: export, minio-server-config
gitlab/gitlab-postgresql-0: data, dshm
gitlab/gitlab-prometheus-server-6d8b995f5b-m9vbf: storage-volume
gitlab/gitlab-redis-master-0: redis-data, redis-tmp-conf, tmp
gitlab/gitlab-registry-86d954c6fd-svgz7: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, registry-server-config
gitlab/gitlab-registry-86d954c6fd-tz66l: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, registry-server-config
gitlab/gitlab-sidekiq-all-in-1-v2-798bdd6664-twthb: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, sidekiq-metrics, sidekiq-secrets
gitlab/gitlab-toolbox-868dd4dcd9-44v4q: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, toolbox-secrets, toolbox-tmp
gitlab/gitlab-webservice-default-84f594c764-c7bkn: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, shared-tmp, shared-upload-directory, webservice-metrics, webservice-secrets, workhorse-secrets
gitlab/gitlab-webservice-default-84f594c764-fhqdt: shared-tmp
Failed:
gitlab/gitlab-gitlab-runner-75cccf78cc-6fdgm: runner-secrets
In Progress:
gitlab/gitlab-webservice-default-84f594c764-fhqdt: webservice-metrics
New:
gitlab/gitlab-webservice-default-84f594c764-fhqdt: etc-pki-ca-trust-extracted-pem, etc-ssl-certs, shared-upload-directory, webservice-secrets, workhorse-secrets
[root@master-1 from104test]#
// 备份数据太大，删除备份执行日志
[root@master-1 gitlab]# velero backup delete from103-backup-all-cluster104 -n velero --kubeconfig /root/wnh/from104test/admin.conf
Are you sure you want to continue (Y/N)? Y
Request to delete backup "from103-backup-all-cluster104" submitted successfully.
The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed.
// 数据恢复测试
// 在103节点恢复104上删除的nginx应用，模拟故障恢复
// 先在103上面进行数据备份
[root@master-1 ~]# velero backup create from103-backup-app-nginx-cluster104 --include-namespaces ingress-nginx -n velero --kubeconfig /root/wnh/from104test/admin.conf
// 在104服务器上面查看应用的信息并访问测试
[root@master ~]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller NodePort 10.1.53.253 <none> 80:31511/TCP,443:32717/TCP 47s
ingress-nginx-controller-admission ClusterIP 10.1.231.73 <none> 443/TCP 47s
[root@master ~]# kubectl get pod -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-controller-6bfd758974-dwgws 1/1 Running 0 6m40s
// 如下链接地址访问正常
https://10.1.6.104:32717/
// 删除此namespace
[root@master ~]# kubectl delete namespace ingress-nginx
// 待完全删除干净后，访问页面，无法访问，做如下的恢复操作
// 在103节点进行恢复操作
[root@master-1 ~]# velero restore create --from-backup from103-backup-app-nginx-cluster104 --include-namespaces ingress-nginx --namespace velero --kubeconfig /root/wnh/from104test/admin.conf
Restore request "from103-backup-app-nginx-cluster104-20240523164442" submitted successfully.
Run `velero restore describe from103-backup-app-nginx-cluster104-20240523164442` or `velero restore logs from103-backup-app-nginx-cluster104-20240523164442` for more details.
[root@master-1 ~]#
// 在104节点查看应用恢复情况，并通过web进行访问验证，发现已经完成了备份恢复操作
参考
Velero官网
Clusterpedia
kubernetes备份恢复