From 1ea5fa49f4f1fd3727b08598e3dc6d7e61f5e032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=8C=AF=E5=AE=87?= <> Date: Tue, 14 Jan 2025 06:06:45 +0800 Subject: [PATCH] feat(k8s): add Azure Blob Storage CSI driver configuration and resources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 孙振宇 <> --- .../freeleaps-devops-system/jenkins/pv.yaml | 14 + .../azure-blob-storage-csi/.gitkeep | 0 .../azure-cloud-provider-secret.yaml | 8 + .../azure-json-base64.sh | 11 + .../azure-blob-storage-csi/azure.json | 12 + .../storage-classes.yaml | 268 ++++++++++++++++++ .../azure-blob-storage-csi/values.yaml | 186 ++++++++++++ 7 files changed, 499 insertions(+) create mode 100644 cluster/manifests/freeleaps-devops-system/jenkins/pv.yaml delete mode 100644 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/.gitkeep create mode 100644 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-cloud-provider-secret.yaml create mode 100755 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-json-base64.sh create mode 100644 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure.json create mode 100644 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/storage-classes.yaml create mode 100644 cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/values.yaml diff --git a/cluster/manifests/freeleaps-devops-system/jenkins/pv.yaml b/cluster/manifests/freeleaps-devops-system/jenkins/pv.yaml new file mode 100644 index 00000000..d4e41096 --- /dev/null +++ b/cluster/manifests/freeleaps-devops-system/jenkins/pv.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jenkins-pv + namespace: freeleaps-devops-system +spec: + storageClassName: freeleaps-node-local + accessModes: + - ReadWriteOnce + capacity: + storage: 20Gi + persistentVolumeReclaimPolicy: Retain + hostPath: + path: /mnt/data/jenkins \ No newline at end of file diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/.gitkeep b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-cloud-provider-secret.yaml b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-cloud-provider-secret.yaml new file mode 100644 index 00000000..f403868e --- /dev/null +++ b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-cloud-provider-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: azure-cloud-provider + namespace: freeleaps-storage-system +type: Opaque +data: + cloud-config: ewogICJjbG91ZCI6ICJBenVyZVB1YmxpY0Nsb3VkIiwKICAidGVuYW50SWQiOiAiY2YxNTFlZTgtNWMyYy00ZmU3LWExYzQtODA5YmE0M2M5ZjI0IiwKICAic3Vic2NyaXB0aW9uSWQiOiAiMGEyODAwNjgtZGVjNC00YmYwLTlmMDQtNjViNjRmNDEyYjUwIiwKICAicmVzb3VyY2VHcm91cCI6ICJrOHMiLAogICJsb2NhdGlvbiI6ICJ3ZXN0dXMyIiwKICAiYWFkQ2xpZW50SWQiOiAiN2NkMWRmMTktMjRlYS00NmQ3LWFjZDMtNTMzNjI4MzEzOWUwIiwKICAiYWFkQ2xpZW50U2VjcmV0IjogIjJWVThRflBDTXFFYmgtelpKTFlhRUVQQml6NTJJQzVieVp4c2liZlYiLAogICJ1c2VNYW5hZ2VkSWRlbnRpdHlFeHRlbnNpb24iOiBmYWxzZSwKICAidXNlckFzc2lnbmVkSWRlbnRpdHlJRCI6ICIiLAogICJ1c2VJbnN0YW5jZU1ldGFkYXRhIjogdHJ1ZQp9Cg== \ No newline at end of file diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-json-base64.sh b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-json-base64.sh new file mode 100755 index 00000000..c4fd6c1d --- /dev/null +++ b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure-json-base64.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# check if the azure.json not exist +if [ ! -f azure.json ]; then + echo "azure.json file not found, exiting" + exit 1 +fi + +cat azure.json | base64 | awk '{printf $0}'; echo \ No newline at end of file diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure.json b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure.json new file mode 100644 index 00000000..393d9fe7 --- /dev/null +++ b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/azure.json @@ -0,0 +1,12 @@ +{ + "cloud": "AzurePublicCloud", + "tenantId": "cf151ee8-5c2c-4fe7-a1c4-809ba43c9f24", + "subscriptionId": "0a280068-dec4-4bf0-9f04-65b64f412b50", + "resourceGroup": "k8s", + "location": "westus2", + "aadClientId": "7cd1df19-24ea-46d7-acd3-5336283139e0", + "aadClientSecret": "2VU8Q~PCMqEbh-zZJLYaEEPBiz52IC5byZxsibfV", + "useManagedIdentityExtension": false, + "userAssignedIdentityID": "", + "useInstanceMetadata": true +} diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/storage-classes.yaml b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/storage-classes.yaml new file mode 100644 index 00000000..89aadde7 --- /dev/null +++ b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/storage-classes.yaml @@ -0,0 +1,268 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-std-lrs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_LRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-premium-lrs +provisioner: blob.csi.azure.com +parameters: + skuName: Premium_LRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-standard-grs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_GRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-standard-ragrs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_RAGRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-standard-zrs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_ZRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-fuse-2-premium-zrs +provisioner: blob.csi.azure.com +parameters: + skuName: Premium_ZRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS + protocol: fuse2 +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-nfs +provisioner: blob.csi.azure.com +parameters: + protocol: nfs +volumeBindingMode: Immediate +allowVolumeExpansion: true +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-std-lrs +provisioner: blob.csi.azure.com +parameters: + skuName: Premium_LRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-premium-lrs +provisioner: blob.csi.azure.com +parameters: + skuName: Premium_LRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-standard-grs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_GRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-standard-ragrs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_RAGRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-standard-zrs +provisioner: blob.csi.azure.com +parameters: + skuName: Standard_ZRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azure-blob-premium-zrs +provisioner: blob.csi.azure.com +parameters: + skuName: Premium_ZRS # available values: Standard_LRS, Premium_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Premium_ZRS +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - -o allow_other + - --file-cache-timeout-in-seconds=120 + - --use-attr-cache=true + - --cancel-list-on-mount-seconds=10 # prevent billing charges on mounting + - -o attr_timeout=120 + - -o entry_timeout=120 + - -o negative_timeout=120 + - --log-level=LOG_WARNING # LOG_WARNING, LOG_INFO, LOG_DEBUG + - --cache-size-mb=1000 # Default will be 80% of available memory, eviction will happen beyond that. \ No newline at end of file diff --git a/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/values.yaml b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/values.yaml new file mode 100644 index 00000000..de871e69 --- /dev/null +++ b/cluster/manifests/freeleaps-storage-system/azure-blob-storage-csi/values.yaml @@ -0,0 +1,186 @@ +image: + baseRepo: mcr.microsoft.com + blob: + repository: /k8s/csi/blob-csi + tag: latest + pullPolicy: IfNotPresent + csiProvisioner: + repository: /oss/kubernetes-csi/csi-provisioner + tag: v5.1.0 + pullPolicy: IfNotPresent + livenessProbe: + repository: /oss/kubernetes-csi/livenessprobe + tag: v2.14.0 + pullPolicy: IfNotPresent + nodeDriverRegistrar: + repository: /oss/kubernetes-csi/csi-node-driver-registrar + tag: v2.12.0 + pullPolicy: IfNotPresent + csiResizer: + repository: /oss/kubernetes-csi/csi-resizer + tag: v1.12.0 + pullPolicy: IfNotPresent + +cloud: AzurePublicCloud + +## Reference to one or more secrets to be used when pulling images +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] +# - name: myRegistryKeySecretName + +serviceAccount: + create: true # When true, service accounts will be created for you. Set to false if you want to use your own. + controller: csi-blob-controller-sa # Name of Service Account to be created or used + node: csi-blob-node-sa # Name of Service Account to be created or used + +rbac: + create: true + name: blob + +## Collection of annotations to add to all the pods +podAnnotations: {} +## Collection of labels to add to all the pods +podLabels: {} +# -- Custom labels to add into metadata +customLabels: {} + # k8s-app: blob-csi-driver + +## Leverage a PriorityClass to ensure your pods survive resource shortages +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +priorityClassName: system-cluster-critical +## Security context give the opportunity to run container as nonroot by setting a securityContext +## by example : +## securityContext: { runAsUser: 1001 } +securityContext: {} + +controller: + name: csi-blob-controller + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: freeleaps-storage-system + allowEmptyCloudConfig: true + hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting + metricsPort: 29634 + livenessProbe: + healthPort: 29632 + replicas: 2 + runOnMaster: false + runOnControlPlane: true + logLevel: 5 + resources: + csiProvisioner: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 800Mi + requests: + cpu: 10m + memory: 20Mi + csiResizer: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + effect: "NoSchedule" + +node: + name: csi-blob-node + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: freeleaps-storage-system + allowEmptyCloudConfig: true + allowInlineVolumeKeyAccessWithIdentity: false + maxUnavailable: 1 + metricsPort: 29635 + livenessProbe: + healthPort: 29633 + logLevel: 5 + enableBlobfuseProxy: true + blobfuseProxy: + installBlobfuse: false + blobfuseVersion: "1.4.5" + installBlobfuse2: true + blobfuse2Version: "2.4.0" + setMaxOpenFileNum: true + maxOpenFileNum: "9000000" + disableUpdateDB: true + migrateK8sRepo: false + setReadAheadSize: true + blobfuseCachePath: /mnt + appendTimeStampInCacheDir: false + mountPermissions: 0777 + resources: + livenessProbe: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + blob: + limits: + memory: 2100Mi + requests: + cpu: 10m + memory: 20Mi + aznfswatchdog: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + affinity: {} + nodeSelector: {} + tolerations: + - operator: "Exists" + enableAznfsMount: true + +feature: + fsGroupPolicy: ReadWriteOnceWithFSType + enableGetVolumeStats: false + +driver: + name: blob.csi.azure.com + customUserAgent: "" + userAgentSuffix: "OSS-helm" + azureGoSDKLogLevel: "INFO" # available values: ""(no logs), DEBUG, INFO, WARNING, ERROR + httpsProxy: "" + httpProxy: "" + +linux: + kubelet: /var/lib/kubelet + distro: debian + +workloadIdentity: + clientID: "" + # [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster + # then set tenantID with the application or user-assigned managed identity tenant ID + tenantID: "" \ No newline at end of file