diff --git a/helm_charts/mdss-nfs-example.yml b/helm_charts/mdss-nfs-example.yml new file mode 100644 index 0000000..aa5a9f9 --- /dev/null +++ b/helm_charts/mdss-nfs-example.yml @@ -0,0 +1,15 @@ + + +mdss_components: + nfsservice: + extraVolumes: + nfsTest: + name: nfs-storage-unit + nfs: + server: + path: + readOnly: no + extraVolumeMounts: + - name: nfs-storage-unit + mountPath: /opt/mdss/nfs/ + subPath: \ No newline at end of file diff --git a/helm_charts/mdss/Chart.yaml b/helm_charts/mdss/Chart.yaml index 03a7411..84974f3 100644 --- a/helm_charts/mdss/Chart.yaml +++ b/helm_charts/mdss/Chart.yaml @@ -7,10 +7,6 @@ long_description: | - All MDSS services in separate pods - A MongoDB database instance pre-configured to be used by MDSS - In addition to the chart, we also provide a number of values files for specific scenarios: - - mdss-aws-eks-values.yml - for deploying in an AWS environment using Amazon EKS - - mdss-azure-aks-values.yml - for deploying in an Azure environment using AKS - ## Installation ### From source @@ -36,4 +32,4 @@ long_description: | type: application -version: 3.3.2-0 +version: 3.3.3-0 diff --git a/helm_charts/mdss/templates/deployments-template.yml b/helm_charts/mdss/templates/deployments-template.yml index 4bb325f..75fb8b8 100644 --- a/helm_charts/mdss/templates/deployments-template.yml +++ b/helm_charts/mdss/templates/deployments-template.yml @@ -13,7 +13,10 @@ spec: selector: matchLabels: app: {{ $component.name | quote }} - strategy: {} + {{- if $component.strategy}} + strategy: + {{- toYaml $component.strategy | nindent 4 }} + {{- end }} template: metadata: labels: @@ -26,16 +29,16 @@ spec: - envFrom: - configMapRef: name: {{ $.Values.mdss_config_map_env_name | quote }} - {{- if $component.env }} - env: - {{- toYaml $component.env | nindent 12 }} - {{- end }} - {{- if $component.custom_repo }} + {{ if $component.custom_repo -}} image: {{ printf "%s/%s:%s" $.Values.mdss_docker_repo $component.image (index $.Values "mdss-common-environment").BRANCH | quote }} - {{- else }} + {{- else -}} image: {{ $component.image | quote }} {{- end }} imagePullPolicy: {{ $.Values.imagePullPolicy }} + {{- if $component.env }} + env: + {{- toYaml $component.env | nindent 12 }} + {{- end }} name: {{ $component.name | quote }} {{- if $component.ports }} ports: @@ -43,6 +46,18 @@ spec: - containerPort: {{ .port }} {{- end }} {{- end }} + {{- if $component.livenessProbe }} + livenessProbe: + {{- toYaml $component.livenessProbe | nindent 12 }} + {{- end }} + {{- if $component.startupProbe }} + startupProbe: + {{- toYaml $component.startupProbe | nindent 12 }} + {{- end }} + {{- if $component.readinessProbe }} + readinessProbe: + {{- toYaml $component.readinessProbe | nindent 12 }} + {{- end }} {{- if $component.command }} command: {{- toYaml $component.command | nindent 12 }} @@ -55,7 +70,7 @@ spec: securityContext: {{- toYaml $component.securityContext | nindent 12 }} {{- end }} - {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.customMounts)}} + {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.extraVolumeMounts)}} volumeMounts: {{- if $component.mountConfig }} - name: {{ $component.mountConfig.configName }} @@ -69,14 +84,21 @@ spec: subPath: {{ $component.name }} {{- end }} {{ end }} - {{- if $component.customMounts }} - {{- toYaml $component.customMounts | nindent 12 }} + {{- if $component.extraVolumeMounts }} + {{- toYaml $component.extraVolumeMounts | nindent 12 }} {{- end }} {{- end }} {{- if $component.resources }} resources: {{- toYaml $component.resources | nindent 12 }} {{- end }} + {{- if $component.sidecars }} + {{- toYaml $component.sidecars | nindent 8 }} + {{- end }} + {{- if $component.initContainers }} + initContainers: + {{- toYaml $component.initContainers | nindent 8 }} + {{- end }} {{- if $component.update_ca }} lifecycle: postStart: @@ -91,7 +113,7 @@ spec: {{- if $component.customPodDefinitions }} {{- toYaml $component.customPodDefinitions | nindent 6 }} {{- end }} - {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.customVolumes)}} + {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.extraVolumes)}} volumes: {{- if $component.mountConfig }} - name: {{ $component.mountConfig.configName }} @@ -107,15 +129,18 @@ spec: {{ else }} - name: {{ $component.name }} persistentVolumeClaim: - claimName: {{ printf "%s-pvc" $.Values.storage_name }} - {{- end }} + claimName: {{ printf "%s" $.Values.storage_name }} + {{ end }} + {{ end }} + {{- if $component.extraVolumes -}} + {{- range $volumeLabel, $volume := $component.extraVolumes }} + - + {{- toYaml $volume | nindent 10 }} {{- end }} - {{- if $component.customVolumes }} - {{- toYaml $component.customVolumes | nindent 8 }} {{- end }} {{- end }} hostAliases: {{- toYaml $.Values.mdssHostAliases | nindent 8 }} {{ end }} -{{ end }} +{{ end }} \ No newline at end of file diff --git a/helm_charts/mdss/values.yaml b/helm_charts/mdss/values.yaml index 3a4450e..ee28f44 100644 --- a/helm_charts/mdss/values.yaml +++ b/helm_charts/mdss/values.yaml @@ -1,7 +1,7 @@ # Default values for mdss. These will also be included in the environment of each mdss container mdss-common-environment: - ENABLED_MODULES: "azureblob,amazonsdk" # Enabled modules by default + ENABLED_MODULES: "azureblob,amazonsdk,nfs" # Enabled modules by default # ENABLED_MODULES: "azureblob,amazonsdk,googlecloud,alibabacloud,azurefiles,smb,box,onedrive,sftp,debug" # All supported modules # This defines the connection string to the db ( should be modified when using an external db) @@ -39,7 +39,7 @@ mdss-common-environment: RABBITMQ_SCANNING_PREFETCH_COUNT: "20" HTTPS_ACTIVE: "no" - BRANCH: "3.3.0" # Sets a custom MDSS branch for testing/preview versions + BRANCH: "3.3.3" # Sets a custom MDSS branch for testing/preview versions # These variables can be changed to reveal debug information in the application LOG_LEVEL: "4" APP_LOG_LEVEL: "INFORMATION" @@ -110,6 +110,42 @@ mdss-common-environment: # This variable is used to force Signature Version 4 for AWS S3/S3 compatible request. FORCE_AWS_SIGNATURE_V4: "1" + NFSSERVICE_URL: "http://nfsservice" + NFSSERVICE_PORT: "5004" + + NFS_SHORT_DEADLINE: "5" + NFS_LONG_DEADLINE: "30" + + NFS_UPLOAD_CHUNK: "2" + + # This value is used to determine how MDSS consider that a file from a smb share is ready to be processed (Default 1-FILEOPEN_CHECK, 0-DEBOUNCE_CHECK) + SMBSERVICE_FILE_READY_CHECK_MODE: "1" + # This can be modified in order to define how many threads can be processed in parallel. + SMBSERVICE_PREFETCH_COUNT: "10" + + DISCOVERY_SERVICE_SFTP_RTP_HANDLING: "0" + DISCOVERY_SERVICE_NFS_RTP_HANDLING: "0" + + # Default number of hours for token expiration, maximum 24 hours + TOKEN_EXPIRY_TIME_HOURS: "1" + + # Discovery debounce time used in order to check if a file is uploaded or not, we check if the file has not been modified between a 5 seconds interval + SFTP_DISCOVERY_DEBOUNCE_TIME_SECONDS: "5" + NFS_DISCOVERY_DEBOUNCE_TIME_SECONDS: "5" + + # time to wait (in minutes) before retrying the register session method in SMB + SMBSERVICE_SESSION_RETRY_WAIT_TIME: "5" + + # maximum amount of retries for the register session method in SMB + SMBSERVICE_SESSION_MAX_RETRIES: "3" + + # number of active connections per smb storage; default is 10, maximum is 100 + SMBSERVICE_SESSIONS_ON_STORAGE: "10" + + # time interval between two consecutive job runs for the discovery completed job + DISCOVERY_COMPLETED_CHECK_SECONDS: "10" + + # Set to true if the respective database engine should also be deployed deploy_with_mdss_db: true # Enable or disable the local in-cluster database, set to false when deploying with an external database service @@ -748,4 +784,53 @@ mdss_components: memory: "0.25Gi" cpu: "0.500" - + # ======================================= NFS MODULE ================================================ + discoverynfsservice: + name: discoverynfsservice + module_name: nfs + custom_repo: true + image: mdcloudservices_discoverynfs + extra_labels: + aws-type: fargate + resources: + requests: + memory: "0.25Gi" + cpu: "1.000" + + remediationsservice-nfs: + name: remediationsservice-nfs + module_name: nfs + custom_repo: true + image: mdcloudservices_remediationsnfs + extra_labels: + aws-type: fargate + resources: + requests: + memory: "0.25Gi" + cpu: "0.100" + + storagesservice-nfs: + name: storagesservice-nfs + module_name: nfs + custom_repo: true + image: mdcloudservices_storagesnfs + extra_labels: + aws-type: fargate + resources: + requests: + memory: "0.25Gi" + cpu: "0.010" + + nfsservice: + name: nfsservice + module_name: nfs + custom_repo: true + image: mdcloudservices_nfsservice + ports: + - port: 5004 + extra_labels: + aws-type: fargate + resources: + requests: + memory: "0.5Gi" + cpu: "0.500" \ No newline at end of file diff --git a/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf b/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf index 0bacdcf..2dac2a6 100644 --- a/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf +++ b/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf @@ -183,6 +183,9 @@ resource "azurerm_eventgrid_system_topic_event_subscription" "eventgrid" { system_topic = azurerm_eventgrid_system_topic.topic.name resource_group_name = var.STORAGE_RG event_delivery_schema = "EventGridSchema" + included_event_types = [ + "Microsoft.Storage.BlobCreated" + ] subject_filter { subject_begins_with = "/blobServices/default/containers/${var.STORAGE_CONTAINERNAME}" }