RELEASE_26 (#37)

OPSWAT · Feb 21, 2024 · 928bea2 · 928bea2
1 parent ade82cf
commit 928bea2
Show file tree

Hide file tree

Showing 5 changed files with 148 additions and 24 deletions.
diff --git a/helm_charts/mdss-nfs-example.yml b/helm_charts/mdss-nfs-example.yml
@@ -0,0 +1,15 @@
+
+
+mdss_components:
+  nfsservice:
+    extraVolumes:
+      nfsTest:
+        name: nfs-storage-unit
+        nfs:
+          server: <NFS_SERVER_IP>
+          path: <PATH_EXPORTED_ON_SERVER>
+          readOnly: no
+    extraVolumeMounts:
+      - name: nfs-storage-unit
+        mountPath: /opt/mdss/nfs/<MOUNT_DIRECTORY>
+        subPath: <SUBPATH_TO_SCAN>
diff --git a/helm_charts/mdss/Chart.yaml b/helm_charts/mdss/Chart.yaml
@@ -7,10 +7,6 @@ long_description: |
   - All MDSS services in separate pods 
   - A MongoDB database instance pre-configured to be used by MDSS
 
-  In addition to the chart, we also provide a number of values files for specific scenarios:
-  - mdss-aws-eks-values.yml - for deploying in an AWS environment using Amazon EKS
-  - mdss-azure-aks-values.yml - for deploying in an Azure environment using AKS
-
   ## Installation
   
   ### From source
@@ -36,4 +32,4 @@ long_description: |
 
 type: application
 
-version: 3.3.2-0
+version: 3.3.3-0
diff --git a/helm_charts/mdss/templates/deployments-template.yml b/helm_charts/mdss/templates/deployments-template.yml
@@ -13,7 +13,10 @@ spec:
   selector:
     matchLabels:
       app: {{ $component.name | quote }}
-  strategy: {}
+  {{- if $component.strategy}}
+  strategy:
+    {{- toYaml $component.strategy | nindent 4 }}
+  {{- end }}
   template:
     metadata:
       labels:
@@ -26,23 +29,35 @@ spec:
         - envFrom:
             - configMapRef:
                 name: {{ $.Values.mdss_config_map_env_name | quote }}
-          {{- if $component.env }}
-          env:
-            {{- toYaml $component.env | nindent 12 }}
-          {{- end }}
-          {{- if $component.custom_repo }}
+          {{ if $component.custom_repo -}}
           image: {{ printf "%s/%s:%s" $.Values.mdss_docker_repo $component.image (index $.Values "mdss-common-environment").BRANCH | quote }}
-          {{- else }}
+          {{- else -}}
           image: {{ $component.image | quote }}
           {{- end }}
           imagePullPolicy: {{ $.Values.imagePullPolicy }}
+          {{- if $component.env }}
+          env:
+            {{- toYaml $component.env | nindent 12 }}
+          {{- end }}
           name: {{ $component.name | quote }}
           {{- if $component.ports }}
           ports:
           {{- range $component.ports }}
             - containerPort: {{ .port }}
           {{- end }}
           {{- end }}
+          {{- if $component.livenessProbe }}
+          livenessProbe:
+            {{- toYaml $component.livenessProbe | nindent 12 }}
+          {{- end }}
+          {{- if $component.startupProbe }}
+          startupProbe:
+            {{- toYaml $component.startupProbe | nindent 12 }}
+          {{- end }}
+          {{- if $component.readinessProbe }}
+          readinessProbe:
+            {{- toYaml $component.readinessProbe | nindent 12 }}
+          {{- end }}
           {{- if $component.command }}
           command: 
             {{- toYaml $component.command | nindent 12 }}
@@ -55,7 +70,7 @@ spec:
           securityContext:
             {{- toYaml $component.securityContext | nindent 12 }}
           {{- end }}
-          {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.customMounts)}}
+          {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.extraVolumeMounts)}}
           volumeMounts:
             {{- if $component.mountConfig }}
             - name: {{ $component.mountConfig.configName }}
@@ -69,14 +84,21 @@ spec:
               subPath: {{ $component.name }}
               {{- end }}
             {{ end }}
-            {{- if $component.customMounts }}
-              {{- toYaml $component.customMounts | nindent 12 }}
+            {{- if $component.extraVolumeMounts }}
+              {{- toYaml $component.extraVolumeMounts | nindent 12 }}
             {{- end }}
           {{- end }}
           {{- if $component.resources }}
           resources:
             {{- toYaml $component.resources | nindent 12 }}
           {{- end }}
+      {{- if $component.sidecars }}
+        {{- toYaml $component.sidecars | nindent 8 }}
+      {{- end }}
+      {{- if $component.initContainers }}
+      initContainers:
+        {{- toYaml $component.initContainers | nindent 8 }}
+      {{- end }}
           {{- if $component.update_ca }}
           lifecycle:
             postStart:
@@ -91,7 +113,7 @@ spec:
       {{- if $component.customPodDefinitions }}
         {{- toYaml $component.customPodDefinitions | nindent 6 }}
       {{- end }}
-      {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.customVolumes)}}
+      {{- if (or $component.mountConfig (and $component.persistentDir $.Values.persistance_enabled) $component.extraVolumes)}}
       volumes:
         {{- if $component.mountConfig }}
         - name: {{ $component.mountConfig.configName }}
@@ -107,15 +129,18 @@ spec:
         {{ else }}
         - name: {{ $component.name }}
           persistentVolumeClaim:
-            claimName: {{ printf "%s-pvc" $.Values.storage_name }}
-        {{- end }}
+            claimName: {{ printf "%s" $.Values.storage_name }}
+            {{ end }}
+        {{ end }}
+        {{- if $component.extraVolumes -}}
+          {{- range $volumeLabel, $volume := $component.extraVolumes }}
+        -
+            {{- toYaml $volume | nindent 10 }}
         {{- end }}
-        {{- if $component.customVolumes }}
-          {{- toYaml $component.customVolumes | nindent 8 }}
         {{- end }}
       {{- end }}
       hostAliases:
         {{- toYaml $.Values.mdssHostAliases | nindent 8 }}
 
 {{ end }}
-{{ end }}
+{{ end }}
diff --git a/helm_charts/mdss/values.yaml b/helm_charts/mdss/values.yaml
@@ -1,7 +1,7 @@
 # Default values for mdss. These will also be included in the environment of each mdss container
 
 mdss-common-environment:
-  ENABLED_MODULES: "azureblob,amazonsdk"                                  # Enabled modules by default
+  ENABLED_MODULES: "azureblob,amazonsdk,nfs"                                  # Enabled modules by default
   # ENABLED_MODULES: "azureblob,amazonsdk,googlecloud,alibabacloud,azurefiles,smb,box,onedrive,sftp,debug"             # All supported modules
 
   # This defines the connection string to the db ( should be modified when using an external db)
@@ -39,7 +39,7 @@ mdss-common-environment:
   RABBITMQ_SCANNING_PREFETCH_COUNT: "20"
 
   HTTPS_ACTIVE: "no"
-  BRANCH: "3.3.0"                                   # Sets a custom MDSS branch for testing/preview versions
+  BRANCH: "3.3.3"                                   # Sets a custom MDSS branch for testing/preview versions
   # These variables can be changed to reveal debug information in the application
   LOG_LEVEL: "4"
   APP_LOG_LEVEL: "INFORMATION"
@@ -110,6 +110,42 @@ mdss-common-environment:
   # This variable is used to force Signature Version 4 for AWS S3/S3 compatible request. 
   FORCE_AWS_SIGNATURE_V4: "1"
 
+  NFSSERVICE_URL: "http://nfsservice"
+  NFSSERVICE_PORT: "5004"
+
+  NFS_SHORT_DEADLINE: "5"
+  NFS_LONG_DEADLINE: "30"
+
+  NFS_UPLOAD_CHUNK: "2"
+
+  # This value is used to determine how MDSS consider that a file from a smb share is ready to be processed (Default 1-FILEOPEN_CHECK, 0-DEBOUNCE_CHECK)
+  SMBSERVICE_FILE_READY_CHECK_MODE: "1"
+  # This can be modified in order to define how many threads can be processed in parallel.
+  SMBSERVICE_PREFETCH_COUNT: "10"
+
+  DISCOVERY_SERVICE_SFTP_RTP_HANDLING: "0"
+  DISCOVERY_SERVICE_NFS_RTP_HANDLING: "0"
+
+  # Default number of hours for token expiration, maximum 24 hours
+  TOKEN_EXPIRY_TIME_HOURS: "1"
+
+  # Discovery debounce time used in order to check if a file is uploaded or not, we check if the file has not been modified between a 5 seconds interval
+  SFTP_DISCOVERY_DEBOUNCE_TIME_SECONDS: "5"
+  NFS_DISCOVERY_DEBOUNCE_TIME_SECONDS: "5"
+
+  # time to wait (in minutes) before retrying the register session method in SMB
+  SMBSERVICE_SESSION_RETRY_WAIT_TIME: "5"
+
+  # maximum amount of retries for the register session method in SMB
+  SMBSERVICE_SESSION_MAX_RETRIES: "3"
+
+  # number of active connections per smb storage; default is 10, maximum is 100
+  SMBSERVICE_SESSIONS_ON_STORAGE: "10"
+
+  # time interval between two consecutive job runs for the discovery completed job
+  DISCOVERY_COMPLETED_CHECK_SECONDS: "10"
+
+
 
 # Set to true if the respective database engine should also be deployed
 deploy_with_mdss_db: true                 # Enable or disable the local in-cluster database, set to false when deploying with an external database service
@@ -748,4 +784,53 @@ mdss_components:
         memory: "0.25Gi"
         cpu: "0.500"
 
-
+  # ======================================= NFS MODULE ================================================
+  discoverynfsservice:
+    name: discoverynfsservice
+    module_name: nfs
+    custom_repo: true
+    image: mdcloudservices_discoverynfs
+    extra_labels:
+      aws-type: fargate
+    resources:
+      requests:
+        memory: "0.25Gi"
+        cpu: "1.000"
+
+  remediationsservice-nfs:
+    name: remediationsservice-nfs
+    module_name: nfs
+    custom_repo: true
+    image: mdcloudservices_remediationsnfs
+    extra_labels:
+      aws-type: fargate
+    resources:
+      requests:
+        memory: "0.25Gi"
+        cpu: "0.100"
+
+  storagesservice-nfs:
+    name: storagesservice-nfs
+    module_name: nfs
+    custom_repo: true
+    image: mdcloudservices_storagesnfs
+    extra_labels:
+      aws-type: fargate
+    resources:
+      requests:
+        memory: "0.25Gi"
+        cpu: "0.010"
+
+  nfsservice:
+    name: nfsservice
+    module_name: nfs
+    custom_repo: true
+    image: mdcloudservices_nfsservice
+    ports:
+      - port: 5004
+    extra_labels:
+      aws-type: fargate
+    resources:
+      requests:
+        memory: "0.5Gi"
+        cpu: "0.500"
diff --git a/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf b/terraform/CloudFunctions/Azure/webhook-notification/terraform/main.tf
@@ -183,6 +183,9 @@ resource "azurerm_eventgrid_system_topic_event_subscription" "eventgrid" {
   system_topic = azurerm_eventgrid_system_topic.topic.name
   resource_group_name = var.STORAGE_RG
   event_delivery_schema = "EventGridSchema"
+  included_event_types = [
+    "Microsoft.Storage.BlobCreated"
+  ]
   subject_filter {
     subject_begins_with = "/blobServices/default/containers/${var.STORAGE_CONTAINERNAME}"
   }