Reactivate the integration tests for remote file access (GCS, S3) + f…

…ix khiopspydev image build for ubuntu (when installing khiops_core)

Docker is installed in the khiopspydev docker image so that it can start the local fake servers
The container must be started with the --privileged option otherwise docker inside the container would not start
The fake servers are pre-provisioned with sample files to be ready before the tests run

.github/workflows/integration-tests.yml

@@ -0,0 +1,94 @@
+---
+name: Integration Tests
+env:
+  DEFAULT_SAMPLES_REVISION: main
+on:
+  workflow_dispatch:
+    inputs:
+      samples-revision:
+        default: main
+        description: Git tag, branch or commit for the khiops-samples repository
+      run-long-tests:
+        type: boolean
+        required: false
+        default: false
+        description: Execute long tests
+  pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  test-access-to-remote-files:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/khiopsml/khiops-python/khiopspydev-ubuntu22.04:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: --privileged  # as docker will run inside the container we need this option otherwise docker would not start
+    permissions:
+      id-token: write
+      contents: read
+      checks: write
+      packages: read
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          # Get Git tags so that versioneer can function correctly
+          # See issue https://github.com/actions/checkout/issues/701
+          fetch-depth: 0
+      - name: Install khiops-python dev dependencies
+        run: |
+          # The following git command is required, 
+          # as the Git repository is in a directory the current user does not own,
+          # Python versioneer fails to compute the current version correctly otherwise
+          git config --global --add safe.directory $(realpath .)
+          python setup.py egg_info
+          pip install `grep -v "^\[" khiops.egg-info/requires.txt`
+          rm -rf khiops.egg-info
+      - name: Setup and Install Test Requirements
+        run: |
+          pip install -r test-requirements.txt
+          pip install unittest-xml-reporting
+      - name: Launch the tests
+        env:
+          # for the tests with GCS
+          GCS_BUCKET_NAME: gcs-bucket
+          STORAGE_EMULATOR_HOST: http://localhost:4443
+          # for the tests with S3
+          S3_BUCKET_NAME: s3-bucket
+          AWS_SECRET_ACCESS_KEY: XXX
+          AWS_ENDPOINT_URL: http://localhost:4569
+          AWS_ACCESS_KEY_ID: XXX
+          # common vars
+          # Force > 2 CPU cores to launch mpiexec
+          KHIOPS_PROC_NUMBER: 4
+        run: |-
+          /scripts/run_fake_remote_file_servers.sh  # launch the servers in the background
+          python -m xmlrunner tests.test_remote_access.KhiopsGCSRemoteFileTests -o reports/
+          python -m xmlrunner tests.test_remote_access.KhiopsS3RemoteFileTests -o reports/
+      - name: Display Integration Test Reports
+        uses: dorny/test-reporter@v1
+        with:
+          name: Integration Tests
+          path: >-
+            reports/TEST-tests.test_remote_access.*.xml,
+          reporter: java-junit
+          path-replace-backslashes: 'true'  # Necessary for windows paths
+      - name: Upload Test Reports as Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-reports
+          path: |-
+            reports/TEST-tests.test_remote_access.*.xml
+            test/resources/tmp/khiops-cicd/*/*
+            tests/resources/scenario_generation/*/ref/*._kh
+            tests/resources/scenario_generation/*/output/*._kh
+            tests/resources/*/output_reports/*.txt
+            tests/resources/*/ref_reports/*.txt
+            tests/resources/dictionary/ref_kdic/*.kdic
+            tests/resources/dictionary/output_kdic/*.kdic
+            tests/resources/dictionary/copy_output_kdic/*.kdic
+            tests/resources/general_options/general_options/*/*._kh
+          retention-days: 7

khiops/core/internals/runner.py

@@ -1165,8 +1165,15 @@ def _initialize_mpi_command_args(self):
                     )
         # In Conda or local installations, expect mpiexec in the PATH
         else:
-            mpiexec_path = os.environ.get("KHIOPS_MPIEXEC_PATH") or shutil.which(
-                "mpiexec"
+            link_to_mpiexec = shutil.which("mpiexec")
+            # on Arch/Ubuntu systems /bin symlinks to /usr/bin,
+            # therefore we must follow the link to find the executable
+            # otherwise openmpi will raise the error :
+            # "A prefix was supplied to mpiexec that only contained slashes."
+            mpiexec_path = (
+                os.environ.get("KHIOPS_MPIEXEC_PATH")
+                or link_to_mpiexec
+                and os.readlink(link_to_mpiexec)
             )
         # If mpiexec is not in the path, and the installation method is local,
         # then try to load MPI environment module so that mpiexec is in the path
@@ -1592,6 +1599,11 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False):
             encoding="utf8",
             errors="replace",
             universal_newlines=True,
+            # the environment variables must be passed to khiops
+            # (especially the ones used by the drivers for the remote files,
+            # e.g `STORAGE_EMULATOR_HOST` for GCS, `AWS_*` for S3
+            # )
+            env=os.environ.copy(),
         ) as khiops_process:
             stdout, stderr = khiops_process.communicate()
 

packaging/docker/khiopspydev/Dockerfile.ubuntu

@@ -16,9 +16,8 @@ RUN true \
   # Obtain the Khiops native package \
   && KHIOPS_PKG_FILE=$KHIOPS_REVISION/khiops-core-openmpi_$KHIOPS_REVISION-1-$VERSION_CODENAME.amd64.deb \
   && wget -O KHIOPS_CORE.deb "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_PKG_FILE}" \
-  # Install the Khiops native package \
-  && dpkg -i --force-all KHIOPS_CORE.deb \
-  && apt-get -f -y install \
+  # Install the Khiops native package : do not break immediately if dpkg fails because apt will install the missing dependencies \
+  && (dpkg -i --force-all KHIOPS_CORE.deb || apt-get -f -y install) \
   && rm -f KHIOPS_CORE.deb \
   # Set python to python3 \
   && update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
@@ -34,6 +33,18 @@ RUN true \
   && rm -fr /var/lib/apt/lists/* \
   && apt-get clean \
   && rm -rf ./khiops \
+  # Install docker within the image to be able to launch the fake remote file servers for S3 & GCS.
+  # Please note that the container based on the current image must be ran with the --privileged flag
+  # otherwise docker would not start
+  && apt-get update \
+  && apt-get install -y ca-certificates curl \
+  && mkdir -p /etc/apt/keyrings && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
+  && chmod a+r /etc/apt/keyrings/docker.asc \
+  && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
+  && apt-get update \
+  && apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
+  && sed -i 's/ulimit -Hn/# ulimit -Hn/g' /etc/init.d/docker \
   && true
 
 # set up all the supported Python environments under conda (for the unit tests)
@@ -48,7 +59,8 @@ done'
 
 RUN mkdir -p /scripts
 COPY ./run_service.sh /scripts/run_service.sh
-RUN chmod +x /scripts/run_service.sh && \
+COPY ./run_fake_remote_file_servers.sh /scripts/run_fake_remote_file_servers.sh
+RUN chmod +x /scripts/*.sh && \
   useradd -rm -d /home/ubuntu -s /bin/bash -g root -u 1000 ubuntu
 
 FROM ghcr.io/khiopsml/khiops-server:${SERVER_REVISION} AS server

packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# first start the docker daemon
+service docker start
+
+# File server for GCS
+# WARNING : there are 3 major features actived by the options ...
+# -v : the volume mounted exposes pre-provisioned files to be read remotely : the direct child folders will be the bucket names
+# -filesystem-root : let upload and read new files remotely at the same location as the source
+# -public-host : must expose localhost (https://github.com/fsouza/fake-gcs-server/issues/201)
+docker run --rm \
+  -d \
+  -p 4443:4443 \
+  --name fake-gcs-server \
+  -v ./tests/resources/remote-access:/data fsouza/fake-gcs-server \
+  -scheme http \
+  -filesystem-root /data \
+  -public-host localhost
+
+# File server for S3
+# WARNING :
+# -v : the volume mounted exposes pre-provisioned files : the direct child folders will be the bucket names
+#      these files were uploaded once because fake-s3 creates metadata
+docker run --rm \
+  -d \
+  -p 4569:4569 \
+  --name my_s3 \
+  -v ./tests/resources/remote-access:/fakes3_root lphoward/fake-s3

run_fake_remote_file_servers.sh

@@ -0,0 +1 @@
+packaging/docker/khiopspydev/run_fake_remote_file_servers.sh

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/Iris/Iris.kdic

@@ -0,0 +1,9 @@
+
+Dictionary	Iris
+{
+	Numerical	SepalLength	;	
+	Numerical	SepalWidth	;	
+	Numerical	PetalLength	;	
+	Numerical	PetalWidth	;	
+	Categorical	Class	;	
+};

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/Iris/Iris.txt

@@ -0,0 +1,151 @@
+SepalLength	SepalWidth	PetalLength	PetalWidth	Class
+5.1	3.5	1.4	0.2	Iris-setosa
+4.9	3.0	1.4	0.2	Iris-setosa
+4.7	3.2	1.3	0.2	Iris-setosa
+4.6	3.1	1.5	0.2	Iris-setosa
+5.0	3.6	1.4	0.2	Iris-setosa
+5.4	3.9	1.7	0.4	Iris-setosa
+4.6	3.4	1.4	0.3	Iris-setosa
+5.0	3.4	1.5	0.2	Iris-setosa
+4.4	2.9	1.4	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+5.4	3.7	1.5	0.2	Iris-setosa
+4.8	3.4	1.6	0.2	Iris-setosa
+4.8	3.0	1.4	0.1	Iris-setosa
+4.3	3.0	1.1	0.1	Iris-setosa
+5.8	4.0	1.2	0.2	Iris-setosa
+5.7	4.4	1.5	0.4	Iris-setosa
+5.4	3.9	1.3	0.4	Iris-setosa
+5.1	3.5	1.4	0.3	Iris-setosa
+5.7	3.8	1.7	0.3	Iris-setosa
+5.1	3.8	1.5	0.3	Iris-setosa
+5.4	3.4	1.7	0.2	Iris-setosa
+5.1	3.7	1.5	0.4	Iris-setosa
+4.6	3.6	1.0	0.2	Iris-setosa
+5.1	3.3	1.7	0.5	Iris-setosa
+4.8	3.4	1.9	0.2	Iris-setosa
+5.0	3.0	1.6	0.2	Iris-setosa
+5.0	3.4	1.6	0.4	Iris-setosa
+5.2	3.5	1.5	0.2	Iris-setosa
+5.2	3.4	1.4	0.2	Iris-setosa
+4.7	3.2	1.6	0.2	Iris-setosa
+4.8	3.1	1.6	0.2	Iris-setosa
+5.4	3.4	1.5	0.4	Iris-setosa
+5.2	4.1	1.5	0.1	Iris-setosa
+5.5	4.2	1.4	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+5.0	3.2	1.2	0.2	Iris-setosa
+5.5	3.5	1.3	0.2	Iris-setosa
+4.9	3.1	1.5	0.1	Iris-setosa
+4.4	3.0	1.3	0.2	Iris-setosa
+5.1	3.4	1.5	0.2	Iris-setosa
+5.0	3.5	1.3	0.3	Iris-setosa
+4.5	2.3	1.3	0.3	Iris-setosa
+4.4	3.2	1.3	0.2	Iris-setosa
+5.0	3.5	1.6	0.6	Iris-setosa
+5.1	3.8	1.9	0.4	Iris-setosa
+4.8	3.0	1.4	0.3	Iris-setosa
+5.1	3.8	1.6	0.2	Iris-setosa
+4.6	3.2	1.4	0.2	Iris-setosa
+5.3	3.7	1.5	0.2	Iris-setosa
+5.0	3.3	1.4	0.2	Iris-setosa
+7.0	3.2	4.7	1.4	Iris-versicolor
+6.4	3.2	4.5	1.5	Iris-versicolor
+6.9	3.1	4.9	1.5	Iris-versicolor
+5.5	2.3	4.0	1.3	Iris-versicolor
+6.5	2.8	4.6	1.5	Iris-versicolor
+5.7	2.8	4.5	1.3	Iris-versicolor
+6.3	3.3	4.7	1.6	Iris-versicolor
+4.9	2.4	3.3	1.0	Iris-versicolor
+6.6	2.9	4.6	1.3	Iris-versicolor
+5.2	2.7	3.9	1.4	Iris-versicolor
+5.0	2.0	3.5	1.0	Iris-versicolor
+5.9	3.0	4.2	1.5	Iris-versicolor
+6.0	2.2	4.0	1.0	Iris-versicolor
+6.1	2.9	4.7	1.4	Iris-versicolor
+5.6	2.9	3.6	1.3	Iris-versicolor
+6.7	3.1	4.4	1.4	Iris-versicolor
+5.6	3.0	4.5	1.5	Iris-versicolor
+5.8	2.7	4.1	1.0	Iris-versicolor
+6.2	2.2	4.5	1.5	Iris-versicolor
+5.6	2.5	3.9	1.1	Iris-versicolor
+5.9	3.2	4.8	1.8	Iris-versicolor
+6.1	2.8	4.0	1.3	Iris-versicolor
+6.3	2.5	4.9	1.5	Iris-versicolor
+6.1	2.8	4.7	1.2	Iris-versicolor
+6.4	2.9	4.3	1.3	Iris-versicolor
+6.6	3.0	4.4	1.4	Iris-versicolor
+6.8	2.8	4.8	1.4	Iris-versicolor
+6.7	3.0	5.0	1.7	Iris-versicolor
+6.0	2.9	4.5	1.5	Iris-versicolor
+5.7	2.6	3.5	1.0	Iris-versicolor
+5.5	2.4	3.8	1.1	Iris-versicolor
+5.5	2.4	3.7	1.0	Iris-versicolor
+5.8	2.7	3.9	1.2	Iris-versicolor
+6.0	2.7	5.1	1.6	Iris-versicolor
+5.4	3.0	4.5	1.5	Iris-versicolor
+6.0	3.4	4.5	1.6	Iris-versicolor
+6.7	3.1	4.7	1.5	Iris-versicolor
+6.3	2.3	4.4	1.3	Iris-versicolor
+5.6	3.0	4.1	1.3	Iris-versicolor
+5.5	2.5	4.0	1.3	Iris-versicolor
+5.5	2.6	4.4	1.2	Iris-versicolor
+6.1	3.0	4.6	1.4	Iris-versicolor
+5.8	2.6	4.0	1.2	Iris-versicolor
+5.0	2.3	3.3	1.0	Iris-versicolor
+5.6	2.7	4.2	1.3	Iris-versicolor
+5.7	3.0	4.2	1.2	Iris-versicolor
+5.7	2.9	4.2	1.3	Iris-versicolor
+6.2	2.9	4.3	1.3	Iris-versicolor
+5.1	2.5	3.0	1.1	Iris-versicolor
+5.7	2.8	4.1	1.3	Iris-versicolor
+6.3	3.3	6.0	2.5	Iris-virginica
+5.8	2.7	5.1	1.9	Iris-virginica
+7.1	3.0	5.9	2.1	Iris-virginica
+6.3	2.9	5.6	1.8	Iris-virginica
+6.5	3.0	5.8	2.2	Iris-virginica
+7.6	3.0	6.6	2.1	Iris-virginica
+4.9	2.5	4.5	1.7	Iris-virginica
+7.3	2.9	6.3	1.8	Iris-virginica
+6.7	2.5	5.8	1.8	Iris-virginica
+7.2	3.6	6.1	2.5	Iris-virginica
+6.5	3.2	5.1	2.0	Iris-virginica
+6.4	2.7	5.3	1.9	Iris-virginica
+6.8	3.0	5.5	2.1	Iris-virginica
+5.7	2.5	5.0	2.0	Iris-virginica
+5.8	2.8	5.1	2.4	Iris-virginica
+6.4	3.2	5.3	2.3	Iris-virginica
+6.5	3.0	5.5	1.8	Iris-virginica
+7.7	3.8	6.7	2.2	Iris-virginica
+7.7	2.6	6.9	2.3	Iris-virginica
+6.0	2.2	5.0	1.5	Iris-virginica
+6.9	3.2	5.7	2.3	Iris-virginica
+5.6	2.8	4.9	2.0	Iris-virginica
+7.7	2.8	6.7	2.0	Iris-virginica
+6.3	2.7	4.9	1.8	Iris-virginica
+6.7	3.3	5.7	2.1	Iris-virginica
+7.2	3.2	6.0	1.8	Iris-virginica
+6.2	2.8	4.8	1.8	Iris-virginica
+6.1	3.0	4.9	1.8	Iris-virginica
+6.4	2.8	5.6	2.1	Iris-virginica
+7.2	3.0	5.8	1.6	Iris-virginica
+7.4	2.8	6.1	1.9	Iris-virginica
+7.9	3.8	6.4	2.0	Iris-virginica
+6.4	2.8	5.6	2.2	Iris-virginica
+6.3	2.8	5.1	1.5	Iris-virginica
+6.1	2.6	5.6	1.4	Iris-virginica
+7.7	3.0	6.1	2.3	Iris-virginica
+6.3	3.4	5.6	2.4	Iris-virginica
+6.4	3.1	5.5	1.8	Iris-virginica
+6.0	3.0	4.8	1.8	Iris-virginica
+6.9	3.1	5.4	2.1	Iris-virginica
+6.7	3.1	5.6	2.4	Iris-virginica
+6.9	3.1	5.1	2.3	Iris-virginica
+5.8	2.7	5.1	1.9	Iris-virginica
+6.8	3.2	5.9	2.3	Iris-virginica
+6.7	3.3	5.7	2.5	Iris-virginica
+6.7	3.0	5.2	2.3	Iris-virginica
+6.3	2.5	5.0	1.9	Iris-virginica
+6.5	3.0	5.2	2.0	Iris-virginica
+6.2	3.4	5.4	2.3	Iris-virginica
+5.9	3.0	5.1	1.8	Iris-virginica

tests/resources/remote-access/gcs-bucket/khiops-cicd/samples/SpliceJunction/SpliceJunction.kdic

@@ -0,0 +1,14 @@
+
+Root	Dictionary	SpliceJunction(SampleId)
+{
+	Categorical	SampleId	;	
+	Categorical	Class	;	
+	Table(SpliceJunctionDNA)	DNA	;
+};
+
+Dictionary	SpliceJunctionDNA(SampleId)
+{
+	Categorical	SampleId	;	
+	Numerical	Pos	;	
+	Categorical	Char	;	
+};