Merge pull request #25 from a0x8o/main

True up the documentation branch to main

.github/ISSUE_TEMPLATE/feature_request.md

@@ -18,3 +18,6 @@ A clear and concise description of any alternative solutions or features you've
 
 **Additional context**
 Add any other context or screenshots about the feature request here.
+
+**Contributing to Mosaic**
+Guidelines for contributing to Mosaic can be found in `CONTRIBUTING.md`.

.github/actions/python_build/action.yml

@@ -0,0 +1,27 @@
+name: build mosaic python
+description: build mosaic python
+runs:
+  using: "composite"
+  steps:
+    - name: Configure python interpreter
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install python dependencies
+      shell: bash
+      run: |
+        # - install pip libs
+        #   note: gdal requires the extra args
+        cd python
+        pip install build wheel pyspark==${{ matrix.spark }} numpy==${{ matrix.numpy }}
+        pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==${{ matrix.gdal }}
+        pip install .
+    - name: Test and build python package
+      shell: bash
+      run: |
+        cd python
+        python -m unittest
+        python -m build
+    - name: Copy python artifacts to GH Actions run
+      shell: bash
+      run: cp python/dist/*.whl staging

.github/actions/r_build/action.yml

@@ -0,0 +1,92 @@
+name: build mosaic R
+description: build mosaic R
+runs:
+  using: "composite"
+  steps:
+    - name: Setup R build environment
+      shell: bash
+      run: |
+        sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev
+    - name: Configure python interpreter
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install python dependencies
+      shell: bash
+      run: |
+        # - install pip libs
+        #   note: gdal requires the extra args
+        cd python
+        pip install build wheel pyspark==${{ matrix.spark }} numpy==${{ matrix.numpy }}
+        pip install --no-build-isolation --no-cache-dir --force-reinstall gdal==${{ matrix.gdal }}
+    - name: Create download location for Spark
+      shell: bash
+      run: |
+        sudo mkdir -p /usr/spark-download-${{ matrix.spark }}/unzipped
+        sudo mkdir -p /usr/spark-download-${{ matrix.spark }}/raw
+        sudo chown -R $USER: /usr/spark-download-${{ matrix.spark }}/
+    - name: Cache Spark download
+      id: cache-spark
+      uses: actions/cache@v3
+      with:
+        path: /usr/spark-download-${{ matrix.spark }}/unzipped
+        key: r_build-spark
+    - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }}
+      name: Download and unpack Spark
+      shell: bash
+      run: |
+        wget -P /usr/spark-download-${{ matrix.spark }}/raw https://archive.apache.org/dist/spark/spark-${{ matrix.spark }}/spark-${{ matrix.spark }}-bin-hadoop3.tgz
+        tar zxvf /usr/spark-download-${{ matrix.spark }}/raw/spark-${{ matrix.spark }}-bin-hadoop3.tgz -C /usr/spark-download-${{ matrix.spark }}/unzipped
+    - name: Create R environment
+      shell: bash
+      run: |
+        sudo mkdir -p /usr/lib/R/site-library
+        sudo chown -R $USER: /usr/lib/R/site-library
+    - name: Setup R
+      uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: ${{ matrix.R }}
+        use-public-rspm: true
+    - name: Install R dependencies
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla install_deps.R
+    - name: Generate R bindings
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla generate_R_bindings.R ../src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
+    - name: Build R docs
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla generate_docs.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Build R package
+      shell: bash
+      run: |
+        cd R
+        Rscript --vanilla build_r_package.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Test SparkR package
+      shell: bash
+      run: |
+        cd R/sparkR-mosaic
+        Rscript --vanilla tests.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Test sparklyr package
+      shell: bash
+      run: |
+        cd R/sparklyr-mosaic
+        Rscript --vanilla tests.R
+      env:
+        SPARK_HOME: /usr/spark-download-${{ matrix.spark }}/unzipped/spark-${{ matrix.spark }}-bin-hadoop3
+    - name: Copy R artifacts to GH Actions run
+      shell: bash
+      run: |
+        cp R/sparkR-mosaic/*.tar.gz staging
+        cp R/sparklyr-mosaic/*.tar.gz staging

.github/actions/scala_build/action.yml

@@ -0,0 +1,50 @@
+name: build mosaic scala
+description: build mosaic scala
+inputs:
+  skip_tests:
+    description: 'Skip Scala Tests? true or false'
+    default: 'false'
+runs:
+  using: "composite"
+  steps:
+        - name: Configure JDK
+          uses: actions/setup-java@v2
+          with:
+            java-version: '8'
+            distribution: 'zulu'
+        - name: Configure python interpreter
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python }}
+        - name: Add packaged GDAL dependencies
+          shell: bash
+          run : |
+            # - update apt
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-backports main universe multiverse restricted"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-updates main universe multiverse restricted"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-security main multiverse restricted universe"
+            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main multiverse restricted universe"
+            sudo apt-get update -y
+            # - install natives
+            sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7
+            sudo apt-get install -y gdal-bin libgdal-dev python3-numpy python3-gdal zip unzip
+            # - install pip libs
+            pip install --upgrade pip
+            pip install gdal==${{ matrix.gdal }}
+            # - add the so files 
+            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so
+            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so.30
+            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so.30.0.3
+        - name: Test and build the scala JAR - skip tests is false
+          if: inputs.skip_tests == 'false'
+          shell: bash
+          run: sudo mvn -q clean install
+        - name: Build the scala JAR - skip tests is true
+          if: inputs.skip_tests == 'true'
+          shell: bash
+          run: sudo mvn -q clean install -DskipTests -Dscoverage.skip
+        - name: Copy Scala artifacts to GH Actions run
+          shell: bash
+          run: |
+            mkdir staging
+            cp target/mosaic*.jar staging

.github/actions/upload_artefacts/action.yml

@@ -0,0 +1,10 @@
+name: upload mosaic artefacts
+description: upload mosaic artefacts
+runs:
+  using: "composite"
+  steps:
+    - name: upload artefacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: artefacts
+        path: staging/*

.github/dependabot.yml

@@ -0,0 +1,14 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "maven"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "pip"
+    directory: "/python"
+    schedule:
+      interval: "weekly"

.github/workflows/build_main.yml

@@ -0,0 +1,34 @@
+name: build main
+on:
+  push:
+    branches-ignore:
+      - "R/**"
+      - "r/**"
+      - "python/**"
+      - "scala/**"
+  pull_request:
+    branches:
+      - "**"
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.10.12 ]
+        numpy: [ 1.22.4 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.1 ]
+        R: [ 4.2.2 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala
+        uses: ./.github/actions/scala_build
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: build R
+        uses: ./.github/actions/r_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts

.github/workflows/build_python.yml

@@ -0,0 +1,30 @@
+name: build_python
+# build for python, skip scala tests and R build
+on:
+  push:
+    branches:
+      - "python/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.10.12 ]
+        numpy: [ 1.22.4 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.1 ]
+        R: [ 4.2.2 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala with skipping tests
+        uses: ./.github/actions/scala_build
+        with:
+          skip_tests: "true"
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts

.github/workflows/build_r.yml

@@ -0,0 +1,31 @@
+name: build_R
+# build for R, skip scala tests and python build
+on:
+  push:
+    branches:
+      - 'r/**'
+      - 'R/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.10.12 ]
+        numpy: [ 1.22.4 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.1 ]
+        R: [ 4.2.2 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala with skipping tests
+        uses: ./.github/actions/scala_build
+        with:
+          skip_tests: 'true'
+      - name: build r artefacts
+        uses: ./.github/actions/r_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts

.github/workflows/build_scala.yml

@@ -0,0 +1,29 @@
+name: build_scala
+on:
+  push:
+    branches:
+      - "scala/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.10.12 ]
+        numpy: [ 1.22.4 ]
+        gdal: [ 3.4.1 ]
+        spark: [ 3.4.1 ]
+        R: [ 4.2.2 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala with tests
+        uses: ./.github/actions/scala_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts
+      - name: Publish test coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}