chore(dremio-driver): add basic integration tests for dremio cloud (#…

…9046)

.github/actions/integration/dremio.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Dremio [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio
+
+echo "::endgroup::"

.github/workflows/push.yml

@@ -316,6 +316,7 @@ jobs:
     env:
       CLOUD_DATABASES: >
         firebolt
+        dremio
       # Athena (just to check for secrets availability)
       DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
 
@@ -324,7 +325,7 @@ jobs:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio'
         ]
       fail-fast: false
 
@@ -386,6 +387,10 @@ jobs:
           DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
           DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
           DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
+          # Dremio Integration
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
 
   integration-smoke:
     needs: [ latest-tag-sha, build-cubestore ]

packages/cubejs-dremio-driver/.gitignore

@@ -0,0 +1 @@
+dist

packages/cubejs-dremio-driver/package.json

@@ -13,6 +13,11 @@
   },
   "main": "driver/DremioDriver.js",
   "scripts": {
+    "tsc": "tsc",
+    "watch": "tsc -w",
+    "test": "yarn integration",
+    "integration": "npm run integration:dremio",
+    "integration:dremio": "jest --verbose dist/test",
     "lint": "eslint driver/*.js",
     "lint:fix": "eslint driver/*.js"
   },
@@ -26,14 +31,18 @@
   },
   "devDependencies": {
     "@cubejs-backend/linter": "^1.0.0",
+    "@cubejs-backend/testing-shared": "1.1.9",
     "jest": "^27"
   },
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public"
   },
   "jest": {
-    "testEnvironment": "node"
+    "testEnvironment": "node",
+    "setupFiles": [
+      "./test/test-env.js"
+    ]
   },
   "eslintConfig": {
     "extends": "../cubejs-linter"

packages/cubejs-dremio-driver/test/DremioDriver.test.ts

@@ -0,0 +1,21 @@
+import { DriverTests } from '@cubejs-backend/testing-shared';
+
+const DremioDriver = require('../../driver/DremioDriver');
+
+describe('DremioDriver', () => {
+  let tests: DriverTests;
+
+  jest.setTimeout(10 * 60 * 1000); // Engine needs to spin up
+
+  beforeAll(async () => {
+    tests = new DriverTests(new DremioDriver({}), { expectStringFields: false });
+  });
+
+  afterAll(async () => {
+    await tests.release();
+  });
+
+  test('query', async () => {
+    await tests.testQuery();
+  });
+});

packages/cubejs-dremio-driver/test/DremioQuery.test.ts

@@ -0,0 +1,121 @@
+import { prepareCompiler as originalPrepareCompiler } from '@cubejs-backend/schema-compiler';
+
+const DremioQuery = require('../../driver/DremioQuery');
+
+const prepareCompiler = (content: string) => originalPrepareCompiler({
+  localPath: () => __dirname,
+  dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content }]),
+});
+
+describe('DremioQuery', () => {
+
+  jest.setTimeout(10 * 60 * 1000); // Engine needs to spin up
+
+  const { compiler, joinGraph, cubeEvaluator } = prepareCompiler(
+    `
+cube(\`sales\`, {
+  sql: \` select * from public.sales \`,
+
+  measures: {
+    count: {
+      type: 'count'
+    }
+  },
+  dimensions: {
+    category: {
+      type: 'string',
+      sql: 'category'
+    },
+    salesDatetime: {
+      type: 'time',
+      sql: 'sales_datetime'
+    },
+    isShiped: {
+      type: 'boolean',
+      sql: 'is_shiped',
+    },
+  }
+});
+`,
+  );
+
+  it('should use DATE_TRUNC for time granularity dimensions', () => compiler.compile().then(() => {
+    const query = new DremioQuery(
+      { joinGraph, cubeEvaluator, compiler },
+      {
+        measures: ['sales.count'],
+        timeDimensions: [
+          {
+            dimension: 'sales.salesDatetime',
+            granularity: 'day',
+            dateRange: ['2017-01-01', '2017-01-02'],
+          },
+        ],
+        timezone: 'America/Los_Angeles',
+        order: [
+          {
+            id: 'sales.salesDatetime',
+          },
+        ],
+      }
+    );
+
+    const queryAndParams = query.buildSqlAndParams();
+
+    expect(queryAndParams[0]).toContain(
+      'DATE_TRUNC(\'day\', CONVERT_TIMEZONE(\'-08:00\', "sales".sales_datetime))'
+    );
+  }));
+
+  it('should cast BOOLEAN', () => compiler.compile().then(() => {
+    const query = new DremioQuery(
+      { joinGraph, cubeEvaluator, compiler },
+      {
+        measures: ['sales.count'],
+        filters: [
+          {
+            member: 'sales.isShiped',
+            operator: 'equals',
+            values: ['true']
+          }
+        ]
+      }
+    );
+
+    const queryAndParams = query.buildSqlAndParams();
+
+    expect(queryAndParams[0]).toContain(
+      '("sales".is_shiped = CAST(? AS BOOLEAN))'
+    );
+
+    expect(queryAndParams[1]).toEqual(['true']);
+  }));
+
+  it('should cast timestamp', () => compiler.compile().then(() => {
+    const query = new DremioQuery(
+      { joinGraph, cubeEvaluator, compiler },
+      {
+        measures: ['sales.count'],
+        timeDimensions: [
+          {
+            dimension: 'sales.salesDatetime',
+            granularity: 'day',
+            dateRange: ['2017-01-01', '2017-01-02'],
+          },
+        ],
+        timezone: 'America/Los_Angeles',
+        order: [
+          {
+            id: 'sales.salesDatetime',
+          },
+        ],
+      }
+    );
+
+    const queryAndParams = query.buildSqlAndParams();
+
+    expect(queryAndParams[0]).toContain(
+      '("sales".sales_datetime >= TO_TIMESTAMP(?, \'YYYY-MM-DD"T"HH24:MI:SS.FFF\') AND "sales".sales_datetime <= TO_TIMESTAMP(?, \'YYYY-MM-DD"T"HH24:MI:SS.FFF\'))'
+    );
+  }));
+});

packages/cubejs-dremio-driver/test/test-env.js

@@ -0,0 +1,12 @@
+const REQUIRED_ENV_VARS = [
+  'CUBEJS_DB_URL',
+  'CUBEJS_DB_NAME',
+  'CUBEJS_DB_DREMIO_AUTH_TOKEN',
+];
+
+REQUIRED_ENV_VARS.forEach((key) => {
+  // Trying to populate from DRIVERS_TESTS_DREMIO_* vars
+  if (process.env[`DRIVERS_TESTS_DREMIO_${key}`] !== undefined) {
+    process.env[key] = process.env[`DRIVERS_TESTS_DREMIO_${key}`];
+  }
+});

packages/cubejs-dremio-driver/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": [
+    "src",
+    "test"
+  ],
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": ".",
+    "baseUrl": ".",
+    "resolveJsonModule": true
+  }
+}

tsconfig.json

@@ -55,6 +55,9 @@
     {
       "path": "packages/cubejs-duckdb-driver"
     },
+    {
+      "path": "packages/cubejs-dremio-driver"
+    },
     {
       "path": "packages/cubejs-questdb-driver"
     },