From 1cf867ac752d940c32fc1b5c69f005183db6ac74 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 16 Dec 2024 12:47:30 +0200 Subject: [PATCH] chore(dremio-driver): add basic integration tests for dremio cloud (#9046) --- .github/actions/integration/dremio.sh | 10 ++ .github/workflows/push.yml | 7 +- packages/cubejs-dremio-driver/.gitignore | 1 + packages/cubejs-dremio-driver/package.json | 11 +- .../test/DremioDriver.test.ts | 21 +++ .../test/DremioQuery.test.ts | 121 ++++++++++++++++++ .../cubejs-dremio-driver/test/test-env.js | 12 ++ packages/cubejs-dremio-driver/tsconfig.json | 13 ++ tsconfig.json | 3 + 9 files changed, 197 insertions(+), 2 deletions(-) create mode 100755 .github/actions/integration/dremio.sh create mode 100644 packages/cubejs-dremio-driver/.gitignore create mode 100644 packages/cubejs-dremio-driver/test/DremioDriver.test.ts create mode 100644 packages/cubejs-dremio-driver/test/DremioQuery.test.ts create mode 100644 packages/cubejs-dremio-driver/test/test-env.js create mode 100644 packages/cubejs-dremio-driver/tsconfig.json diff --git a/.github/actions/integration/dremio.sh b/.github/actions/integration/dremio.sh new file mode 100755 index 0000000000000..ad1510e15ff96 --- /dev/null +++ b/.github/actions/integration/dremio.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eo pipefail + +# Debug log for test containers +export DEBUG=testcontainers + +echo "::group::Dremio [cloud]" +yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio + +echo "::endgroup::" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f3c47359f3c6..fdf8070b53cee 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -316,6 +316,7 @@ jobs: env: CLOUD_DATABASES: > firebolt + dremio # Athena (just to check for secrets availability) DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }} @@ -324,7 +325,7 @@ jobs: node-version: [20.x] db: [ 'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb', - 'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt' + 'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio' ] fail-fast: false @@ -386,6 +387,10 @@ jobs: DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }} DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }} DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }} + # Dremio Integration + DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }} + DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }} + DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }} integration-smoke: needs: [ latest-tag-sha, build-cubestore ] diff --git a/packages/cubejs-dremio-driver/.gitignore b/packages/cubejs-dremio-driver/.gitignore new file mode 100644 index 0000000000000..53c37a16608c0 --- /dev/null +++ b/packages/cubejs-dremio-driver/.gitignore @@ -0,0 +1 @@ +dist \ No newline at end of file diff --git a/packages/cubejs-dremio-driver/package.json b/packages/cubejs-dremio-driver/package.json index 02a8cfe795200..1f747ca7a00ca 100644 --- a/packages/cubejs-dremio-driver/package.json +++ b/packages/cubejs-dremio-driver/package.json @@ -13,6 +13,11 @@ }, "main": "driver/DremioDriver.js", "scripts": { + "tsc": "tsc", + "watch": "tsc -w", + "test": "yarn integration", + "integration": "npm run integration:dremio", + "integration:dremio": "jest --verbose dist/test", "lint": "eslint driver/*.js", "lint:fix": "eslint driver/*.js" }, @@ -26,6 +31,7 @@ }, "devDependencies": { "@cubejs-backend/linter": "^1.0.0", + "@cubejs-backend/testing-shared": "1.1.9", "jest": "^27" }, "license": "Apache-2.0", @@ -33,7 +39,10 @@ "access": "public" }, "jest": { - "testEnvironment": "node" + "testEnvironment": "node", + "setupFiles": [ + "./test/test-env.js" + ] }, "eslintConfig": { "extends": "../cubejs-linter" diff --git a/packages/cubejs-dremio-driver/test/DremioDriver.test.ts b/packages/cubejs-dremio-driver/test/DremioDriver.test.ts new file mode 100644 index 0000000000000..35346c804a0d8 --- /dev/null +++ b/packages/cubejs-dremio-driver/test/DremioDriver.test.ts @@ -0,0 +1,21 @@ +import { DriverTests } from '@cubejs-backend/testing-shared'; + +const DremioDriver = require('../../driver/DremioDriver'); + +describe('DremioDriver', () => { + let tests: DriverTests; + + jest.setTimeout(10 * 60 * 1000); // Engine needs to spin up + + beforeAll(async () => { + tests = new DriverTests(new DremioDriver({}), { expectStringFields: false }); + }); + + afterAll(async () => { + await tests.release(); + }); + + test('query', async () => { + await tests.testQuery(); + }); +}); diff --git a/packages/cubejs-dremio-driver/test/DremioQuery.test.ts b/packages/cubejs-dremio-driver/test/DremioQuery.test.ts new file mode 100644 index 0000000000000..94dec6a15aeb3 --- /dev/null +++ b/packages/cubejs-dremio-driver/test/DremioQuery.test.ts @@ -0,0 +1,121 @@ +import { prepareCompiler as originalPrepareCompiler } from '@cubejs-backend/schema-compiler'; + +const DremioQuery = require('../../driver/DremioQuery'); + +const prepareCompiler = (content: string) => originalPrepareCompiler({ + localPath: () => __dirname, + dataSchemaFiles: () => Promise.resolve([{ fileName: 'main.js', content }]), +}); + +describe('DremioQuery', () => { + + jest.setTimeout(10 * 60 * 1000); // Engine needs to spin up + + const { compiler, joinGraph, cubeEvaluator } = prepareCompiler( + ` +cube(\`sales\`, { + sql: \` select * from public.sales \`, + + measures: { + count: { + type: 'count' + } + }, + dimensions: { + category: { + type: 'string', + sql: 'category' + }, + salesDatetime: { + type: 'time', + sql: 'sales_datetime' + }, + isShiped: { + type: 'boolean', + sql: 'is_shiped', + }, + } +}); +`, + ); + + it('should use DATE_TRUNC for time granularity dimensions', () => compiler.compile().then(() => { + const query = new DremioQuery( + { joinGraph, cubeEvaluator, compiler }, + { + measures: ['sales.count'], + timeDimensions: [ + { + dimension: 'sales.salesDatetime', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-02'], + }, + ], + timezone: 'America/Los_Angeles', + order: [ + { + id: 'sales.salesDatetime', + }, + ], + } + ); + + const queryAndParams = query.buildSqlAndParams(); + + expect(queryAndParams[0]).toContain( + 'DATE_TRUNC(\'day\', CONVERT_TIMEZONE(\'-08:00\', "sales".sales_datetime))' + ); + })); + + it('should cast BOOLEAN', () => compiler.compile().then(() => { + const query = new DremioQuery( + { joinGraph, cubeEvaluator, compiler }, + { + measures: ['sales.count'], + filters: [ + { + member: 'sales.isShiped', + operator: 'equals', + values: ['true'] + } + ] + } + ); + + const queryAndParams = query.buildSqlAndParams(); + + expect(queryAndParams[0]).toContain( + '("sales".is_shiped = CAST(? AS BOOLEAN))' + ); + + expect(queryAndParams[1]).toEqual(['true']); + })); + + it('should cast timestamp', () => compiler.compile().then(() => { + const query = new DremioQuery( + { joinGraph, cubeEvaluator, compiler }, + { + measures: ['sales.count'], + timeDimensions: [ + { + dimension: 'sales.salesDatetime', + granularity: 'day', + dateRange: ['2017-01-01', '2017-01-02'], + }, + ], + timezone: 'America/Los_Angeles', + order: [ + { + id: 'sales.salesDatetime', + }, + ], + } + ); + + const queryAndParams = query.buildSqlAndParams(); + + expect(queryAndParams[0]).toContain( + '("sales".sales_datetime >= TO_TIMESTAMP(?, \'YYYY-MM-DD"T"HH24:MI:SS.FFF\') AND "sales".sales_datetime <= TO_TIMESTAMP(?, \'YYYY-MM-DD"T"HH24:MI:SS.FFF\'))' + ); + })); +}); diff --git a/packages/cubejs-dremio-driver/test/test-env.js b/packages/cubejs-dremio-driver/test/test-env.js new file mode 100644 index 0000000000000..8c0ba6dbf5e9f --- /dev/null +++ b/packages/cubejs-dremio-driver/test/test-env.js @@ -0,0 +1,12 @@ +const REQUIRED_ENV_VARS = [ + 'CUBEJS_DB_URL', + 'CUBEJS_DB_NAME', + 'CUBEJS_DB_DREMIO_AUTH_TOKEN', +]; + +REQUIRED_ENV_VARS.forEach((key) => { + // Trying to populate from DRIVERS_TESTS_DREMIO_* vars + if (process.env[`DRIVERS_TESTS_DREMIO_${key}`] !== undefined) { + process.env[key] = process.env[`DRIVERS_TESTS_DREMIO_${key}`]; + } +}); diff --git a/packages/cubejs-dremio-driver/tsconfig.json b/packages/cubejs-dremio-driver/tsconfig.json new file mode 100644 index 0000000000000..6af4af6de9a44 --- /dev/null +++ b/packages/cubejs-dremio-driver/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "include": [ + "src", + "test" + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": ".", + "resolveJsonModule": true + } +} diff --git a/tsconfig.json b/tsconfig.json index a467a74d4ce97..8fb24316e942b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -55,6 +55,9 @@ { "path": "packages/cubejs-duckdb-driver" }, + { + "path": "packages/cubejs-dremio-driver" + }, { "path": "packages/cubejs-questdb-driver" },