From 4c31fb6ad31af87f4c87aed5bf18d9f04a827dd9 Mon Sep 17 00:00:00 2001 From: zyh Date: Fri, 6 Dec 2024 16:58:58 +0800 Subject: [PATCH 1/3] feature:add oceanbase support --- .../database/OceanBaseMysqlDatabaseMeta.java | 28 +++ .../database/OceanBaseOracleDatabaseMeta.java | 52 ++++++ .../OceanBaseOracleDatabaseMetaTest.java | 174 ++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 core/src/main/java/org/pentaho/di/core/database/OceanBaseMysqlDatabaseMeta.java create mode 100644 core/src/main/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMeta.java create mode 100644 core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java diff --git a/core/src/main/java/org/pentaho/di/core/database/OceanBaseMysqlDatabaseMeta.java b/core/src/main/java/org/pentaho/di/core/database/OceanBaseMysqlDatabaseMeta.java new file mode 100644 index 000000000000..6d20f650aa4c --- /dev/null +++ b/core/src/main/java/org/pentaho/di/core/database/OceanBaseMysqlDatabaseMeta.java @@ -0,0 +1,28 @@ +/*! ****************************************************************************** + * + * Pentaho + * + * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file. + * + * Change Date: 2029-07-20 + ******************************************************************************/ +package org.pentaho.di.core.database; + +import org.pentaho.di.core.plugins.DatabaseMetaPlugin; + +@DatabaseMetaPlugin( + type = "OCEANBASE_MYSQL", + typeDescription = "oceanbase mysql mode" +) +public class OceanBaseMysqlDatabaseMeta extends MySQLDatabaseMeta { + public OceanBaseMysqlDatabaseMeta() { + } + + public int getDefaultDatabasePort() { + return this.getAccessType() == 0 ? 2883 : -1; + } + +} diff --git a/core/src/main/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMeta.java b/core/src/main/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMeta.java new file mode 100644 index 000000000000..1757f40759e2 --- /dev/null +++ b/core/src/main/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMeta.java @@ -0,0 +1,52 @@ +/*! ****************************************************************************** + * + * Pentaho + * + * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file. + * + * Change Date: 2029-07-20 + ******************************************************************************/ +package org.pentaho.di.core.database; + +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.plugins.DatabaseMetaPlugin; + +@DatabaseMetaPlugin( + type = "OCEANBASE_ORACLE", + typeDescription = "oceanbase oracle mode" +) +public class OceanBaseOracleDatabaseMeta extends OracleDatabaseMeta { + public OceanBaseOracleDatabaseMeta() { + } + + public int getDefaultDatabasePort() { + return this.getAccessType() == 0 ? 2883 : -1; + } + + public int[] getAccessTypeList() { + return new int[]{0, 4}; + } + + public String getDriverClass() { + return "com.alipay.oceanbase.obproxy.mysql.jdbc.Driver"; + } + + public String getURL(String hostname, String port, String databaseName) throws KettleDatabaseException { + if (this.getAccessType() == 1) { + return "jdbc:odbc:" + databaseName; + } else if (this.getAccessType() == 0) { + return String.format("jdbc:oceanbase://%s?useUnicode=true&characterEncoding=%s&connectTimeout=30000&rewriteBatchedStatements=true", hostname + ":" + port, "utf8"); + } else if (databaseName != null && databaseName.length() > 0) { + return hostname != null && hostname.length() > 0 && port != null && port.length() > 0 ? "jdbc:oracle:oci:@(description=(address=(host=" + hostname + ")(protocol=tcp)(port=" + port + "))(connect_data=(sid=" + databaseName + ")))" : "jdbc:oracle:oci:@" + databaseName; + } else { + throw new KettleDatabaseException("Unable to construct a JDBC URL: at least the database name must be specified"); + } + } + + public String[] getUsedLibraries() { + return new String[]{"oceanbase-client-2.4.7.2.jar"}; + } +} diff --git a/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java new file mode 100644 index 000000000000..ebc8bb1db70e --- /dev/null +++ b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java @@ -0,0 +1,174 @@ +/*! ****************************************************************************** + * + * Pentaho + * + * Copyright (C) 2024 by Hitachi Vantara, LLC : http://www.pentaho.com + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file. + * + * Change Date: 2029-07-20 + ******************************************************************************/ +package org.pentaho.di.core.database; + +import org.junit.Test; +import org.pentaho.di.core.row.value.ValueMetaBigNumber; +import org.pentaho.di.core.row.value.ValueMetaBinary; +import org.pentaho.di.core.row.value.ValueMetaBoolean; +import org.pentaho.di.core.row.value.ValueMetaDate; +import org.pentaho.di.core.row.value.ValueMetaInteger; +import org.pentaho.di.core.row.value.ValueMetaInternetAddress; +import org.pentaho.di.core.row.value.ValueMetaNumber; +import org.pentaho.di.core.row.value.ValueMetaString; +import org.pentaho.di.core.row.value.ValueMetaTimestamp; +import org.pentaho.di.core.variables.Variables; + +import static org.junit.Assert.*; + +public class OceanBaseOracleDatabaseMetaTest { + + OceanBaseOracleDatabaseMeta nativeMeta; + + @Test + public void testOverriddenSettings() throws Exception { + // Tests the settings of the Oracle Database Meta + // according to the features of the DB as we know them + + assertEquals( 1521, nativeMeta.getDefaultDatabasePort() ); + assertFalse( nativeMeta.supportsAutoInc() ); + assertFalse( nativeMeta.needsToLockAllTables() ); + assertEquals( "com.alipay.oceanbase.obproxy.mysql.jdbc.Driver", nativeMeta.getDriverClass() ); + assertEquals( "jdbc:oceanbase:@FOO:1024:BAR", nativeMeta.getURL( "FOO", "1024", "BAR" ) ); + assertEquals( "jdbc:oceanbase:@FOO:11:BAR", nativeMeta.getURL( "FOO", "11", ":BAR" ) ); + assertEquals( "jdbc:oceanbase:@BAR:65534/FOO", nativeMeta.getURL( "BAR", "65534", "/FOO" ) ); + assertEquals( "jdbc:oceanbase:@FOO", nativeMeta.getURL( "", "", "FOO" ) ); + assertEquals( "jdbc:oceanbase:@FOO", nativeMeta.getURL( null, "-1", "FOO" ) ); + assertEquals( "jdbc:oceanbase:@FOO", nativeMeta.getURL( null, null, "FOO" ) ); + assertEquals( "jdbc:oceanbase:@FOO:1234:BAR", nativeMeta.getURL( "FOO", "1234", "BAR" ) ); + assertEquals( "jdbc:oceanbase:@", nativeMeta.getURL( "", "", "" ) ); // Pretty sure this is a bug... + assertFalse( nativeMeta.supportsOptionsInURL() ); + assertTrue( nativeMeta.supportsSequences() ); + assertTrue( nativeMeta.supportsSequenceNoMaxValueOption() ); + assertTrue( nativeMeta.useSchemaNameForTableList() ); + assertTrue( nativeMeta.supportsSynonyms() ); + String[] reservedWords = + new String[] { "ACCESS", "ADD", "ALL", "ALTER", "AND", "ANY", "ARRAYLEN", "AS", "ASC", "AUDIT", "BETWEEN", "BY", + "CHAR", "CHECK", "CLUSTER", "COLUMN", "COMMENT", "COMPRESS", "CONNECT", "CREATE", "CURRENT", "DATE", + "DECIMAL", "DEFAULT", "DELETE", "DESC", "DISTINCT", "DROP", "ELSE", "EXCLUSIVE", "EXISTS", "FILE", "FLOAT", + "FOR", "FROM", "GRANT", "GROUP", "HAVING", "IDENTIFIED", "IMMEDIATE", "IN", "INCREMENT", "INDEX", "INITIAL", + "INSERT", "INTEGER", "INTERSECT", "INTO", "IS", "LEVEL", "LIKE", "LOCK", "LONG", "MAXEXTENTS", "MINUS", + "MODE", "MODIFY", "NOAUDIT", "NOCOMPRESS", "NOT", "NOTFOUND", "NOWAIT", "NULL", "NUMBER", "OF", "OFFLINE", + "ON", "ONLINE", "OPTION", "OR", "ORDER", "PCTFREE", "PRIOR", "PRIVILEGES", "PUBLIC", "RAW", "RENAME", + "RESOURCE", "REVOKE", "ROW", "ROWID", "ROWLABEL", "ROWNUM", "ROWS", "SELECT", "SESSION", "SET", "SHARE", + "SIZE", "SMALLINT", "SQLBUF", "START", "SUCCESSFUL", "SYNONYM", "SYSDATE", "TABLE", "THEN", "TO", "TRIGGER", + "UID", "UNION", "UNIQUE", "UPDATE", "USER", "VALIDATE", "VALUES", "VARCHAR", "VARCHAR2", "VIEW", "WHENEVER", + "WHERE", "WITH" }; + assertArrayEquals( reservedWords, nativeMeta.getReservedWords() ); + assertEquals( "http://download.oracle.com/docs/cd/B19306_01/java.102/b14355/urls.htm#i1006362", nativeMeta + .getExtraOptionsHelpText() ); + assertEquals( "oceanbase-client-2.4.7.2.jar", nativeMeta.getUsedLibraries() ); + assertTrue( nativeMeta.requiresCreateTablePrimaryKeyAppend() ); + assertTrue( nativeMeta.supportsPreparedStatementMetadataRetrieval() ); // Since PDI-19514 + String quoteTest1 = "FOO 'BAR' \r TEST \n"; + String quoteTest2 = "FOO 'BAR' \\r TEST \\n"; + assertEquals( "'FOO ''BAR'' '||chr(10)||' TEST '||chr(13)||''", nativeMeta.quoteSQLString( quoteTest1 ) ); + assertEquals( "'FOO ''BAR'' \\r TEST \\n'", nativeMeta.quoteSQLString( quoteTest2 ) ); + assertFalse( nativeMeta.releaseSavepoint() ); + Variables v = new Variables(); + v.setVariable( "FOOVARIABLE", "FOOVALUE" ); + DatabaseMeta dm = new DatabaseMeta(); + dm.setDatabaseInterface( nativeMeta ); + assertEquals( "TABLESPACE FOOVALUE", nativeMeta.getTablespaceDDL( v, dm, "${FOOVARIABLE}" ) ); + assertEquals( "", nativeMeta.getTablespaceDDL( v, dm, "" ) ); + assertFalse( nativeMeta.supportsErrorHandlingOnBatchUpdates() ); + assertTrue( nativeMeta.supportsRepository() ); + assertEquals( 2000, nativeMeta.getMaxVARCHARLength() ); + assertFalse( nativeMeta.supportsTimestampDataType() ); + assertEquals( 32, nativeMeta.getMaxColumnsInIndex() ); + } + + @Test + public void testOverriddenSQLStatements() throws Exception { + assertEquals( " WHERE ROWNUM <= 5", nativeMeta.getLimitClause( 5 ) ); + String reusedFieldsQuery = "SELECT * FROM FOO WHERE 1=0"; + assertEquals( reusedFieldsQuery, nativeMeta.getSQLQueryFields( "FOO" ) ); + assertEquals( reusedFieldsQuery, nativeMeta.getSQLTableExists( "FOO" ) ); + String reusedColumnsQuery = "SELECT FOO FROM BAR WHERE 1=0"; + assertEquals( reusedColumnsQuery, nativeMeta.getSQLQueryColumnFields( "FOO", "BAR" ) ); + assertEquals( reusedColumnsQuery, nativeMeta.getSQLColumnExists( "FOO", "BAR" ) ); + assertEquals( "SELECT * FROM USER_SEQUENCES WHERE SEQUENCE_NAME = 'FOO'", nativeMeta.getSQLSequenceExists( "FOO" ) ); + assertEquals( "SELECT * FROM USER_SEQUENCES WHERE SEQUENCE_NAME = 'FOO'", nativeMeta.getSQLSequenceExists( "foo" ) ); + + assertEquals( "SELECT * FROM ALL_SEQUENCES WHERE SEQUENCE_NAME = 'BAR' AND SEQUENCE_OWNER = 'FOO'", nativeMeta + .getSQLSequenceExists( "FOO.BAR" ) ); + assertEquals( "SELECT * FROM ALL_SEQUENCES WHERE SEQUENCE_NAME = 'BAR' AND SEQUENCE_OWNER = 'FOO'", nativeMeta + .getSQLSequenceExists( "foo.bar" ) ); + + assertEquals( "SELECT FOO.currval FROM DUAL", nativeMeta.getSQLCurrentSequenceValue( "FOO" ) ); + assertEquals( "SELECT FOO.nextval FROM dual", nativeMeta.getSQLNextSequenceValue( "FOO" ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO DATE ) ", + nativeMeta.getAddColumnStatement( "FOO", new ValueMetaTimestamp( "FOO" ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO DATE ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaDate( "FOO" ), "", + false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO VARCHAR2(15) ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaString( + "FOO", 15, 0 ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO INTEGER ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaInteger( + "FOO", 15, 0 ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO NUMBER(15, 10) ) ", nativeMeta.getAddColumnStatement( "FOO", + new ValueMetaBigNumber( + "FOO", 15, 10 ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO NUMBER(15, 10) ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaNumber( + "FOO", 15, 10 ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO BLOB ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaBinary( + "FOO", 2048, 0 ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO CHAR(1) ) ", nativeMeta.getAddColumnStatement( "FOO", new ValueMetaBoolean( + "FOO" ), "", false, "", false ) ); + assertEquals( "ALTER TABLE FOO ADD ( FOO UNKNOWN ) ", nativeMeta.getAddColumnStatement( "FOO", + new ValueMetaInternetAddress( "FOO" ), "", false, "", false ) ); + + String lineSep = System.getProperty( "line.separator" ); + assertEquals( "ALTER TABLE FOO DROP ( BAR ) " + lineSep, nativeMeta.getDropColumnStatement( + "FOO", new ValueMetaString( "BAR" ), "", false, "", false ) ); + String modColStmtExpected = + "ALTER TABLE FOO ADD ( BAR_KTL VARCHAR2(2000) ) ;" + lineSep + "UPDATE FOO SET BAR_KTL=BAR;" + lineSep + + "ALTER TABLE FOO DROP ( BAR ) " + lineSep + ";" + lineSep + "ALTER TABLE FOO ADD ( BAR VARCHAR2(2000) ) ;" + + lineSep + "UPDATE FOO SET BAR=BAR_KTL;" + lineSep + "ALTER TABLE FOO DROP ( BAR_KTL ) " + lineSep; + assertEquals( modColStmtExpected, nativeMeta.getModifyColumnStatement( "FOO", new ValueMetaString( "BAR" ), "", false, "", + false ) ); + modColStmtExpected = + "ALTER TABLE \"FOO\" ADD ( BAR_KTL VARCHAR2(2000) ) ;" + lineSep + "UPDATE \"FOO\" SET BAR_KTL=BAR;" + lineSep + + "ALTER TABLE \"FOO\" DROP ( BAR ) " + lineSep + ";" + lineSep + "ALTER TABLE \"FOO\" ADD ( BAR VARCHAR2(2000) ) ;" + + lineSep + "UPDATE \"FOO\" SET BAR=BAR_KTL;" + lineSep + "ALTER TABLE \"FOO\" DROP ( BAR_KTL ) " + lineSep; + assertEquals( modColStmtExpected, nativeMeta.getModifyColumnStatement( "\"FOO\"", new ValueMetaString( "BAR" ), "", false, "", + false ) ); + + + modColStmtExpected = + "ALTER TABLE FOO ADD ( A12345678901234567890123456789_KTL VARCHAR2(2000) ) ;" + lineSep + + "UPDATE FOO SET A12345678901234567890123456789_KTL=A1234567890123456789012345678901234567890;" + lineSep + + "ALTER TABLE FOO DROP ( A1234567890123456789012345678901234567890 ) " + lineSep + ";" + lineSep + + "ALTER TABLE FOO ADD ( A1234567890123456789012345678901234567890 VARCHAR2(2000) ) ;" + lineSep + + "UPDATE FOO SET A1234567890123456789012345678901234567890=A12345678901234567890123456789_KTL;" + lineSep + + "ALTER TABLE FOO DROP ( A12345678901234567890123456789_KTL ) " + lineSep; + assertEquals( modColStmtExpected, nativeMeta.getModifyColumnStatement( "FOO", new ValueMetaString( "A1234567890123456789012345678901234567890" ), "", false, "", + false ) ); + + String expectedProcSql = + "SELECT DISTINCT DECODE(package_name, NULL, '', package_name||'.') || object_name " + "FROM user_arguments " + + "ORDER BY 1"; + + assertEquals( expectedProcSql, nativeMeta.getSQLListOfProcedures() ); + + String expectedLockOneItem = "LOCK TABLE FOO IN EXCLUSIVE MODE;" + lineSep; + assertEquals( expectedLockOneItem, nativeMeta.getSQLLockTables( new String[] { "FOO" } ) ); + String expectedLockMultiItem = + "LOCK TABLE FOO IN EXCLUSIVE MODE;" + lineSep + "LOCK TABLE BAR IN EXCLUSIVE MODE;" + lineSep; + assertEquals( expectedLockMultiItem, nativeMeta.getSQLLockTables( new String[] { "FOO", "BAR" } ) ); + assertNull( nativeMeta.getSQLUnlockTables( null ) ); // Commit unlocks tables + assertEquals( "SELECT SEQUENCE_NAME FROM all_sequences", nativeMeta.getSQLListOfSequences() ); + assertEquals( + "BEGIN EXECUTE IMMEDIATE 'DROP TABLE FOO'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; END;", + nativeMeta.getDropTableIfExistsStatement( "FOO" ) ); + + } +} From de8892f5d4020fd06bb24175de92d1b7d974e1c9 Mon Sep 17 00:00:00 2001 From: zyh Date: Fri, 6 Dec 2024 17:58:18 +0800 Subject: [PATCH 2/3] feature:add oceanbase support --- .../di/core/database/OceanBaseOracleDatabaseMetaTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java index ebc8bb1db70e..bc6bc966c36b 100644 --- a/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java +++ b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java @@ -11,6 +11,7 @@ ******************************************************************************/ package org.pentaho.di.core.database; +import org.junit.Before; import org.junit.Test; import org.pentaho.di.core.row.value.ValueMetaBigNumber; import org.pentaho.di.core.row.value.ValueMetaBinary; @@ -29,6 +30,12 @@ public class OceanBaseOracleDatabaseMetaTest { OceanBaseOracleDatabaseMeta nativeMeta; + @Before + public void setupBefore() { + nativeMeta = new OceanBaseOracleDatabaseMeta(); + nativeMeta.setAccessType( DatabaseMeta.TYPE_ACCESS_NATIVE ); + } + @Test public void testOverriddenSettings() throws Exception { // Tests the settings of the Oracle Database Meta From fa068d8b4b5a9af143127bbc3527cc5b0c2ca102 Mon Sep 17 00:00:00 2001 From: zyh Date: Mon, 9 Dec 2024 09:27:25 +0800 Subject: [PATCH 3/3] feature:add oceanbase support --- .../di/core/database/OceanBaseOracleDatabaseMetaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java index bc6bc966c36b..bbade4595362 100644 --- a/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java +++ b/core/src/test/java/org/pentaho/di/core/database/OceanBaseOracleDatabaseMetaTest.java @@ -41,7 +41,7 @@ public void testOverriddenSettings() throws Exception { // Tests the settings of the Oracle Database Meta // according to the features of the DB as we know them - assertEquals( 1521, nativeMeta.getDefaultDatabasePort() ); + assertEquals( 2883, nativeMeta.getDefaultDatabasePort() ); assertFalse( nativeMeta.supportsAutoInc() ); assertFalse( nativeMeta.needsToLockAllTables() ); assertEquals( "com.alipay.oceanbase.obproxy.mysql.jdbc.Driver", nativeMeta.getDriverClass() );