diff --git a/setup_app/installers/rdbm.py b/setup_app/installers/rdbm.py index 0eae676e7..7e7cedf7d 100644 --- a/setup_app/installers/rdbm.py +++ b/setup_app/installers/rdbm.py @@ -56,8 +56,11 @@ def prepare(self): self.gluu_attributes += schema_.get('attributeTypes', []) + @property + def output_dir(self): + return os.path.join(Config.outputFolder, Config.rdbm_type) + def install(self): - self.output_dir = os.path.join(Config.outputFolder, Config.rdbm_type) if not os.path.exists(self.output_dir): self.createDirs(self.output_dir) diff --git a/setup_app/utils/db_utils.py b/setup_app/utils/db_utils.py index 3a3196d40..04dd2913d 100644 --- a/setup_app/utils/db_utils.py +++ b/setup_app/utils/db_utils.py @@ -1010,9 +1010,9 @@ def import_ldif(self, ldif_files, bucket=None, force=None): sqlalchCls = self.Base.classes[table_name] - for col in sqlalchCls.__table__.columns: - if isinstance(col.type, self.json_dialects_instance) and col.name not in vals: - vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else [] + #for col in sqlalchCls.__table__.columns: + # if isinstance(col.type, self.json_dialects_instance) and col.name not in vals: + # vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else [] sqlalchObj = sqlalchCls() diff --git a/tools/ldap_to_mysql/README.md b/tools/ldap_to_mysql/README.md deleted file mode 100644 index a77ba97d6..000000000 --- a/tools/ldap_to_mysql/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Gluu OpenDJ to MySQL Migration - -This script migrates data from OpenDJ to MySQL. Please note that this tool is experimental. -Note!: Test this script on non-production server. - -# Prerequisites - -* Insptall `python3-ldap` package. - - On RHEL-8/CentOS-8-AppStream: `yum install python3-ldap` - - On Ubuntun: `apt install python3-ldap` - -* Install and Configure MySQL - - Install MySQL that upports a native JSON data type (See https://dev.mysql.com/doc/refman/5.7/en/json.html). - MySQL Server shipped with >=Ubuntu 20 and >=RHEL 8/CentOS-8-Appstream is fine. - Create a database, namely `gluudb`, and create - a user, namely `gluu`. User should have all previleges on created database. Sample MySQL commands - (If you installed MySQL on a seperate server, modify commands accordingly): - - ``` - CREATE DATABASE gluudb; - CREATE USER 'gluu'@'localhost' IDENTIFIED BY 'TopSecret'; - GRANT ALL PRIVILEGES ON gluudb.* TO 'gluu'@'localhost'; - ``` - -# Migration - - - Download migration script: - ``` - wget https://raw.githubusercontent.com/GluuFederation/community-edition-setup/tools/ldap_to_mysql/ldap2mysql.py -O /install/community-edition-setup/ldap2mysql.py - ``` - - - Execute script: - ``` - cd /install/community-edition-setup/ - python3 ldap2mysql.py -rdbm-user=gluu -rdbm-password=TopSecret -rdbm-db=gluudb -rdbm-host=localhost - ``` diff --git a/tools/ldap_to_rdbm/README.md b/tools/ldap_to_rdbm/README.md new file mode 100644 index 000000000..76cbf8782 --- /dev/null +++ b/tools/ldap_to_rdbm/README.md @@ -0,0 +1,68 @@ +# Gluu OpenDJ to MySQL Migration + +This script migrates data from OpenDJ to MySQL. Please note that this tool is experimental. +Note!: Test this script on non-production server. + +# Prerequisites + +* Insptall `python3-ldap` package. + + On RHEL-8/CentOS-8-AppStream: `yum install python3-ldap` + + On Ubuntun: `apt install python3-ldap` + +* If you are migration to MySQL + + Install MySQL that upports a native JSON data type (See https://dev.mysql.com/doc/refman/5.7/en/json.html). + MySQL Server shipped with >=Ubuntu 20 and >=RHEL 8/CentOS-8-Appstream is fine. + Create a database, namely `gluudb`, and create + a user, namely `gluu`. User should have all previleges on created database. Sample MySQL commands + (If you installed MySQL on a seperate server, modify commands accordingly): + + ``` + CREATE DATABASE gluudb; + CREATE USER 'gluu'@'localhost' IDENTIFIED BY 'TopSecret'; + GRANT ALL PRIVILEGES ON gluudb.* TO 'gluu'@'localhost'; + ``` + +* If you are migration to PostgreSQL + + Install postgresql server on your system (version should be at least 14.0) or any host that can be reachable from gluu host. + Add the following line at the beginning of file `pg_hba.conf` (You can learn location of this file by executing command `sudo su - postgres -c 'psql -U postgres -d postgres -t -c "SHOW hba_file;"'`): + + `host gluudb gluu 0.0.0.0/0 md5` + + To crate database, user and adjust previleges, connect to postgresql server by command + `sudo su - postgres -c 'psql'` + + Execute the following sql commands: + + ``` + CREATE DATABASE gluudb; + CREATE USER gluu WITH PASSWORD 'TopSecret'; + GRANT ALL PRIVILEGES ON DATABASE gluudb TO gluu; + ALTER DATABASE gluudb OWNER TO gluu; + ``` + +# Migration + + - Download migration script: + ``` + wget https://raw.githubusercontent.com/GluuFederation/community-edition-setup/tools/ldap_to_rdbm/ldap2rdbm.py -O /install/community-edition-setup/ldap2rdbm.py + ``` + + - Execute script (for MySQL): + ``` + cd /install/community-edition-setup/ + python3 ldap2rdbm.py -rdbm-type="mysql" -rdbm-user="gluu" -rdbm-password="TopSecret" -rdbm-db="gluudb" -rdbm-host="localhost" + ``` + + - Execute script (for PostgreSQL): + ``` + cd /install/community-edition-setup/ + python3 ldap2rdbm.py -rdbm-type="pgsql" -rdbm-user="gluu" -rdbm-password="TopSecret" -rdbm-db="gluudb" -rdbm-host="localhost" -rdbm-port="5432" + + ``` + + Replace `localhost` with hostname of RDBM server + diff --git a/tools/ldap_to_mysql/ldap2mysql.py b/tools/ldap_to_rdbm/ldap2rdbm.py similarity index 70% rename from tools/ldap_to_mysql/ldap2mysql.py rename to tools/ldap_to_rdbm/ldap2rdbm.py index 1969887df..013bb9789 100644 --- a/tools/ldap_to_mysql/ldap2mysql.py +++ b/tools/ldap_to_rdbm/ldap2rdbm.py @@ -20,14 +20,14 @@ parser = argparse.ArgumentParser(description="Gluu CE LDAP to RDBM migrator script") -parser.add_argument('-remote-rdbm', choices=['mysql'], help="Enables using remote RDBM server", default='mysql') +parser.add_argument('-rdbm-type', choices=['mysql', 'pgsql'], help="Enables using remote RDBM server", default='mysql') parser.add_argument('-rdbm-user', help="RDBM username", required = True) parser.add_argument('-rdbm-password', help="RDBM password", required = True) parser.add_argument('-rdbm-port', help="RDBM port", type=int) parser.add_argument('-rdbm-db', help="RDBM database", required = True) parser.add_argument('-rdbm-host', help="RDBM host", required = True) argsp = parser.parse_args() -rdbm_config_params = ('rdbm_user', 'rdbm_password', 'rdbm_host', 'rdbm_db', 'rdbm_host', 'rdbm_port') +rdbm_config_params = ('rdbm_type', 'rdbm_user', 'rdbm_password', 'rdbm_host', 'rdbm_db', 'rdbm_host', 'rdbm_port') argsp_dict = { a: getattr(argsp, a) for a in rdbm_config_params } sys.argv = [sys.argv[0]] @@ -50,9 +50,7 @@ from setup_app.installers.opendj import OpenDjInstaller from setup_app.installers.rdbm import RDBMInstaller from setup_app.pylib.ldif4.ldif import LDIFWriter - - - +from ldap3.utils import dn as dnutils Config.init(paths.INSTALL_DIR) Config.determine_version() @@ -74,28 +72,61 @@ gluuInstaller.createLdapPw() -current_ldif_fn = os.path.join(Config.outputFolder, 'current_data.ldif') - -print("Dumping all database from LDAP to {}. This may take a while...".format(current_ldif_fn)) - -gluuInstaller.run(' '.join([ - '/opt/opendj/bin/ldapsearch', - '-X', '-Z', '-D', - '"{}"'.format(Config.ldap_binddn), - '-j', - Config.ldapPassFn, - '-h', - Config.ldap_hostname, - '-p', - '1636', - '-b', - 'o=gluu', - 'ObjectClass=*', - '>', - current_ldif_fn]), shell=True) +out_dir = os.path.join(Config.outputFolder, 'ldap_dump') +if not os.path.exists(out_dir): + os.mkdir(out_dir) +static_ldif_fn = os.path.join(out_dir, 'statistic_data.ldif') +current_ldif_fn = os.path.join(out_dir, 'current_data.ldif') + +def dump_ldap(ldap_filter, output_fn): + print(f"Dumping data from LDAP with filter {ldap_filter} to {output_fn}. This may take a while...") + + + gluuInstaller.run(' '.join([ + '/opt/opendj/bin/ldapsearch', + '-X', '-Z', '-D', + '"{}"'.format(Config.ldap_binddn), + '-j', + Config.ldapPassFn, + '-h', + Config.ldap_hostname, + '-p', + '1636', + '-b', + 'o=gluu', + ldap_filter, + '>', + output_fn]), shell=True) + +print("Dumping data from LDAP Server") + +dump_ldap("'(objectClass=jansStatEntry)'", static_ldif_fn) +dump_ldap("'(&(objectClass=*)(!(objectClass=jansStatEntry)))'", current_ldif_fn) gluuInstaller.deleteLdapPw() +print("Migrating Statistic Data") + +stat_ldif_parser = myLdifParser(static_ldif_fn) +stat_ldif_parser.parse() + +migrated_static_ldif_fn = static_ldif_fn + '.migrated' +with open(migrated_static_ldif_fn, 'wb') as w: + stat_ldif_writer = LDIFWriter(w, cols=10000) + + for dn, entry in stat_ldif_parser.entries: + rd_list = dnutils.parse_dn(dn) + stat_ou = rd_list[1][1] + if not rd_list[0][1].endswith(f'_{stat_ou}'): + jans_id = rd_list[0][1] + '_' + stat_ou + rd_list[0] = (rd_list[0][0], jans_id, rd_list[0][2]) + entry['jansId'] = [jans_id] + dn = ','.join(['='.join([dne[0], dne[1]]) for dne in rd_list]) + + stat_ldif_writer.unparse(dn, entry) + +static_ldif_fn = migrated_static_ldif_fn + print("Preparing custom schema from attributes") schema = {'attributeTypes':[], 'objectClasses':[]} @@ -187,8 +218,10 @@ os.rename(current_ldif_tmp_fn, current_ldif_fn) -print("Importing data into RDBM from {}. This may take a while ...".format(current_ldif_fn)) -rdbmInstaller.dbUtils.import_ldif([current_ldif_fn]) +for ldif_fn in (current_ldif_fn, static_ldif_fn): + print("Importing data into RDBM from {}. This may take a while ...".format(ldif_fn)) + rdbmInstaller.dbUtils.import_ldif([ldif_fn]) + print("Creating indexes...") rdbmInstaller.create_indexes()