feat: ldap2pgsql migration (#1046)

* feat: ldap2pgsql migration Signed-off-by: Mustafa Baser <[email protected]> * fix: ldap2rpm statistic data migration Signed-off-by: Mustafa Baser <[email protected]> * fix: set empty json values as None Signed-off-by: Mustafa Baser <[email protected]> --------- Signed-off-by: Mustafa Baser <[email protected]>
GluuFederation · Oct 7, 2024 · dda3906 · dda3906
1 parent 9c34bd8
commit dda3906
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 69 deletions.
diff --git a/setup_app/installers/rdbm.py b/setup_app/installers/rdbm.py
@@ -56,8 +56,11 @@ def prepare(self):
             self.gluu_attributes += schema_.get('attributeTypes', [])
 
 
+    @property
+    def output_dir(self):
+        return os.path.join(Config.outputFolder, Config.rdbm_type)
+
     def install(self):
-        self.output_dir = os.path.join(Config.outputFolder, Config.rdbm_type)
         if not os.path.exists(self.output_dir):
             self.createDirs(self.output_dir)
 

diff --git a/setup_app/utils/db_utils.py b/setup_app/utils/db_utils.py
@@ -1010,9 +1010,9 @@ def import_ldif(self, ldif_files, bucket=None, force=None):
 
                         sqlalchCls = self.Base.classes[table_name]
 
-                        for col in sqlalchCls.__table__.columns:
-                            if isinstance(col.type, self.json_dialects_instance) and col.name not in vals:
-                                vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else []
+                        #for col in sqlalchCls.__table__.columns:
+                        #    if isinstance(col.type, self.json_dialects_instance) and col.name not in vals:
+                        #        vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else []
 
                         sqlalchObj = sqlalchCls()
 

diff --git a/tools/ldap_to_mysql/README.md b/tools/ldap_to_mysql/README.md
diff --git a/tools/ldap_to_rdbm/README.md b/tools/ldap_to_rdbm/README.md
@@ -0,0 +1,68 @@
+# Gluu OpenDJ to MySQL Migration
+
+This script migrates data from OpenDJ to MySQL. Please note that this tool is experimental.
+Note!: Test this script on non-production server.
+
+# Prerequisites
+
+* Insptall `python3-ldap` package.
+
+  On RHEL-8/CentOS-8-AppStream: `yum install python3-ldap`
+
+  On Ubuntun: `apt install python3-ldap`
+
+* If you are migration to MySQL
+
+  Install MySQL that upports a native JSON data type (See https://dev.mysql.com/doc/refman/5.7/en/json.html).
+  MySQL Server shipped with >=Ubuntu 20 and >=RHEL 8/CentOS-8-Appstream is fine.
+  Create a database, namely `gluudb`, and create
+  a user, namely `gluu`. User should have all previleges on created database. Sample MySQL commands 
+  (If you installed MySQL on a seperate server, modify commands accordingly):
+
+  ```
+  CREATE DATABASE gluudb;
+  CREATE USER 'gluu'@'localhost' IDENTIFIED BY 'TopSecret';
+  GRANT ALL PRIVILEGES ON gluudb.* TO 'gluu'@'localhost';
+  ```
+
+* If you are migration to PostgreSQL
+
+  Install postgresql server on your system (version should be at least 14.0) or any host that can be reachable from gluu host.
+  Add the following line at the beginning of file `pg_hba.conf` (You can learn location of this file by executing command `sudo su - postgres -c 'psql -U postgres -d postgres -t -c "SHOW hba_file;"'`):
+
+  `host    gluudb    gluu    0.0.0.0/0    md5`
+
+  To crate database, user and adjust previleges, connect to postgresql server by command
+  `sudo su - postgres -c 'psql'`
+
+  Execute the following sql commands:
+
+  ```
+  CREATE DATABASE gluudb;
+  CREATE USER gluu WITH PASSWORD 'TopSecret';
+  GRANT ALL PRIVILEGES ON DATABASE gluudb TO gluu;
+  ALTER DATABASE gluudb OWNER TO gluu;
+  ```
+
+# Migration
+
+  - Download migration script:
+    ```
+    wget https://raw.githubusercontent.com/GluuFederation/community-edition-setup/tools/ldap_to_rdbm/ldap2rdbm.py -O /install/community-edition-setup/ldap2rdbm.py
+    ```
+
+  - Execute script (for MySQL):
+    ```
+    cd /install/community-edition-setup/
+    python3 ldap2rdbm.py -rdbm-type="mysql" -rdbm-user="gluu" -rdbm-password="TopSecret" -rdbm-db="gluudb" -rdbm-host="localhost"
+    ```
+
+    - Execute script (for PostgreSQL):
+    ```
+    cd /install/community-edition-setup/
+    python3 ldap2rdbm.py -rdbm-type="pgsql" -rdbm-user="gluu" -rdbm-password="TopSecret" -rdbm-db="gluudb" -rdbm-host="localhost" -rdbm-port="5432"
+
+    ```
+
+    Replace `localhost` with hostname of RDBM server
+
diff --git a/tools/ldap_to_mysql/ldap2mysql.py → tools/ldap_to_rdbm/ldap2rdbm.py b/tools/ldap_to_mysql/ldap2mysql.py → tools/ldap_to_rdbm/ldap2rdbm.py
@@ -20,14 +20,14 @@
 
 
 parser = argparse.ArgumentParser(description="Gluu CE LDAP to RDBM migrator script")
-parser.add_argument('-remote-rdbm', choices=['mysql'], help="Enables using remote RDBM server", default='mysql')
+parser.add_argument('-rdbm-type', choices=['mysql', 'pgsql'], help="Enables using remote RDBM server", default='mysql')
 parser.add_argument('-rdbm-user', help="RDBM username",  required = True)
 parser.add_argument('-rdbm-password', help="RDBM password",  required = True)
 parser.add_argument('-rdbm-port', help="RDBM port", type=int)
 parser.add_argument('-rdbm-db', help="RDBM database",  required = True)
 parser.add_argument('-rdbm-host', help="RDBM host",  required = True)
 argsp = parser.parse_args()
-rdbm_config_params = ('rdbm_user', 'rdbm_password', 'rdbm_host', 'rdbm_db', 'rdbm_host', 'rdbm_port')
+rdbm_config_params = ('rdbm_type', 'rdbm_user', 'rdbm_password', 'rdbm_host', 'rdbm_db', 'rdbm_host', 'rdbm_port')
 argsp_dict = { a: getattr(argsp, a) for a in rdbm_config_params }
 sys.argv = [sys.argv[0]]
 
@@ -50,9 +50,7 @@
 from setup_app.installers.opendj import OpenDjInstaller
 from setup_app.installers.rdbm import RDBMInstaller
 from setup_app.pylib.ldif4.ldif import LDIFWriter
-
-
-
+from ldap3.utils import dn as dnutils
 
 Config.init(paths.INSTALL_DIR)
 Config.determine_version()
@@ -74,28 +72,61 @@
 
 gluuInstaller.createLdapPw()
 
-current_ldif_fn = os.path.join(Config.outputFolder, 'current_data.ldif')
-
-print("Dumping all database from LDAP to {}. This may take a while...".format(current_ldif_fn))
-
-gluuInstaller.run(' '.join([
-                        '/opt/opendj/bin/ldapsearch',
-                        '-X', '-Z', '-D',
-                        '"{}"'.format(Config.ldap_binddn),
-                        '-j',
-                        Config.ldapPassFn,
-                        '-h',
-                        Config.ldap_hostname,
-                        '-p',
-                        '1636',
-                        '-b',
-                        'o=gluu',
-                        'ObjectClass=*',
-                        '>',
-                        current_ldif_fn]), shell=True)
+out_dir = os.path.join(Config.outputFolder, 'ldap_dump')
+if not os.path.exists(out_dir):
+    os.mkdir(out_dir)
+static_ldif_fn = os.path.join(out_dir, 'statistic_data.ldif')
+current_ldif_fn = os.path.join(out_dir, 'current_data.ldif')
+
+def dump_ldap(ldap_filter, output_fn):
+    print(f"Dumping data from LDAP with filter {ldap_filter} to {output_fn}. This may take a while...")
+
+
+    gluuInstaller.run(' '.join([
+                            '/opt/opendj/bin/ldapsearch',
+                            '-X', '-Z', '-D',
+                            '"{}"'.format(Config.ldap_binddn),
+                            '-j',
+                            Config.ldapPassFn,
+                            '-h',
+                            Config.ldap_hostname,
+                            '-p',
+                            '1636',
+                            '-b',
+                            'o=gluu',
+                            ldap_filter,
+                            '>',
+                            output_fn]), shell=True)
+
+print("Dumping data from LDAP Server")
+
+dump_ldap("'(objectClass=jansStatEntry)'", static_ldif_fn)
+dump_ldap("'(&(objectClass=*)(!(objectClass=jansStatEntry)))'", current_ldif_fn)
 
 gluuInstaller.deleteLdapPw()
 
+print("Migrating Statistic Data")
+
+stat_ldif_parser = myLdifParser(static_ldif_fn)
+stat_ldif_parser.parse()
+
+migrated_static_ldif_fn = static_ldif_fn + '.migrated'
+with open(migrated_static_ldif_fn, 'wb') as w:
+    stat_ldif_writer = LDIFWriter(w, cols=10000)
+
+    for dn, entry in stat_ldif_parser.entries:
+        rd_list = dnutils.parse_dn(dn)
+        stat_ou = rd_list[1][1]
+        if not rd_list[0][1].endswith(f'_{stat_ou}'):
+            jans_id = rd_list[0][1] + '_' + stat_ou
+            rd_list[0] = (rd_list[0][0], jans_id, rd_list[0][2])
+            entry['jansId'] = [jans_id]
+            dn = ','.join(['='.join([dne[0], dne[1]])  for dne in rd_list])
+
+        stat_ldif_writer.unparse(dn, entry)
+
+static_ldif_fn = migrated_static_ldif_fn
+
 print("Preparing custom schema from attributes")
 schema = {'attributeTypes':[], 'objectClasses':[]}
 
@@ -187,8 +218,10 @@
 os.rename(current_ldif_tmp_fn, current_ldif_fn)
 
 
-print("Importing data into RDBM from {}. This may take a while ...".format(current_ldif_fn))
-rdbmInstaller.dbUtils.import_ldif([current_ldif_fn])
+for ldif_fn in (current_ldif_fn, static_ldif_fn):
+    print("Importing data into RDBM from {}. This may take a while ...".format(ldif_fn))
+    rdbmInstaller.dbUtils.import_ldif([ldif_fn])
+
 print("Creating indexes...")
 rdbmInstaller.create_indexes()