Issue 5843 - dsconf / dscreate should be able to handle lmdb paramete…

…rs (#5943) * Issue 5843 - dsconf / dscreate should be able to handle lmdb parameters Description: dscreate changes: - make db_lib a standard option instead of an advanced one - add mdb_max_size as standard option dsconf instance backend config set changes: - add --mdb_max_size option associated with nsslapd-mdb-max-size - add --mdb_max_readers option associated with nsslapd-mdb_max_readers - add --mdb_max_dbs option associated with nsslapd-mdb_max_dbs Issue: #5843 Reviewed by: @Firstyear and @droideck ( Thanks ! )
389ds · Oct 13, 2023 · 415eead · 415eead
1 parent 4f99f8b
commit 415eead
Show file tree

Hide file tree

Showing 11 changed files with 304 additions and 59 deletions.
diff --git a/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py b/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py
@@ -44,7 +44,7 @@ def init_user(topo_m2, request):
     def fin():
         try:
             test_user.delete()
-        except ldap.NO_SUCH_OBJECT:
+        except (ldap.NO_SUCH_OBJECT, ldap.SERVER_DOWN):
             pass
 
     request.addfinalizer(fin)

diff --git a/dirsrvtests/tests/suites/config/config_test.py b/dirsrvtests/tests/suites/config/config_test.py
@@ -10,17 +10,20 @@
 import logging
 import pytest
 import os
-from lib389 import pid_from_file
+from lib389 import DirSrv, pid_from_file
 from lib389.tasks import *
 from lib389.topologies import topology_m2, topology_st as topo
 from lib389.utils import *
 from lib389._constants import DN_CONFIG, DEFAULT_SUFFIX, DEFAULT_BENAME
+from lib389._mapped_object import DSLdapObjects
+from lib389.cli_base import FakeArgs
+from lib389.cli_conf.backend import db_config_set
 from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES
 from lib389.idm.group import Groups
-from lib389.backend import *
+from lib389.instance.setup import SetupDs
 from lib389.config import LDBMConfig, BDB_LDBMConfig, Config
 from lib389.cos import CosPointerDefinitions, CosTemplates
-from lib389.backend import Backends
+from lib389.backend import Backends, DatabaseConfig
 from lib389.monitor import MonitorLDBM, Monitor
 from lib389.plugins import ReferentialIntegrityPlugin
 
@@ -665,6 +668,88 @@ def test_changing_threadnumber(topo):
     time.sleep(3)
     check_number_of_threads(int(cfgnbthreads), monitor, pid)
 
+
+@pytest.fixture(scope="module")
+def create_lmdb_instance(request):
+    verbose = log.level > logging.DEBUG
+    instname = 'i_lmdb'
+    assert SetupDs(verbose=True, log=log).create_from_dict( {
+        'general' : {},
+        'slapd' : {
+            'instance_name': instname,
+            'db_lib': 'mdb',
+            'mdb_max_size': '0.5 Gb',
+        },
+        'backend-userroot': {
+            'sample_entries': 'yes',
+            'suffix': DEFAULT_SUFFIX,
+        },
+    } )
+    inst = DirSrv(verbose=verbose, external_log=log)
+    inst.local_simple_allocate(instname, binddn=DN_DM, password=PW_DM)
+    inst.setup_ldapi()
+
+    def fin():
+        inst.delete()
+
+    request.addfinalizer(fin)
+    inst.open()
+    return inst
+
+
+def set_and_check(inst, db_config, dsconf_attr, ldap_attr, val):
+    val = str(val)
+    args = FakeArgs()
+    setattr(args, dsconf_attr, val)
+    db_config_set(inst, db_config.dn, log, args)
+    cfg_vals = db_config.get()
+    assert ldap_attr in cfg_vals
+    assert cfg_vals[ldap_attr][0] == val
+
+
+def test_lmdb_config(create_lmdb_instance):
+    """Test nsslapd-ignore-virtual-attrs configuration attribute
+
+    :id: bca28086-61cf-11ee-a064-482ae39447e5
+    :setup: Custom instance named 'i_lmdb' having db_lib=mdb and lmdb_size=0.5
+    :steps:
+        1. Get dscreate create-template output
+        2. Check that 'db_lib' is in output
+        3. Check that 'lmdb_size' is in output
+        4. Get the database config
+        5. Check that nsslapd-backend-implement is mdb
+        6. Check that nsslapd-mdb-max-size is 536870912 (i.e 0.5Gb)
+        7. Set a value for nsslapd-mdb-max-size and test the value is properly set
+        8. Set a value for nsslapd-mdb-max-readers and test the value is properly set
+        9. Set a value for nsslapd-mdb-max-dbs and test the value is properly set
+    :expectedresults:
+        1. Success
+        2. Success
+        3. Success
+        4. Success
+        5. Success
+        6. Success
+        7. Success
+        8. Success
+        9. Success
+    """
+
+    res = subprocess.run(('dscreate', 'create-template'), stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT, encoding='utf-8')
+    inst = create_lmdb_instance
+    assert 'db_lib' in res.stdout
+    assert 'mdb_max_size' in res.stdout
+    db_config = DatabaseConfig(inst)
+    cfg_vals = db_config.get()
+    assert 'nsslapd-backend-implement' in cfg_vals
+    assert cfg_vals['nsslapd-backend-implement'][0] == 'mdb'
+    assert 'nsslapd-mdb-max-size' in cfg_vals
+    assert cfg_vals['nsslapd-mdb-max-size'][0] == '536870912'
+    set_and_check(inst, db_config, 'mdb_max_size', 'nsslapd-mdb-max-size', parse_size('2G'))
+    set_and_check(inst, db_config, 'mdb_max_readers', 'nsslapd-mdb-max-readers', 200)
+    set_and_check(inst, db_config, 'mdb_max_dbs', 'nsslapd-mdb-max-dbs', 200)
+
+
 if __name__ == '__main__':
     # Run isolated
     # -s for DEBUG mode

diff --git a/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h b/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h
@@ -45,7 +45,7 @@
 #define DBMDB_DB_MINSIZE             ( 4LL * MEGABYTE )
 #define DBMDB_DISK_RESERVE(disksize) ((disksize)*2ULL/1000ULL)
 #define DBMDB_READERS_MARGIN         10
-#define DBMDB_READERS_DEFAULT        50
+#define DBMDB_READERS_DEFAULT        126  /* default value as described in mdb_env_set_maxreaders */
 #define DBMDB_DBS_MARGIN             10
 #define DBMDB_DBS_DEFAULT            128
 

diff --git a/src/lib389/lib389/_constants.py b/src/lib389/lib389/_constants.py
@@ -241,6 +241,7 @@ class TaskWarning(IntEnum):
 VALGRIND_INVALID_STR = " Invalid (free|read|write)"
 DISORDERLY_SHUTDOWN = ('Detected Disorderly Shutdown last time Directory '
                        'Server was running, recovering database')
+DEFAULT_LMDB_SIZE = '20Gb'
 
 #
 # LOG: see https://access.redhat.com/documentation/en-US/Red_Hat_Directory

diff --git a/src/lib389/lib389/_mapped_object.py b/src/lib389/lib389/_mapped_object.py
@@ -151,6 +151,9 @@ def __unicode__(self):
     def __str__(self):
         return self.__unicode__()
 
+    def __repr__(self):
+        return f'{type(self)}(instance_name="{self._instance.serverid}", dn="{self.__unicode__()}")'
+
     def _unsafe_raw_entry(self):
         """Get an Entry object
 

diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py
@@ -65,6 +65,9 @@
         'deadlock_policy': 'nsslapd-db-deadlock-policy',
         'db_home_directory': 'nsslapd-db-home-directory',
         'db_lib': 'nsslapd-backend-implement',
+        'mdb_max_size': 'nsslapd-mdb-max-size',
+        'mdb_max_readers': 'nsslapd-mdb-max-readers',
+        'mdb_max_dbs': 'nsslapd-mdb-max-dbs',
         # VLV attributes
         'search_base': 'vlvbase',
         'search_scope': 'vlvscope',
@@ -1081,6 +1084,10 @@ def create_parser(subparsers):
     set_db_config_parser.add_argument('--deadlock-policy', help='Adjusts the backend database deadlock policy (Advanced setting)')
     set_db_config_parser.add_argument('--db-home-directory', help='Sets the directory for the database mmapped files (Advanced setting)')
     set_db_config_parser.add_argument('--db-lib', help='Sets which db lib is used. Valid values are: bdb or mdb')
+    set_db_config_parser.add_argument('--mdb-max-size', help='Sets the lmdb database maximum size (in bytes).')
+    set_db_config_parser.add_argument('--mdb-max-readers', help='Sets the lmdb database maximum number of readers (Advanced setting)')
+    set_db_config_parser.add_argument('--mdb-max-dbs', help='Sets the lmdb database maximum number of sub databases (Advanced setting)')
+
 
     #######################################################
     # Database & Suffix Monitor

diff --git a/src/lib389/lib389/cli_ctl/dblib.py b/src/lib389/lib389/cli_ctl/dblib.py
@@ -15,11 +15,13 @@
 import glob
 import shutil
 from lib389.dseldif import DSEldif
+from lib389._constants import DEFAULT_LMDB_SIZE
+from lib389.utils import parse_size, format_size
 import subprocess
+from errno import ENOSPC
 
 
 DBLIB_LDIF_PREFIX = "__dblib-"
-DEFAULT_DBMAP_SIZE = 2 * 1024 * 1024 * 1024
 DBSIZE_MARGIN = 1.2
 DBI_MARGIN = 60
 
@@ -142,14 +144,6 @@ def get_mdb_dbis(dbdir):
     return result
 
 
-def size_fmt(num):
-    for unit in ["B", "KB", "MB", "GB", "TB"]:
-        if (num < 1024):
-            return f"{num:3.1f} {unit}"
-        num /= 1024
-    return f"{num:.1f} TB"
-
-
 def run_dbscan(args):
     prefix = os.environ.get('PREFIX', "")
     prog = f'{prefix}/bin/dbscan'
@@ -235,18 +229,20 @@ def dblib_bdb2mdb(inst, log, args):
         total_entrysize += be['entrysize']
         total_dbi += be['dbi']
 
-    # Round up dbmap size
-    dbmap_size = DEFAULT_DBMAP_SIZE
-    while (total_dbsize * DBSIZE_MARGIN > dbmap_size):
-        dbmap_size *= 1.25
+    required_dbsize = round(total_dbsize * DBSIZE_MARGIN)
+
+    # Compute a dbmap size greater than required_dbsize
+    dbmap_size = parse_size(DEFAULT_LMDB_SIZE)
+    while (required_dbsize > dbmap_size):
+        dbmap_size = round(dbmap_size * 1.25)
 
     # Round up number of dbis
     nbdbis = 1
     while nbdbis < total_dbi + DBI_MARGIN:
         nbdbis *= 2
 
-    log.info(f"Required space for LDIF files is about {size_fmt(total_entrysize)}")
-    log.info(f"Required space for DBMAP files is about {size_fmt(dbmap_size)}")
+    log.info(f"Required space for LDIF files is about {format_size(total_entrysize)}")
+    log.info(f"Required space for DBMAP files is about {format_size(required_dbsize)}")
     log.info(f"Required number of dbi is {nbdbis}")
 
     # Generate the info file (so dbscan could generate the map)
@@ -260,20 +256,29 @@ def dblib_bdb2mdb(inst, log, args):
         f.write(f'MAXDBS={nbdbis}\n')
     os.chown(f'{dbmapdir}/{MDB_INFO}', uid, gid)
 
-    if os.stat(dbmapdir).st_dev == os.stat(tmpdir).st_dev:
-        total, used, free = shutil.disk_usage(dbmapdir)
-        if free < total_entrysize + dbmap_size:
-            log.error(f"Not enough space on {dbmapdir} to migrate to lmdb (Need {size_fmt(total_entrysize + dbmap_size)}, Have {size_fmt(free)})")
-            return
-    else:
-        total, used, free = shutil.disk_usage(dbmapdir)
-        if free < dbmap_size:
-            log.error(f"Not enough space on {dbmapdir} to migrate to lmdb (Need {size_fmt(dbmap_size)}, Have {size_fmt(free)})")
-            return
+    total, used, free = shutil.disk_usage(dbmapdir)
+    if os.stat(dbmapdir).st_dev != os.stat(tmpdir).st_dev:
+        # Ldif and db are on different filesystems
+        # Let check that we have enough space in tmpdir for ldif files
         total, used, free = shutil.disk_usage(tmpdir)
         if free < total_entrysize:
-            log.error("Not enough space on {tmpdir} to migrate to lmdb (Need {size_fmt(total_entrysize)}, Have {size_fmt(free)})")
-            return
+            raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to lmdb " +
+                                  "(In {tmpdir}, {format_size(total_entrysize)} is "+
+                                  "needed but only {format_size(free)} is available)")
+        total_entrysize = 0    # do not count total_entrysize when checking dbmapdir size
+
+    # Let check that we have enough space in dbmapdir for the db and ldif files
+    total, used, free = shutil.disk_usage(dbmapdir)
+    size = required_dbsize + total_entrysize
+    if free < required_dbsize + total_entrysize:
+            raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to lmdb " +
+                                  "(In {dbmapdir}, " +
+                                  "{format_size(required_dbsize + total_entrysize)} is "
+                                  "needed but only {format_size(free)} is available)")
+    # Lets use dbmap_size if possible, otherwise use required_dbsize
+    if free < dbmap_size + total_entrysize:
+        dbmap_size = required_dbsize
+
     progress = 0
     encrypt = False       # Should maybe be a args param
     for bename, be in backends.items():
@@ -376,23 +381,26 @@ def dblib_mdb2bdb(inst, log, args):
     # Clearly over evaluated (but better than nothing )
     total_entrysize = dbmap_size
 
-    log.info(f"Required space for LDIF files is about {size_fmt(total_entrysize)}")
-    log.info(f"Required space for bdb files is about {size_fmt(dbmap_size)}")
+    log.info(f"Required space for LDIF files is about {format_size(total_entrysize)}")
+    log.info(f"Required space for bdb files is about {format_size(dbmap_size)}")
 
-    if os.stat(dbmapdir).st_dev == os.stat(tmpdir).st_dev:
-        total, used, free = shutil.disk_usage(dbmapdir)
-        if free < total_entrysize + dbmap_size:
-            log.error(f"Not enough space on {dbmapdir} to migrate to bdb (Need {size_fmt(total_entrysize + dbmap_size)}, Have {size_fmt(free)})")
-            return
-    else:
-        total, used, free = shutil.disk_usage(dbmapdir)
-        if free < dbmap_size:
-            log.error(f"Not enough space on {dbmapdir} to migrate to bdb (Need {size_fmt(dbmap_size)}, Have {size_fmt(free)})")
-            return
+    if os.stat(dbmapdir).st_dev != os.stat(tmpdir).st_dev:
+        # Ldif and db are on different filesystems
+        # Let check that we have enough space for ldif files
         total, used, free = shutil.disk_usage(tmpdir)
         if free < total_entrysize:
-            log.error("Not enough space on {tmpdir} to migrate to bdb (Need {size_fmt(total_entrysize)}, Have {size_fmt(free)})")
-            return
+            raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to bdb " +
+                                  "(In {tmpdir}, {format_size(total_entrysize)} bytes "+
+                                  "are needed but only {format_size(free)} are available)")
+        total_entrysize = 0    # do not count total_entrysize when checking dbmapdir size
+
+    # Let check that we have enough space for the db and ldif files
+    total, used, free = shutil.disk_usage(dbmapdir)
+    if free < dbmap_size + total_entrysize:
+            raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to bdb " +
+                                  "(In {dbmapdir}, {format_size(dbmap_size+total_entrysize)} "+
+                                  "is needed but only {format_size(free)} is available)")
+
     progress = 0
     encrypt = False       # Should maybe be a args param
     total_dbsize = 0

diff --git a/src/lib389/lib389/instance/options.py b/src/lib389/lib389/instance/options.py
@@ -12,7 +12,7 @@
 import random
 from lib389.paths import Paths
 from lib389._constants import INSTALL_LATEST_CONFIG
-from lib389.utils import get_default_db_lib, socket_check_bind
+from lib389.utils import get_default_db_lib, get_default_mdb_max_size, socket_check_bind
 
 MAJOR, MINOR, _, _, _ = sys.version_info
 
@@ -38,6 +38,7 @@
     'db_lib',
     'ldapi',
     'ldif_dir',
+    'mdb_max_size',
     'lock_dir',
     'log_dir',
     'run_dir',
@@ -322,7 +323,12 @@ def __init__(self, log):
         self._options['db_lib'] = get_default_db_lib()
         self._type['db_lib'] = str
         self._helptext['db_lib'] = "Select the database implementation library (bdb or mdb)."
-        self._advanced['db_lib'] = True
+        self._advanced['db_lib'] = False
+
+        self._options['mdb_max_size'] = get_default_mdb_max_size(ds_paths)
+        self._type['mdb_max_size'] = str
+        self._helptext['mdb_max_size'] = "Select the lmdb database maximum size."
+        self._advanced['mdb_max_size'] = False
 
         self._options['ldif_dir'] = ds_paths.ldif_dir
         self._type['ldif_dir'] = str