diff --git a/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py b/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py index 22dd052092..44316d2d7b 100644 --- a/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py +++ b/dirsrvtests/tests/suites/clu/dsctl_dblib_test.py @@ -44,7 +44,7 @@ def init_user(topo_m2, request): def fin(): try: test_user.delete() - except ldap.NO_SUCH_OBJECT: + except (ldap.NO_SUCH_OBJECT, ldap.SERVER_DOWN): pass request.addfinalizer(fin) diff --git a/dirsrvtests/tests/suites/config/config_test.py b/dirsrvtests/tests/suites/config/config_test.py index 145d69575c..69882d3c99 100644 --- a/dirsrvtests/tests/suites/config/config_test.py +++ b/dirsrvtests/tests/suites/config/config_test.py @@ -10,17 +10,20 @@ import logging import pytest import os -from lib389 import pid_from_file +from lib389 import DirSrv, pid_from_file from lib389.tasks import * from lib389.topologies import topology_m2, topology_st as topo from lib389.utils import * from lib389._constants import DN_CONFIG, DEFAULT_SUFFIX, DEFAULT_BENAME +from lib389._mapped_object import DSLdapObjects +from lib389.cli_base import FakeArgs +from lib389.cli_conf.backend import db_config_set from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES from lib389.idm.group import Groups -from lib389.backend import * +from lib389.instance.setup import SetupDs from lib389.config import LDBMConfig, BDB_LDBMConfig, Config from lib389.cos import CosPointerDefinitions, CosTemplates -from lib389.backend import Backends +from lib389.backend import Backends, DatabaseConfig from lib389.monitor import MonitorLDBM, Monitor from lib389.plugins import ReferentialIntegrityPlugin @@ -665,6 +668,88 @@ def test_changing_threadnumber(topo): time.sleep(3) check_number_of_threads(int(cfgnbthreads), monitor, pid) + +@pytest.fixture(scope="module") +def create_lmdb_instance(request): + verbose = log.level > logging.DEBUG + instname = 'i_lmdb' + assert SetupDs(verbose=True, log=log).create_from_dict( { + 'general' : {}, + 'slapd' : { + 'instance_name': instname, + 'db_lib': 'mdb', + 'mdb_max_size': '0.5 Gb', + }, + 'backend-userroot': { + 'sample_entries': 'yes', + 'suffix': DEFAULT_SUFFIX, + }, + } ) + inst = DirSrv(verbose=verbose, external_log=log) + inst.local_simple_allocate(instname, binddn=DN_DM, password=PW_DM) + inst.setup_ldapi() + + def fin(): + inst.delete() + + request.addfinalizer(fin) + inst.open() + return inst + + +def set_and_check(inst, db_config, dsconf_attr, ldap_attr, val): + val = str(val) + args = FakeArgs() + setattr(args, dsconf_attr, val) + db_config_set(inst, db_config.dn, log, args) + cfg_vals = db_config.get() + assert ldap_attr in cfg_vals + assert cfg_vals[ldap_attr][0] == val + + +def test_lmdb_config(create_lmdb_instance): + """Test nsslapd-ignore-virtual-attrs configuration attribute + + :id: bca28086-61cf-11ee-a064-482ae39447e5 + :setup: Custom instance named 'i_lmdb' having db_lib=mdb and lmdb_size=0.5 + :steps: + 1. Get dscreate create-template output + 2. Check that 'db_lib' is in output + 3. Check that 'lmdb_size' is in output + 4. Get the database config + 5. Check that nsslapd-backend-implement is mdb + 6. Check that nsslapd-mdb-max-size is 536870912 (i.e 0.5Gb) + 7. Set a value for nsslapd-mdb-max-size and test the value is properly set + 8. Set a value for nsslapd-mdb-max-readers and test the value is properly set + 9. Set a value for nsslapd-mdb-max-dbs and test the value is properly set + :expectedresults: + 1. Success + 2. Success + 3. Success + 4. Success + 5. Success + 6. Success + 7. Success + 8. Success + 9. Success + """ + + res = subprocess.run(('dscreate', 'create-template'), stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, encoding='utf-8') + inst = create_lmdb_instance + assert 'db_lib' in res.stdout + assert 'mdb_max_size' in res.stdout + db_config = DatabaseConfig(inst) + cfg_vals = db_config.get() + assert 'nsslapd-backend-implement' in cfg_vals + assert cfg_vals['nsslapd-backend-implement'][0] == 'mdb' + assert 'nsslapd-mdb-max-size' in cfg_vals + assert cfg_vals['nsslapd-mdb-max-size'][0] == '536870912' + set_and_check(inst, db_config, 'mdb_max_size', 'nsslapd-mdb-max-size', parse_size('2G')) + set_and_check(inst, db_config, 'mdb_max_readers', 'nsslapd-mdb-max-readers', 200) + set_and_check(inst, db_config, 'mdb_max_dbs', 'nsslapd-mdb-max-dbs', 200) + + if __name__ == '__main__': # Run isolated # -s for DEBUG mode diff --git a/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h b/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h index ce46c08cb7..faa18dab5a 100644 --- a/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h +++ b/ldap/servers/slapd/back-ldbm/db-mdb/mdb_layer.h @@ -45,7 +45,7 @@ #define DBMDB_DB_MINSIZE ( 4LL * MEGABYTE ) #define DBMDB_DISK_RESERVE(disksize) ((disksize)*2ULL/1000ULL) #define DBMDB_READERS_MARGIN 10 -#define DBMDB_READERS_DEFAULT 50 +#define DBMDB_READERS_DEFAULT 126 /* default value as described in mdb_env_set_maxreaders */ #define DBMDB_DBS_MARGIN 10 #define DBMDB_DBS_DEFAULT 128 diff --git a/src/lib389/lib389/_constants.py b/src/lib389/lib389/_constants.py index e3188eae74..f640e1e13a 100644 --- a/src/lib389/lib389/_constants.py +++ b/src/lib389/lib389/_constants.py @@ -241,6 +241,7 @@ class TaskWarning(IntEnum): VALGRIND_INVALID_STR = " Invalid (free|read|write)" DISORDERLY_SHUTDOWN = ('Detected Disorderly Shutdown last time Directory ' 'Server was running, recovering database') +DEFAULT_LMDB_SIZE = '20Gb' # # LOG: see https://access.redhat.com/documentation/en-US/Red_Hat_Directory diff --git a/src/lib389/lib389/_mapped_object.py b/src/lib389/lib389/_mapped_object.py index 490e47c0ac..9799b8839c 100644 --- a/src/lib389/lib389/_mapped_object.py +++ b/src/lib389/lib389/_mapped_object.py @@ -151,6 +151,9 @@ def __unicode__(self): def __str__(self): return self.__unicode__() + def __repr__(self): + return f'{type(self)}(instance_name="{self._instance.serverid}", dn="{self.__unicode__()}")' + def _unsafe_raw_entry(self): """Get an Entry object diff --git a/src/lib389/lib389/cli_conf/backend.py b/src/lib389/lib389/cli_conf/backend.py index 1ac9217ed2..87a5646cd3 100644 --- a/src/lib389/lib389/cli_conf/backend.py +++ b/src/lib389/lib389/cli_conf/backend.py @@ -65,6 +65,9 @@ 'deadlock_policy': 'nsslapd-db-deadlock-policy', 'db_home_directory': 'nsslapd-db-home-directory', 'db_lib': 'nsslapd-backend-implement', + 'mdb_max_size': 'nsslapd-mdb-max-size', + 'mdb_max_readers': 'nsslapd-mdb-max-readers', + 'mdb_max_dbs': 'nsslapd-mdb-max-dbs', # VLV attributes 'search_base': 'vlvbase', 'search_scope': 'vlvscope', @@ -1081,6 +1084,10 @@ def create_parser(subparsers): set_db_config_parser.add_argument('--deadlock-policy', help='Adjusts the backend database deadlock policy (Advanced setting)') set_db_config_parser.add_argument('--db-home-directory', help='Sets the directory for the database mmapped files (Advanced setting)') set_db_config_parser.add_argument('--db-lib', help='Sets which db lib is used. Valid values are: bdb or mdb') + set_db_config_parser.add_argument('--mdb-max-size', help='Sets the lmdb database maximum size (in bytes).') + set_db_config_parser.add_argument('--mdb-max-readers', help='Sets the lmdb database maximum number of readers (Advanced setting)') + set_db_config_parser.add_argument('--mdb-max-dbs', help='Sets the lmdb database maximum number of sub databases (Advanced setting)') + ####################################################### # Database & Suffix Monitor diff --git a/src/lib389/lib389/cli_ctl/dblib.py b/src/lib389/lib389/cli_ctl/dblib.py index e4fd301e22..56559b8ab9 100644 --- a/src/lib389/lib389/cli_ctl/dblib.py +++ b/src/lib389/lib389/cli_ctl/dblib.py @@ -15,11 +15,13 @@ import glob import shutil from lib389.dseldif import DSEldif +from lib389._constants import DEFAULT_LMDB_SIZE +from lib389.utils import parse_size, format_size import subprocess +from errno import ENOSPC DBLIB_LDIF_PREFIX = "__dblib-" -DEFAULT_DBMAP_SIZE = 2 * 1024 * 1024 * 1024 DBSIZE_MARGIN = 1.2 DBI_MARGIN = 60 @@ -142,14 +144,6 @@ def get_mdb_dbis(dbdir): return result -def size_fmt(num): - for unit in ["B", "KB", "MB", "GB", "TB"]: - if (num < 1024): - return f"{num:3.1f} {unit}" - num /= 1024 - return f"{num:.1f} TB" - - def run_dbscan(args): prefix = os.environ.get('PREFIX', "") prog = f'{prefix}/bin/dbscan' @@ -235,18 +229,20 @@ def dblib_bdb2mdb(inst, log, args): total_entrysize += be['entrysize'] total_dbi += be['dbi'] - # Round up dbmap size - dbmap_size = DEFAULT_DBMAP_SIZE - while (total_dbsize * DBSIZE_MARGIN > dbmap_size): - dbmap_size *= 1.25 + required_dbsize = round(total_dbsize * DBSIZE_MARGIN) + + # Compute a dbmap size greater than required_dbsize + dbmap_size = parse_size(DEFAULT_LMDB_SIZE) + while (required_dbsize > dbmap_size): + dbmap_size = round(dbmap_size * 1.25) # Round up number of dbis nbdbis = 1 while nbdbis < total_dbi + DBI_MARGIN: nbdbis *= 2 - log.info(f"Required space for LDIF files is about {size_fmt(total_entrysize)}") - log.info(f"Required space for DBMAP files is about {size_fmt(dbmap_size)}") + log.info(f"Required space for LDIF files is about {format_size(total_entrysize)}") + log.info(f"Required space for DBMAP files is about {format_size(required_dbsize)}") log.info(f"Required number of dbi is {nbdbis}") # Generate the info file (so dbscan could generate the map) @@ -260,20 +256,29 @@ def dblib_bdb2mdb(inst, log, args): f.write(f'MAXDBS={nbdbis}\n') os.chown(f'{dbmapdir}/{MDB_INFO}', uid, gid) - if os.stat(dbmapdir).st_dev == os.stat(tmpdir).st_dev: - total, used, free = shutil.disk_usage(dbmapdir) - if free < total_entrysize + dbmap_size: - log.error(f"Not enough space on {dbmapdir} to migrate to lmdb (Need {size_fmt(total_entrysize + dbmap_size)}, Have {size_fmt(free)})") - return - else: - total, used, free = shutil.disk_usage(dbmapdir) - if free < dbmap_size: - log.error(f"Not enough space on {dbmapdir} to migrate to lmdb (Need {size_fmt(dbmap_size)}, Have {size_fmt(free)})") - return + total, used, free = shutil.disk_usage(dbmapdir) + if os.stat(dbmapdir).st_dev != os.stat(tmpdir).st_dev: + # Ldif and db are on different filesystems + # Let check that we have enough space in tmpdir for ldif files total, used, free = shutil.disk_usage(tmpdir) if free < total_entrysize: - log.error("Not enough space on {tmpdir} to migrate to lmdb (Need {size_fmt(total_entrysize)}, Have {size_fmt(free)})") - return + raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to lmdb " + + "(In {tmpdir}, {format_size(total_entrysize)} is "+ + "needed but only {format_size(free)} is available)") + total_entrysize = 0 # do not count total_entrysize when checking dbmapdir size + + # Let check that we have enough space in dbmapdir for the db and ldif files + total, used, free = shutil.disk_usage(dbmapdir) + size = required_dbsize + total_entrysize + if free < required_dbsize + total_entrysize: + raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to lmdb " + + "(In {dbmapdir}, " + + "{format_size(required_dbsize + total_entrysize)} is " + "needed but only {format_size(free)} is available)") + # Lets use dbmap_size if possible, otherwise use required_dbsize + if free < dbmap_size + total_entrysize: + dbmap_size = required_dbsize + progress = 0 encrypt = False # Should maybe be a args param for bename, be in backends.items(): @@ -376,23 +381,26 @@ def dblib_mdb2bdb(inst, log, args): # Clearly over evaluated (but better than nothing ) total_entrysize = dbmap_size - log.info(f"Required space for LDIF files is about {size_fmt(total_entrysize)}") - log.info(f"Required space for bdb files is about {size_fmt(dbmap_size)}") + log.info(f"Required space for LDIF files is about {format_size(total_entrysize)}") + log.info(f"Required space for bdb files is about {format_size(dbmap_size)}") - if os.stat(dbmapdir).st_dev == os.stat(tmpdir).st_dev: - total, used, free = shutil.disk_usage(dbmapdir) - if free < total_entrysize + dbmap_size: - log.error(f"Not enough space on {dbmapdir} to migrate to bdb (Need {size_fmt(total_entrysize + dbmap_size)}, Have {size_fmt(free)})") - return - else: - total, used, free = shutil.disk_usage(dbmapdir) - if free < dbmap_size: - log.error(f"Not enough space on {dbmapdir} to migrate to bdb (Need {size_fmt(dbmap_size)}, Have {size_fmt(free)})") - return + if os.stat(dbmapdir).st_dev != os.stat(tmpdir).st_dev: + # Ldif and db are on different filesystems + # Let check that we have enough space for ldif files total, used, free = shutil.disk_usage(tmpdir) if free < total_entrysize: - log.error("Not enough space on {tmpdir} to migrate to bdb (Need {size_fmt(total_entrysize)}, Have {size_fmt(free)})") - return + raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to bdb " + + "(In {tmpdir}, {format_size(total_entrysize)} bytes "+ + "are needed but only {format_size(free)} are available)") + total_entrysize = 0 # do not count total_entrysize when checking dbmapdir size + + # Let check that we have enough space for the db and ldif files + total, used, free = shutil.disk_usage(dbmapdir) + if free < dbmap_size + total_entrysize: + raise OSError(ENOSPC, "Not enough space on {tmpdir} to migrate to bdb " + + "(In {dbmapdir}, {format_size(dbmap_size+total_entrysize)} "+ + "is needed but only {format_size(free)} is available)") + progress = 0 encrypt = False # Should maybe be a args param total_dbsize = 0 diff --git a/src/lib389/lib389/instance/options.py b/src/lib389/lib389/instance/options.py index dcdc6d156c..0da5ff0daf 100644 --- a/src/lib389/lib389/instance/options.py +++ b/src/lib389/lib389/instance/options.py @@ -12,7 +12,7 @@ import random from lib389.paths import Paths from lib389._constants import INSTALL_LATEST_CONFIG -from lib389.utils import get_default_db_lib, socket_check_bind +from lib389.utils import get_default_db_lib, get_default_mdb_max_size, socket_check_bind MAJOR, MINOR, _, _, _ = sys.version_info @@ -38,6 +38,7 @@ 'db_lib', 'ldapi', 'ldif_dir', + 'mdb_max_size', 'lock_dir', 'log_dir', 'run_dir', @@ -322,7 +323,12 @@ def __init__(self, log): self._options['db_lib'] = get_default_db_lib() self._type['db_lib'] = str self._helptext['db_lib'] = "Select the database implementation library (bdb or mdb)." - self._advanced['db_lib'] = True + self._advanced['db_lib'] = False + + self._options['mdb_max_size'] = get_default_mdb_max_size(ds_paths) + self._type['mdb_max_size'] = str + self._helptext['mdb_max_size'] = "Select the lmdb database maximum size." + self._advanced['mdb_max_size'] = False self._options['ldif_dir'] = ds_paths.ldif_dir self._type['ldif_dir'] = str diff --git a/src/lib389/lib389/instance/setup.py b/src/lib389/lib389/instance/setup.py index 2bbd54f1d9..fb2d2274b3 100644 --- a/src/lib389/lib389/instance/setup.py +++ b/src/lib389/lib389/instance/setup.py @@ -42,13 +42,17 @@ ensure_str, ensure_list_str, get_default_db_lib, + get_default_mdb_max_size, normalizeDN, + parse_size, socket_check_open, selinux_label_file, selinux_label_port, resolve_selinux_path, selinux_restorecon, selinux_present) +from lib389.backend import DatabaseConfig + ds_paths = Paths() @@ -59,7 +63,7 @@ def get_port(port, default_port, secure=False): # Get the port number for the interactive installer and validate it - while 1: + while True: if secure: val = input('\nEnter secure port number [{}]: '.format(default_port)).rstrip() else: @@ -324,7 +328,7 @@ def create_from_cli(self): general['full_machine_name'] = val # Instance name - adjust defaults once set - while 1: + while True: slapd['instance_name'] = general['full_machine_name'].split('.', 1)[0] # Check if default server id is taken @@ -383,7 +387,7 @@ def create_from_cli(self): slapd['port'] = port # Self-Signed Cert DB - while 1: + while True: val = input('\nCreate self-signed certificate database [yes]: ').rstrip().lower() if val != "": if val== 'no' or val == "n": @@ -411,7 +415,7 @@ def create_from_cli(self): slapd['secure_port'] = False # Root DN - while 1: + while True: val = input('\nEnter Directory Manager DN [{}]: '.format(slapd['root_dn'])).rstrip() if val != '': # Validate value is a DN @@ -426,7 +430,7 @@ def create_from_cli(self): break # Root DN Password - while 1: + while True: rootpw1 = getpass.getpass('\nEnter the Directory Manager password: ').rstrip() if rootpw1 == '': print('Password can not be empty') @@ -446,6 +450,36 @@ def create_from_cli(self): slapd['root_password'] = rootpw1 break + # Database implementation (db_lib) + while True: + vdef = get_default_db_lib() + val = input(f'\nChoose whether mdb or bdb is used. [{vdef}]: ').rstrip().lower() + if val == '': + val = vdef + if val in [ 'bdb', 'mdb' ]: + slapd['db_lib'] = val + break + else: + print('The value "{}" is not "mdb" nor "bdb".'.format(val)) + continue + + # Database size (mdb_max_size) + while slapd['db_lib'] == 'mdb': + try: + vdef = get_default_mdb_max_size(ds_paths) + val = input(f'\nEnter the lmdb database size [{vdef}]: ').rstrip() + if val == '': + val = vdef + val = parse_size(val) + if val <= 0.0: + print('The value should positive.') + continue + slapd['mdb_max_size'] = val + break + except ValueError: + print('The value "{}" is not a valid real number.'.format(val)) + continue + # Backend [{'name': 'userroot', 'suffix': 'dc=example,dc=com'}] backend = {'name': 'userroot', 'suffix': ''} backends = [backend] @@ -457,7 +491,7 @@ def create_from_cli(self): else: suffix += ",dc=" + comp - while 1: + while True: val = input("\nEnter the database suffix (or enter \"none\" to skip) [{}]: ".format(suffix)).rstrip() if val != '': if val.lower() == "none": @@ -476,7 +510,7 @@ def create_from_cli(self): # Add sample entries or root suffix entry? if len(backends) > 0: - while 1: + while True: val = input("\nCreate sample entries in the suffix [no]: ").rstrip().lower() if val != "": if val == "no" or val == "n": @@ -493,7 +527,7 @@ def create_from_cli(self): if 'sample_entries' not in backend: # Check if they want to create the root node entry instead - while 1: + while True: val = input("\nCreate just the top suffix entry [no]: ").rstrip().lower() if val != "": if val == "no" or val == "n": @@ -509,7 +543,7 @@ def create_from_cli(self): break # Start the instance? - while 1: + while True: val = input('\nDo you want to start the instance after the installation? [yes]: ').rstrip().lower() if val == '' or val == 'yes' or val == 'y': # Default behaviour @@ -522,7 +556,7 @@ def create_from_cli(self): continue # Are you ready? - while 1: + while True: val = input('\nAre you ready to install? [no]: ').rstrip().lower() if val == '' or val == "no" or val == 'n': print('Aborting installation...') @@ -563,6 +597,30 @@ def create_from_inf(self, inf_path): return True + def create_from_dict(self, inf_dict): + """ + Will trigger a create from the settings stored in inf_dict. + Note: Unlike in create_from_args, missing options in the dict are + automatically preset to their default value (by _validate_ds_config) + """ + # Get the inf data + self.log.debug("Using inf from %s" % inf_dict) + config = None + try: + config = configparser.ConfigParser() + config.read_dict(inf_dict) + except Exception as e: + self.log.error("Exception %s occured", e) + return False + + self.log.debug("Configuration %s" % config.sections()) + (general, slapd, backends) = self._validate_ds_config(config) + + # Actually do the setup now. + self.create_from_args(general, slapd, backends, self.extra) + + return True + def _prepare_ds(self, general, slapd, backends): self.log.info("Validate installation settings ...") assert_c(general['defaults'] is not None, "Configuration defaults in section [general] not found") @@ -588,6 +646,8 @@ def _prepare_ds(self, general, slapd, backends): assert_c(socket.gethostbyname(general['full_machine_name']), "Strict hostname check failed. Check your DNS records for %s" % general['full_machine_name']) self.log.debug("PASSED: Hostname strict checking") + assert_c(slapd['db_lib'] in ['bdb', 'mdb'], "Invalid value for slapd['db_lib'] (should be 'bdb' or 'mdb'") + assert_c(slapd['prefix'] is not None, "Configuration prefix in section [slapd] not found") if (slapd['prefix'] != ""): assert_c(os.path.exists(slapd['prefix']), "Prefix location '%s' not found" % slapd['prefix']) @@ -994,6 +1054,11 @@ def _install_ds(self, general, slapd, backends): if slapd['self_sign_cert']: ds_instance.config.set('nsslapd-security', 'on') + # Before we create any backends, set lmdb max size + if slapd['db_lib'] == 'mdb': + mdb_max_size = parse_size(slapd['mdb_max_size']) + DatabaseConfig(ds_instance).set([('nsslapd-mdb-max-size', str(mdb_max_size)),]) + # Before we create any backends, create any extra default indexes that may be # dynamically provisioned, rather than from template-dse.ldif. Looking at you # entryUUID (requires rust enabled). diff --git a/src/lib389/lib389/properties.py b/src/lib389/lib389/properties.py index 7298450982..83ac717fc7 100644 --- a/src/lib389/lib389/properties.py +++ b/src/lib389/lib389/properties.py @@ -27,6 +27,7 @@ SER_LDAPI_SOCKET = 'ldapi_socket' SER_LDAPI_AUTOBIND = 'ldapi_autobind' SER_INST_SCRIPTS_ENABLED = 'InstScriptsEnabled' +SER_DB_LIB = 'db_lib' SER_PROPNAME_TO_ATTRNAME = {SER_HOST: 'nsslapd-localhost', SER_PORT: 'nsslapd-port', @@ -38,6 +39,7 @@ SER_LDAPI_ENABLED: 'nsslapd-ldapilisten', SER_LDAPI_SOCKET: 'nsslapd-ldapifilepath', SER_LDAPI_AUTOBIND: 'nsslapd-ldapiautobind', + SER_DB_LIB: 'nsslapd-backend-implement', } # # Those WITHOUT related attribute name diff --git a/src/lib389/lib389/utils.py b/src/lib389/lib389/utils.py index 97b0dda4c2..e81da689b8 100644 --- a/src/lib389/lib389/utils.py +++ b/src/lib389/lib389/utils.py @@ -54,7 +54,8 @@ def wait(self): from lib389.dseldif import DSEldif from lib389._constants import ( DEFAULT_USER, VALGRIND_WRAPPER, DN_CONFIG, CFGSUFFIX, LOCALHOST, - ReplicaRole, CONSUMER_REPLICAID, SENSITIVE_ATTRS, DEFAULT_DB_LIB + ReplicaRole, CONSUMER_REPLICAID, SENSITIVE_ATTRS, DEFAULT_DB_LIB, + DEFAULT_LMDB_SIZE ) from lib389.properties import ( SER_HOST, SER_USER_ID, SER_GROUP_ID, SER_STRICT_HOSTNAME_CHECKING, SER_PORT, @@ -180,6 +181,12 @@ def wait(self): "~": u"\u02de", } +# +# Size parser constants +# +SIZE_UNITS = { 't': 2**40, 'g': 2**30, 'm': 2**20, 'k': 2**10, '': 1, } +SIZE_PATTERN = r'\s*(\d*\.?\d*)\s*([tgmk]?)b?\s*' + # # Utilities # @@ -1701,10 +1708,71 @@ def is_valid_hostname(hostname): return all(allowed.match(x) for x in hostname.split(".")) +def parse_size(size): + """ + Parse a string representing a size (like "5 kb" or "2.5Gb") and + return the size in bytes. + :param size: The size to parse + :type size: str: + :return int: + :raise ValueError: if the string cannot be parsed. + """ + try: + val,unit = re.fullmatch(SIZE_PATTERN, size, flags=re.IGNORECASE).groups() + return round(float(val) * SIZE_UNITS[unit.lower()]) + except AttributeError: + raise ValueError(f'Unable to parse "{size}" as a size.') + + +def format_size(size): + """ + Return a string representing a size (like "5 kb" or "2.5Gb") + :param size: The size int bytes to format + :type size: int: + :return str: + """ + for unit in ["B", "KB", "MB", "GB",]: + if (size < 1024): + return f"{size:3.1f} {unit}" + size /= 1024 + return f"{size:.1f} TB" + + def get_default_db_lib(): + """ + Get the default value for the database implementation + :return str: Should be either 'bdb' or 'mdb' + """ return os.getenv('NSSLAPD_DB_LIB', default=DEFAULT_DB_LIB) +def get_default_mdb_max_size(paths): + """ + Get the default maximum size for the lmdb database. + :return str: A size that can be parsed with parse_size() + """ + if paths is None: + paths = Paths() + mdb_max_size = DEFAULT_LMDB_SIZE + size = parse_size(mdb_max_size) + # Make sure that there is enough available disk space + # otherwise decrease the value + dbdir = paths.db_dir + while '{' in dbdir: + dbdir = os.path.dirname(dbdir) + try: + statvfs = os.statvfs(dbdir) + avail = statvfs.f_frsize * statvfs.f_bavail + avail *= 0.8 # Reserve 20% as margin + if size > avail: + mdb_max_size = str(avail) + except (TimeoutError, InterruptedError) as e: + raise e + except OSError as e: + log.warning(f'Cannot determine the free space in the file system containing {dbdir} because of {e}') + return mdb_max_size + + def is_fips(): if os.path.exists('/proc/sys/crypto/fips_enabled'): with open('/proc/sys/crypto/fips_enabled', 'r') as f: