From c12b9b9943097b4a30f98913b27614002f74b0bf Mon Sep 17 00:00:00 2001 From: k1o0 Date: Tue, 12 Nov 2024 21:48:02 +0200 Subject: [PATCH] Improve setup (#164) * Updated setup function to reset parameters and updated the FAQs. * Setup in silent mode now uses defaults if base url matches default one * Resolves #155 * Rerun setup when default client params deleted * FAQ formatting --------- Co-authored-by: Rai Pranav --- CHANGELOG.md | 2 ++ docs/FAQ.md | 39 +++++++++++++++++++++++++-------------- one/params.py | 36 +++++++++++++++++++++++++++++++++--- one/tests/test_one.py | 14 ++++++++++++-- one/tests/test_params.py | 32 ++++++++++++++++++++++++++++---- 5 files changed, 100 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f837857..03957f89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Also adds a new ALFPath class to replace alf path functions. - bugfix: one.alf.path.full_path_parts didn't always raise when invalid path passed - one.alf.path module containing ALFPath class - ALF cache table generation has lower memory footprint +- setup in silent mode now uses defaults if base url matches default one ### Added @@ -21,6 +22,7 @@ Also adds a new ALFPath class to replace alf path functions. - one.alf.cache.EMPTY_DATASETS_FRAME and EMPTY_SESSION_FRAME vars for table column, index, and dtype template - pyproject.toml replaces deprecated setup file - one.alf.exceptions.InvalidALF exception +- one.params.delete_params ### Removed diff --git a/docs/FAQ.md b/docs/FAQ.md index 562c38a9..1c41b67f 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -24,20 +24,20 @@ Note: There are two different definitions of caches that are used in ONE2: This is refreshed every night and uploaded to Flatiron and downloaded onto your computer every 24hr (this is what the datetime object returned as output of the `ONE().refresh_cache('remote')` command is showing, i.e. when this cache was last updated). -This table is used in all one.search, one.load, one.list functions. When doing -`ONE().refresh_cache('remote')`, you are basically forcing ONE to re-download this table +This table is used in all one.search, one.load, one.list functions. When doing +`ONE().refresh_cache('remote')`, you are basically forcing ONE to re-download this table regardless of when it was last downloaded from Flatiron. -2. When running remote queries (anything that uses `one.alyx.rest(....)`), -ONE stores the results of these queries for 24 hours, so that if you -repeatedly make the same query over and over you don't hit the database +2. When running remote queries (anything that uses `one.alyx.rest(....)`), +ONE stores the results of these queries for 24 hours, so that if you +repeatedly make the same query over and over you don't hit the database each time but can use the local cached result. A problem can arise if something on the Alyx database changes in between the same query: - For example, at time X a given query returns an empty result (e.g. no histology session for a given subject). At time X+1, data is registered onto Alyx. At time X+2, you run the same query again. - Because you had already made the query earlier, ONE uses the local result that - it had previously and displays that there isn't a histology session. + Because you had already made the query earlier, ONE uses the local result that + it had previously and displays that there isn't a histology session. To circumvent this, use the `no_cache=True` argument in `one.alyx.rest(..., no_cache=True)` or the `no_cache` web client context. More information can be found [here](https://int-brain-lab.github.io/ONE/notebooks/one_modes.html#REST-caching). Use this only if necessary, as these methods are not optimized. @@ -48,6 +48,15 @@ Usually you can re-run your setup with the following command: from one.api import ONE ONE.setup(base_url='https://alyx.example.com') ``` +⚠️ Note: 'alyx.example.com' is just an example URL - replace with your actual database URL + +## How do I reset my ONE parameters to use Open Alyx? +To reset your ONE configuration to use the public Open Alyx database with default settings: +```python +from one.api import ONE +ONE.setup(base_url='https://openalyx.internationalbrainlab.org', make_default=True) +``` +**Note**: The `make_default=True` argument saves these settings as your default configuration. This means future ONE instances will use these settings unless specified otherwise. ## How do I change my download (a.k.a. cache) directory? To **permanently** change the directory, simply re-run the setup routine: @@ -64,7 +73,9 @@ from one.api import ONE one = ONE(base_url='https://alyx.example.com', cache_dir=Path.home() / 'new_download_dir') ``` -**NB**: This will (down)load the cache tables in the newly specified location. To avoid this, specify the cache table location separately using the `tables_dir` kwarg. +⚠️ Note: 'alyx.example.com' is just an example URL - replace with your actual database URL + +**Note**: This will (down)load the cache tables in the newly specified location. To avoid this, specify the cache table location separately using the `tables_dir` kwarg. ## How do I load cache tables from a different location? By default, the cache tables are in the cache_dir root. You can load cache tables in a different location in the following two ways: @@ -77,7 +88,7 @@ one = ONE(tables_dir=Path.home() / 'tables_dir') # 2. Specify location after instantiation one.load_cache(Path.home() / 'tables_dir') ``` -**NB**: Avoid using the same location for different database cache tables: by default ONE will automatically overwrite tables when a newer version is available. To avoid automatic downloading, set `mode='local'`. +**Note**: Avoid using the same location for different database cache tables: by default ONE will automatically overwrite tables when a newer version is available. To avoid automatic downloading, set `mode='local'`. ## How do check who I'm logged in as? ```python @@ -103,15 +114,15 @@ one.alyx.authenticate(username='other_user', cache_token=False, force=True) ``` ## What to do if I am seeing a certificate error? -If you are using the Windows platform, you may see a certificate error when initially trying to connect with ONE. The last few -lines of the traceback should like this: +If you are using the Windows platform, you may see a certificate error when initially trying to connect with ONE. The last few +lines of the traceback should like this: ```powershell File "C:\Users\User\anaconda3\envs\ONE\lib\urllib\request.py", line 1351, in do_open raise URLError(err) urllib.error.URLError: ``` This has a relatively easy fix: -* Open `Microsoft Edge` or `Internet Explorer` and navigate to the URL https://alyx.internationalbrainlab.org, or whichever alyx +* Open `Microsoft Edge` or `Internet Explorer` and navigate to the URL https://alyx.internationalbrainlab.org, or whichever alyx site you are attempting to access with ONE (no need to log in) * Reattempt to run any ONE query or setup on the command line * Simply visiting the website with a Microsoft web browser should be enough to get the site's certificate to be stored properly. @@ -139,7 +150,7 @@ You can check your version with the following: `print(ONE.version)`.\ The latest version can be found in the CHANGELOG, [here](https://github.com/int-brain-lab/ONE/blob/main/CHANGELOG.md). \ To update to the latest available version run `pip install -U ONE-api`. -## How do I use ONE in a read-only environment? +## How do I use ONE in a read-only environment? To use ONE without any write access or internet access, simply instantiate in local mode: ```python from one.api import ONE @@ -155,7 +166,7 @@ one = ONE(base_url='https://openalyx.internationalbrainlab.org', cache_rest=None assert one.offline and one.alyx.cache_mode is None ``` -## Why does the search return a LazyID object? +## Why does the search return a LazyID object? When in remote mode using one.search or one.search_insertions, a LazyID object is returned instead of a list. It behaves exactly the same as a list (you can index, slice and get its length). Instead of retrieving all the values from the database query it will fetch only the items you index from the list. This greatly speeds up diff --git a/one/params.py b/one/params.py index db378487..842aa05f 100644 --- a/one/params.py +++ b/one/params.py @@ -110,11 +110,18 @@ def setup(client=None, silent=False, make_default=None, username=None, cache_dir """ # First get default parameters par_default = default() - client_key = _key_from_url(client or par_default.ALYX_URL) + default_url = par_default.ALYX_URL + client_key = _key_from_url(client or default_url) # If a client URL has been provided, set it as the default URL - par_default = par_default.set('ALYX_URL', client or par_default.ALYX_URL) - par_current = iopar.read(f'{_PAR_ID_STR}/{client_key}', par_default) + par_default = par_default.set('ALYX_URL', client or default_url) + + # When silent=True, if setting up default database use default parameters + # instead of current ones to reset credentials + if silent and client_key == _key_from_url(default_url): + par_current = par_default + else: + par_current = iopar.read(f'{_PAR_ID_STR}/{client_key}', par_default) if username: par_current = par_current.set('ALYX_LOGIN', username) @@ -232,6 +239,7 @@ def get(client=None, silent=False, username=None): IBLParams A Params object for the AlyxClient. """ + client = client or get_default_client(include_schema=True) client_key = _key_from_url(client) if client else None cache_map = iopar.read(f'{_PAR_ID_STR}/{_CLIENT_ID_STR}', {}) # If there are no params for this client, run setup routine @@ -335,6 +343,28 @@ def check_cache_conflict(cache_dir): assert not any(x == str(cache_dir) for x in cache_map.values()) +def delete_params(base_url=None): + """Delete parameter files. + + This will fully reset the ONE database and remote client parameters. + + Parameters + ---------- + base_url : str, optional + If provided, delete specific database parameters. If None, all parameters are removed. + """ + if base_url: + client_key = _key_from_url(base_url) + params_file = Path(iopar.getfile(f'{_PAR_ID_STR}/{client_key}')) + if params_file.exists(): + params_file.unlink() + else: + warnings.warn(f'{base_url}: params file not found') + else: + if (params_dir := get_params_dir()).exists(): + shutil.rmtree(params_dir) + + def _patch_params(par): """ Patch previous version of parameters, if required. diff --git a/one/tests/test_one.py b/one/tests/test_one.py index f741af62..52c59788 100644 --- a/one/tests/test_one.py +++ b/one/tests/test_one.py @@ -1905,10 +1905,16 @@ def test_setup_silent(self): client_pars = Path(self.tempdir.name).rglob(f'.{one_obj.alyx.base_url.split("/")[-1]}') self.assertEqual(len(list(client_pars)), 1) - # Check uses defaults on second instantiation with mock.patch('iblutil.io.params.getfile', new=self.get_file): + # Check uses defaults on second instantiation one_obj = ONE(mode='local') - self.assertEqual(one_obj.alyx.base_url, one.params.default().ALYX_URL) + url = one.params.default().ALYX_URL + self.assertEqual(url, one_obj.alyx.base_url) + # With the default database in silent mode the defaults should be used + one.params.save(one_obj.alyx._par.set('ALYX_LOGIN', 'foobar'), url) + one.params.setup(url, silent=True) + one_obj = ONE(mode='local') + self.assertEqual(one.params.default().ALYX_LOGIN, one_obj.alyx._par.ALYX_LOGIN) # Check saves base_url arg with self.subTest('Test setup with base URL'): @@ -1919,6 +1925,10 @@ def test_setup_silent(self): self.assertEqual(one_obj.alyx.base_url, TEST_DB_1['base_url']) params_url = one.params.get(client=TEST_DB_1['base_url']).ALYX_URL self.assertEqual(params_url, one_obj.alyx.base_url) + # With non-default database in silent mode the previous pars should be used + one.params.setup(params_url, silent=True) + user = one.params.get(client=params_url).ALYX_LOGIN + self.assertEqual(TEST_DB_1['username'], user) def test_setup_username(self): """Test setting up parameters with a provided username. diff --git a/one/tests/test_params.py b/one/tests/test_params.py index 9aeccd28..abd03d69 100644 --- a/one/tests/test_params.py +++ b/one/tests/test_params.py @@ -88,12 +88,12 @@ def test_setup(self, _): class TestONEParamUtil(unittest.TestCase): - """Test class for one.params utility functions""" + """Test class for one.params utility functions.""" def setUp(self) -> None: pass def test_key_from_url(self): - """Test for one.params._key_from_url""" + """Test for one.params._key_from_url.""" key = params._key_from_url('https://sub.domain.org/') self.assertEqual(key, 'sub.domain.org') @@ -109,7 +109,7 @@ def test_get_params_dir(self): self.assertEqual('path/to/params/.one', path.as_posix()) def test_get_default_client(self): - """Test for one.params.get_default_client""" + """Test for one.params.get_default_client.""" temp_dir = util.set_up_env() self.addCleanup(temp_dir.cleanup) with mock.patch('iblutil.io.params.getfile', new=partial(util.get_file, temp_dir.name)): @@ -123,7 +123,7 @@ def test_get_default_client(self): self.assertEqual(client, 'openalyx.internationalbrainlab.org') def test_get_cache_dir(self): - """Test for one.params.get_cache_dir""" + """Test for one.params.get_cache_dir.""" temp_dir = util.set_up_env() cache_dir = Path(temp_dir.name) / 'download_cache' assert not cache_dir.exists() @@ -134,6 +134,30 @@ def test_get_cache_dir(self): self.assertEqual(cache_dir, out) self.assertTrue(cache_dir.exists()) + def test_delete_params(self): + """Test for one.params.delete_params.""" + with tempfile.TemporaryDirectory() as tmp: + par_dir = Path(tmp, f'.{params._PAR_ID_STR}') + # Change the location of the parameters to our temp dir + get_file = partial(util.get_file, tmp) + with mock.patch('iblutil.io.params.getfile', new=get_file): + # Set up some params + params.setup(silent=True) + assert par_dir.exists() + # Test deleting all params + params.delete_params() + self.assertFalse(par_dir.exists()) + # Test deleting specific params + params.setup(silent=True) + url = params.default().ALYX_URL + caches_file = par_dir.joinpath('.caches') + db_params = par_dir.joinpath(f'.{url[8:]}') + assert caches_file.exists() and db_params.exists() + params.delete_params(params.default().ALYX_URL) + self.assertFalse(db_params.exists()) + self.assertTrue(caches_file.exists()) + self.assertWarns(UserWarning, params.delete_params, params.default().ALYX_URL) + if __name__ == '__main__': unittest.main(exit=False, verbosity=2)