diff --git a/lua/persisted/init.lua b/lua/persisted/init.lua
index f3aab9c..6493be1 100644
--- a/lua/persisted/init.lua
+++ b/lua/persisted/init.lua
@@ -190,14 +190,11 @@ end
 ---@param opts? {dir?: string}
 ---@return boolean
 function M.allowed_dir(opts)
-  if next(config.allowed_dirs) == nil and next(config.ignored_dirs) == nil then
-    return true
-  end
-
   opts = opts or {}
   local dir = opts.dir or vim.fn.getcwd()
 
-  return utils.dirs_match(dir, config.allowed_dirs) and not utils.dirs_match(dir, config.ignored_dirs)
+  return (next(config.allowed_dirs) and utils.dirs_match(dir, config.allowed_dirs) or true)
+    and not (next(config.ignored_dirs) and utils.dirs_match(dir, config.ignored_dirs) or false)
 end
 
 ---Get an ordered list of sessions, sorted by modified time
diff --git a/tests/dirs_spec.lua b/tests/dirs_spec.lua
index e35f9e1..5328855 100644
--- a/tests/dirs_spec.lua
+++ b/tests/dirs_spec.lua
@@ -25,4 +25,18 @@ describe("Directory utilities:", function()
     match = utils.dirs_match(cwd, allowed_dirs)
     assert.equals(true, match)
   end)
+
+  it("can handle only ignore directories", function()
+    local cwd = "~/Code/Neovim/persisted.nvim"
+    local allowed_dirs = {}
+    local ignored_dirs = { { "/tmp" } }
+    local allowed_match = utils.dirs_match(cwd, allowed_dirs)
+    local ignored_match = utils.dirs_match(cwd, ignored_dirs)
+    -- This looks weird, I know. That is because we expect dirs_match to return
+    -- false for allowed_dirs since allowed dirs is empty.
+    -- Therefore this is actually testing to ensure we are getting false and false
+    -- This test specifically addresses the change added in
+    -- https://github.com/olimorris/persisted.nvim/pull/152
+    assert.equals(true, not allowed_match and not ignored_match)
+  end)
 end)