diff --git a/tasktiger/worker.py b/tasktiger/worker.py
index 93785be0..16aab09b 100644
--- a/tasktiger/worker.py
+++ b/tasktiger/worker.py
@@ -1159,6 +1159,12 @@ def _refresh_queue_set(self):
         queued_key = self._key(QUEUED)
         ignored_key = self._ignored_queues_key()
 
+        if not ignored_key:
+            self._queue_set = set(
+                self._filter_queues(self.connection.smembers(queued_key))
+            )
+            return
+
         queues = self.connection.sdiff(queued_key, ignored_key)
         self._queue_set = set(self._filter_queues(queues))
 
@@ -1168,12 +1174,15 @@ def _refresh_queue_set(self):
             self.connection.sinterstore(ignored_key, ignored_key, queued_key)
 
     def _ignored_queues_key(self):
-        signature = ",".join(
+        signature = [
             *sorted(self.only_queues),
             *sorted(f"-{queue}" for queue in self.exclude_queues),
-        )
+        ]
+
+        if signature:
+            return self._key(IGNORED, ",".join(signature))
 
-        return self._key(IGNORED, signature)
+        return None
 
     def run(self, once=False, force_once=False):
         """