From 5a4d8a71075a4529be8d5ef22d76d69861240f45 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 20 Feb 2023 09:39:33 -0500 Subject: [PATCH] Closes #11787: Rebuild any missing search cache entires after upgrade --- docs/release-notes/version-3.4.md | 2 ++ netbox/extras/management/commands/reindex.py | 31 ++++++++++++++------ netbox/netbox/search/backends.py | 14 ++++++++- upgrade.sh | 5 ++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/release-notes/version-3.4.md b/docs/release-notes/version-3.4.md index 485b8571931..a006eb7bf42 100644 --- a/docs/release-notes/version-3.4.md +++ b/docs/release-notes/version-3.4.md @@ -7,6 +7,7 @@ * [#11110](https://github.com/netbox-community/netbox/issues/11110) - Add `start_address` and `end_address` filters for IP ranges * [#11592](https://github.com/netbox-community/netbox/issues/11592) - Introduce `FILE_UPLOAD_MAX_MEMORY_SIZE` configuration parameter * [#11685](https://github.com/netbox-community/netbox/issues/11685) - Match on containing prefixes and aggregates when querying for IP addresses using global search +* [#11787](https://github.com/netbox-community/netbox/issues/11787) - Upgrade script will automatically rebuild missing search cache ### Bug Fixes @@ -20,6 +21,7 @@ * [#11683](https://github.com/netbox-community/netbox/issues/11683) - Fix CSV header attribute detection when auto-detecting import format * [#11711](https://github.com/netbox-community/netbox/issues/11711) - Fix CSV import for multiple-object custom fields * [#11723](https://github.com/netbox-community/netbox/issues/11723) - Circuit terminations should link to their associated circuits (rather than site or provider network) +* [#11775](https://github.com/netbox-community/netbox/issues/11775) - Skip checking for old search cache records when creating a new object * [#11786](https://github.com/netbox-community/netbox/issues/11786) - List only applicable object types in form widget when filtering custom fields --- diff --git a/netbox/extras/management/commands/reindex.py b/netbox/extras/management/commands/reindex.py index b601a1ac1cd..9a29c54f588 100644 --- a/netbox/extras/management/commands/reindex.py +++ b/netbox/extras/management/commands/reindex.py @@ -15,6 +15,11 @@ def add_arguments(self, parser): nargs='*', help='One or more apps or models to reindex', ) + parser.add_argument( + '--lazy', + action='store_true', + help="For each model, reindex objects only if no cache entries already exist" + ) def _get_indexers(self, *model_names): indexers = {} @@ -60,14 +65,15 @@ def handle(self, *model_labels, **kwargs): raise CommandError("No indexers found!") self.stdout.write(f'Reindexing {len(indexers)} models.') - # Clear all cached values for the specified models - self.stdout.write('Clearing cached values... ', ending='') - self.stdout.flush() - content_types = [ - ContentType.objects.get_for_model(model) for model in indexers.keys() - ] - deleted_count = search_backend.clear(content_types) - self.stdout.write(f'{deleted_count} entries deleted.') + # Clear all cached values for the specified models (if not being lazy) + if not kwargs['lazy']: + self.stdout.write('Clearing cached values... ', ending='') + self.stdout.flush() + content_types = [ + ContentType.objects.get_for_model(model) for model in indexers.keys() + ] + deleted_count = search_backend.clear(content_types) + self.stdout.write(f'{deleted_count} entries deleted.') # Index models self.stdout.write('Indexing models') @@ -76,11 +82,18 @@ def handle(self, *model_labels, **kwargs): model_name = model._meta.model_name self.stdout.write(f' {app_label}.{model_name}... ', ending='') self.stdout.flush() + + if kwargs['lazy']: + content_type = ContentType.objects.get_for_model(model) + if cached_count := search_backend.count(object_types=[content_type]): + self.stdout.write(f'Skipping (found {cached_count} existing).') + continue + i = search_backend.cache(model.objects.iterator(), remove_existing=False) if i: self.stdout.write(f'{i} entries cached.') else: - self.stdout.write(f'None found.') + self.stdout.write(f'No objects found.') msg = f'Completed.' if total_count := search_backend.size: diff --git a/netbox/netbox/search/backends.py b/netbox/netbox/search/backends.py index 14b5a987cde..f428842f5d1 100644 --- a/netbox/netbox/search/backends.py +++ b/netbox/netbox/search/backends.py @@ -80,7 +80,13 @@ def remove(self, instance): def clear(self, object_types=None): """ - Delete *all* cached data. + Delete *all* cached data (optionally filtered by object type). + """ + raise NotImplementedError + + def count(self, object_types=None): + """ + Return a count of all cache entries (optionally filtered by object type). """ raise NotImplementedError @@ -218,6 +224,12 @@ def clear(self, object_types=None): # Call _raw_delete() on the queryset to avoid first loading instances into memory return qs._raw_delete(using=qs.db) + def count(self, object_types=None): + qs = CachedValue.objects.all() + if object_types: + qs = qs.filter(object_type__in=object_types) + return qs.count() + @property def size(self): return CachedValue.objects.count() diff --git a/upgrade.sh b/upgrade.sh index 161d65e3261..cac046a9fbe 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -103,6 +103,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND || exit 1 +# Rebuild the search cache (lazily) +COMMAND="python3 netbox/manage.py reindex --lazy" +echo "Rebuilding search cache ($COMMAND)..." +eval $COMMAND || exit 1 + # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..."