Merge remote-tracking branch 'origin/master' into sidecar-multi-confi…

…g-and-tags

full-backend-tests/src/test/java/org/graylog/plugins/views/ViewsResourceIT.java

@@ -93,6 +93,6 @@ void testInvalidSearchType() {
                 .then()
                 .statusCode(400)
                 .assertThat()
-                .body("message", equalTo("Search types do not correspond to view/search types, missing searches: [967d2217-fd99-48a6-b829-5acdab906807]"));
+                .body("message", equalTo("Search types do not correspond to view/search types, missing searches [967d2217-fd99-48a6-b829-5acdab906808]; search types: [967d2217-fd99-48a6-b829-5acdab906807]; state types: [967d2217-fd99-48a6-b829-5acdab906808]"));
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/views/search/rest/ViewsResource.java

@@ -211,15 +211,51 @@ private void validateIntegrity(ViewDTO dto, SearchUser searchUser, boolean newCr
         final Search search = searchDomain.getForUser(dto.searchId(), searchUser)
                 .orElseThrow(() -> new BadRequestException("Search " + dto.searchId() + " not available"));
 
+        validateSearchProperties(dto, search);
+
+
+        if (!newCreation) {
+            final ViewDTO originalView = dbService.get(dto.id()).orElseThrow(() -> new BadRequestException("Cannot update a view that does not exist : id = " + dto.id()));
+            final String originalViewSearchId = originalView.searchId();
+            final Search originalSearch = searchDomain.getForUser(originalViewSearchId, searchUser)
+                    .orElseThrow(() -> new BadRequestException("Search " + originalViewSearchId + " not available"));
+
+            final Set<UsesSearchFilters> originalSearchFilterUsages = getSearchFiltersUsages(originalView, originalSearch);
+            final Set<String> originalReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(originalSearchFilterUsages);
+            final Set<UsesSearchFilters> newSearchFilterUsages = getSearchFiltersUsages(dto, search);
+            final Set<String> newReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(newSearchFilterUsages);
+
+            final SearchFilterVisibilityCheckStatus searchFilterVisibilityCheckStatus = searchFilterVisibilityChecker.checkSearchFilterVisibility(
+                    filterID -> isPermitted(RestPermissions.SEARCH_FILTERS_READ, filterID), newReferencedSearchFiltersIds);
+            if (!searchFilterVisibilityCheckStatus.allSearchFiltersVisible(originalReferencedSearchFiltersIds)) {
+                throw new BadRequestException(searchFilterVisibilityCheckStatus.toMessage(originalReferencedSearchFiltersIds));
+            }
+
+        } else {
+            final Set<UsesSearchFilters> newSearchFilterUsages = getSearchFiltersUsages(dto, search);
+            final Set<String> newReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(newSearchFilterUsages);
+            final SearchFilterVisibilityCheckStatus searchFilterVisibilityCheckStatus = searchFilterVisibilityChecker.checkSearchFilterVisibility(
+                    filterID -> isPermitted(RestPermissions.SEARCH_FILTERS_READ, filterID), newReferencedSearchFiltersIds);
+            if (!searchFilterVisibilityCheckStatus.allSearchFiltersVisible()) {
+                throw new BadRequestException(searchFilterVisibilityCheckStatus.toMessage());
+            }
+        }
+
+    }
+
+    protected void validateSearchProperties(ViewDTO dto, Search search) {
         final Set<String> searchQueries = search.queries().stream()
                 .map(Query::id)
                 .collect(Collectors.toSet());
 
         final Set<String> stateQueries = dto.state().keySet();
 
         if (!searchQueries.containsAll(stateQueries)) {
-            final Sets.SetView<String> diff = Sets.difference(searchQueries, stateQueries);
-            throw new BadRequestException("Search queries do not correspond to view/state queries, missing query IDs: " + diff);
+            final Sets.SetView<String> diff = Sets.difference(stateQueries, searchQueries);
+            final String message = String.format(Locale.ROOT,
+                    "Search queries do not correspond to view/state queries, missing query IDs: %s; search queries: %s; state queries: %s",
+                    diff, searchQueries, stateQueries);
+            throw new BadRequestException(message);
         }
 
         final Set<String> searchTypes = search.queries().stream()
@@ -234,8 +270,11 @@ private void validateIntegrity(ViewDTO dto, SearchUser searchUser, boolean newCr
                 .collect(Collectors.toSet());
 
         if(!searchTypes.containsAll(stateTypes)) {
-            final Sets.SetView<String> diff = Sets.difference(searchTypes, stateTypes);
-            throw new BadRequestException("Search types do not correspond to view/search types, missing searches: " + diff);
+            final Sets.SetView<String> diff = Sets.difference(stateTypes, searchTypes);
+            final String message = String.format(Locale.ROOT,
+                    "Search types do not correspond to view/search types, missing searches %s; search types: %s; state types: %s",
+                    diff, searchTypes, stateTypes);
+            throw new BadRequestException(message);
         }
 
         final Set<String> widgetIds = dto.state().values().stream()
@@ -247,38 +286,13 @@ private void validateIntegrity(ViewDTO dto, SearchUser searchUser, boolean newCr
                 .flatMap(v -> v.widgetPositions().keySet().stream()).collect(Collectors.toSet());
 
         if (!widgetPositions.containsAll(widgetIds)) {
-            final Sets.SetView<String> diff = Sets.difference(widgetPositions, widgetIds);
-            throw new BadRequestException("Widget positions don't correspond to widgets, missing widget possitions: " + diff);
-        }
+            final Sets.SetView<String> diff = Sets.difference(widgetIds, widgetPositions);
+            final String message = String.format(Locale.ROOT,
+                    "Widget positions don't correspond to widgets, missing widget positions %s; widget IDs: %s; widget positions: %s",
+                    diff, widgetIds, widgetPositions);
+            throw new BadRequestException(message);
 
-
-        if (!newCreation) {
-            final ViewDTO originalView = dbService.get(dto.id()).orElseThrow(() -> new BadRequestException("Cannot update a view that does not exist : id = " + dto.id()));
-            final String originalViewSearchId = originalView.searchId();
-            final Search originalSearch = searchDomain.getForUser(originalViewSearchId, searchUser)
-                    .orElseThrow(() -> new BadRequestException("Search " + originalViewSearchId + " not available"));
-
-            final Set<UsesSearchFilters> originalSearchFilterUsages = getSearchFiltersUsages(originalView, originalSearch);
-            final Set<String> originalReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(originalSearchFilterUsages);
-            final Set<UsesSearchFilters> newSearchFilterUsages = getSearchFiltersUsages(dto, search);
-            final Set<String> newReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(newSearchFilterUsages);
-
-            final SearchFilterVisibilityCheckStatus searchFilterVisibilityCheckStatus = searchFilterVisibilityChecker.checkSearchFilterVisibility(
-                    filterID -> isPermitted(RestPermissions.SEARCH_FILTERS_READ, filterID), newReferencedSearchFiltersIds);
-            if (!searchFilterVisibilityCheckStatus.allSearchFiltersVisible(originalReferencedSearchFiltersIds)) {
-                throw new BadRequestException(searchFilterVisibilityCheckStatus.toMessage(originalReferencedSearchFiltersIds));
-            }
-
-        } else {
-            final Set<UsesSearchFilters> newSearchFilterUsages = getSearchFiltersUsages(dto, search);
-            final Set<String> newReferencedSearchFiltersIds = referencedSearchFiltersHelper.getReferencedSearchFiltersIds(newSearchFilterUsages);
-            final SearchFilterVisibilityCheckStatus searchFilterVisibilityCheckStatus = searchFilterVisibilityChecker.checkSearchFilterVisibility(
-                    filterID -> isPermitted(RestPermissions.SEARCH_FILTERS_READ, filterID), newReferencedSearchFiltersIds);
-            if (!searchFilterVisibilityCheckStatus.allSearchFiltersVisible()) {
-                throw new BadRequestException(searchFilterVisibilityCheckStatus.toMessage());
-            }
         }
-
     }
 
     private Set<UsesSearchFilters> getSearchFiltersUsages(final ViewDTO view, final Search referencedSearch) {

graylog2-server/src/main/java/org/graylog2/bootstrap/CmdLineTool.java

@@ -265,13 +265,14 @@ public void run() {
     }
 
     public void doRun(Level logLevel) {
-        if (isDumpDefaultConfig()) {
-            dumpDefaultConfigAndExit();
-        }
         // This is holding all our metrics.
         MetricRegistry metricRegistry = MetricRegistryFactory.create();
         featureFlags = getFeatureFlags(metricRegistry);
 
+        if (isDumpDefaultConfig()) {
+            dumpDefaultConfigAndExit();
+        }
+
         installConfigRepositories();
         installCommandConfig();
 

graylog2-server/src/test/java/org/graylog/plugins/views/search/rest/ViewsResourceTest.java

@@ -19,15 +19,22 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import org.apache.shiro.subject.Subject;
+import org.graylog.plugins.views.search.Query;
 import org.graylog.plugins.views.search.Search;
 import org.graylog.plugins.views.search.SearchDomain;
 import org.graylog.plugins.views.search.permissions.SearchUser;
 import org.graylog.plugins.views.search.searchfilters.ReferencedSearchFiltersHelper;
 import org.graylog.plugins.views.search.searchfilters.db.SearchFilterVisibilityCheckStatus;
 import org.graylog.plugins.views.search.searchfilters.db.SearchFilterVisibilityChecker;
+import org.graylog.plugins.views.search.searchtypes.MessageList;
+import org.graylog.plugins.views.search.views.Position;
+import org.graylog.plugins.views.search.views.UnknownWidgetConfigDTO;
 import org.graylog.plugins.views.search.views.ViewDTO;
 import org.graylog.plugins.views.search.views.ViewResolver;
 import org.graylog.plugins.views.search.views.ViewService;
+import org.graylog.plugins.views.search.views.ViewStateDTO;
+import org.graylog.plugins.views.search.views.WidgetDTO;
+import org.graylog.plugins.views.search.views.WidgetPositionDTO;
 import org.graylog.security.UserContext;
 import org.graylog2.dashboards.events.DashboardDeletedEvent;
 import org.graylog2.events.ClusterEventBus;
@@ -61,6 +68,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -224,6 +232,79 @@ public void updatesDashboardSuccessfullyIfInvisibleFilterWasPresentBefore() thro
         verify(viewService).update(view);
     }
 
+    @Test
+    void testVerifyIntegrity() {
+        final ViewDTO view = ViewDTO.builder()
+                .searchId("123")
+                .title("my-search")
+                .state(Collections.emptyMap())
+                .build();
+
+        final Search search = Search.builder()
+                .id("123")
+                .build();
+
+        // empty search, nothing to validate, should succeed
+        assertDoesNotThrow(() -> viewsResource.validateSearchProperties(view, search));
+
+        final Search searchWithQuery = search.toBuilder().queries(ImmutableSet.of(Query.builder().id("Q-111").build())).build();
+        final ViewDTO viewWithQuery = view.toBuilder().state(Collections.singletonMap("Q-123", ViewStateDTO.builder()
+                        .widgets(Collections.emptySet())
+                        .widgetMapping(Collections.emptyMap())
+                        .widgetPositions(Collections.emptyMap())
+                        .build()))
+                .build();
+
+        // the query with ID Q-123 is present in the view  state, but the search itself doesn't have it => invalid combination
+        assertThatThrownBy(() -> viewsResource.validateSearchProperties(viewWithQuery, searchWithQuery))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessageContaining("Search queries do not correspond to view/state queries, missing query IDs: [Q-123]; search queries: [Q-111]; state queries: [Q-123]");
+
+        final Search searchWithTypes = search.toBuilder().queries(ImmutableSet.of(Query.builder().id("Q-111")
+                .searchTypes(ImmutableSet.of(MessageList.builder().id("T-111").build()))
+                .build())).build();
+
+        final ViewDTO viewWithWidgets = view.toBuilder().state(Collections.singletonMap("Q-111", ViewStateDTO.builder()
+                .widgetMapping(Collections.singletonMap("W-123", Collections.singleton("T-123")))
+                .widgetPositions(Collections.emptyMap())
+                .widgets(Collections.emptySet())
+                .build())).build();
+
+        // view contains widget mappings for type T-123, but the search knows only a type T-111 => invalid
+        assertThatThrownBy(() -> viewsResource.validateSearchProperties(viewWithWidgets, searchWithTypes))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessageContaining("missing searches [T-123]; search types: [T-111]; state types: [T-123]");
+
+
+        final Search searchWithValidTypes = search.toBuilder().queries(ImmutableSet.of(Query.builder().id("Q-111")
+                .searchTypes(ImmutableSet.of(MessageList.builder().id("T-123").build()))
+                .build())).build();
+
+        final ViewDTO viewWithWidgetPositions = view.toBuilder()
+                .state(Collections.singletonMap("Q-111", ViewStateDTO.builder()
+                                .widgets(Collections.singleton(WidgetDTO.builder()
+                                        .id("W-123")
+                                                .type("my-type")
+                                                .config(UnknownWidgetConfigDTO.create(Collections.emptyMap()))
+                                        .build()))
+                                .widgetPositions(Collections.singletonMap("W-111",
+                                        WidgetPositionDTO.Builder.create()
+                                                .col(Position.fromInt(1))
+                                                .row(Position.fromInt(1))
+                                                .height(Position.fromInt(100))
+                                                .width(Position.fromInt(100))
+                                                .build()))
+                        .widgetMapping(Collections.singletonMap("W-123", Collections.singleton("T-123")))
+                        .build())).build();
+
+        assertThatThrownBy(() -> viewsResource.validateSearchProperties(viewWithWidgetPositions, searchWithValidTypes))
+                .isInstanceOf(BadRequestException.class)
+                .hasMessageContaining("Widget positions don't correspond to widgets, missing widget positions [W-123]; widget IDs: [W-123]; widget positions: [W-111]");
+
+
+
+    }
+
     private void prepareUpdate(final ViewDTO.Type viewType) {
         this.viewsResource = spy(new ViewsTestResource(viewService, clusterEventBus, userService, searchDomain, referencedSearchFiltersHelper));
 

graylog2-server/src/test/java/org/graylog/testing/completebackend/SearchServerInstanceFactoryByVersion.java

@@ -39,7 +39,7 @@ public SearchServerInstance create(Network network) {
         if (version.satisfies(ELASTICSEARCH, "^7.0.0")) {
             return doCreate("org.graylog.storage.elasticsearch7.testing.ElasticsearchInstanceES7", version, network);
         } else if (version.satisfies(OPENSEARCH, "^1.0.0")) {
-            return doCreate("org.graylog.storage.elasticsearch7.testing.OpensearchInstance", version, network);
+            return doCreate("org.graylog.storage.elasticsearch7.testing.OpenSearch13Instance", version, network);
         } else if (version.satisfies(OPENSEARCH, "^2.0.0")) {
             return doCreate("org.graylog.storage.opensearch2.testing.OpenSearchInstance", version, network);
         } else {

graylog2-web-interface/src/contexts/DefaultQueryClientProvider.tsx

@@ -16,22 +16,26 @@
  */
 import * as React from 'react';
 import { useMemo } from 'react';
+import type { QueryClientConfig } from '@tanstack/react-query';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { merge } from 'lodash';
 
 type Props = {
   children: React.ReactNode,
+  options?: QueryClientConfig
 };
 
-const options = {
+const defaultOptions = {
   defaultOptions: {
     queries: {
       refetchOnWindowFocus: false,
     },
   },
 };
 
-const DefaultQueryClientProvider = ({ children }: Props) => {
-  const queryClient = useMemo(() => new QueryClient(options), []);
+const DefaultQueryClientProvider = ({ children, options: optionsProp }: Props) => {
+  const options = optionsProp ? merge({}, defaultOptions, optionsProp) : defaultOptions;
+  const queryClient = useMemo(() => new QueryClient(options), [options]);
 
   return (
     <QueryClientProvider client={queryClient}>
@@ -40,4 +44,8 @@ const DefaultQueryClientProvider = ({ children }: Props) => {
   );
 };
 
+DefaultQueryClientProvider.defaultProps = {
+  options: undefined,
+};
+
 export default DefaultQueryClientProvider;

graylog2-web-interface/src/views/components/Search.dashboard.test.tsx

@@ -47,6 +47,7 @@ jest.mock('views/stores/ViewMetadataStore', () => ({
   ),
 }));
 
+jest.mock('views/logic/fieldtypes/useFieldTypes');
 jest.mock('hooks/useElementDimensions', () => () => ({ width: 1024, height: 768 }));
 
 const mockedQueryIds = Immutable.OrderedSet(['query-id-1', 'query-id-2']);

graylog2-web-interface/src/views/components/searchbar/queryinput/QueryInput.test.tsx

@@ -23,6 +23,8 @@ import { validationError } from 'fixtures/queryValidationState';
 
 import QueryInput from './QueryInput';
 
+jest.mock('views/logic/fieldtypes/useFieldTypes');
+
 jest.mock('views/actions/QueryValidationActions', () => ({
   displayValidationErrors: jest.fn(),
 }));

graylog2-web-interface/src/views/components/widgets/Widget.aggregations.test.tsx

@@ -52,6 +52,7 @@ const mockedUnixTime = 1577836800000; // 2020-01-01 00:00:00.000
 
 jest.mock('./WidgetHeader', () => 'widget-header');
 jest.mock('./WidgetColorContext', () => ({ children }) => children);
+jest.mock('views/logic/fieldtypes/useFieldTypes');
 
 const MockWidgetStoreState = Immutable.Map();
 

graylog2-web-interface/test/WrappingContainer.tsx

@@ -27,8 +27,16 @@ type Props = {
   children: React.ReactNode,
 }
 
+const queryClientOptions = {
+  defaultOptions: {
+    queries: {
+      retry: false,
+    },
+  },
+};
+
 const WrappingContainer = ({ children }: Props) => (
-  <DefaultQueryClientProvider>
+  <DefaultQueryClientProvider options={queryClientOptions}>
     <Router history={history}>
       <DefaultProviders>
         {children}