Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MODFQMMGR-468]: Aggregate Tenant locations across all tenants #440

Closed
wants to merge 4 commits into from
Closed

[MODFQMMGR-468]: Aggregate Tenant locations across all tenants #440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0b88229
MODFQMMGR-468: Aggregate tenant locations across all tenants
bvsharp Sep 24, 2024
8913718
Overload simplehttpclient get with tenant param
bvsharp Sep 26, 2024
399f9c6
fix compile issues post-rebase
bvsharp Sep 30, 2024
a407e54
Experimenting
bvsharp Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/main/java/org/folio/fqm/client/SimpleHttpClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;

import java.util.Map;

Expand All @@ -21,4 +22,15 @@ public interface SimpleHttpClient {
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams);

/**
* Retrieve arbitrary data from a FOLIO API endpoint for the specified tenant.
*
* @param path - the path of the API endpoint
* @param queryParams - a map of query parameters to pass to the API endpoint
* @param tenant - FOLIO tenant from which to retrieve data
* @return the body of the response (JSON)
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams, @RequestHeader("X-Okapi-Tenant") String tenant);
}
18 changes: 17 additions & 1 deletion src/main/java/org/folio/fqm/service/CrossTenantQueryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,37 @@ public class CrossTenantQueryService {

private static final String COMPOSITE_INSTANCES_ID = "6b08439b-4f8e-4468-8046-ea620f5cfb74";
private static final String SIMPLE_INSTANCES_ID = "8fc4a9d2-7ccf-4233-afb8-796911839862";
private static final String SIMPLE_HOLDINGS_ID = "30a5cfad-1868-4f46-86b9-a6ef67e2d9bf";
private static final String SIMPLE_ITEMS_ID = "372e025a-9444-473a-9ffd-fa0f63db1674";

public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossTenantQuery) {
log.info("GETTING TENANTS TO QUERY");
if (!forceCrossTenantQuery
&& !Boolean.TRUE.equals(entityType.getCrossTenantQueriesEnabled())
&& !COMPOSITE_INSTANCES_ID.equals(entityType.getId())) {
log.info("Cross tenant queries are not enabled for entity type {}", entityType.getName());
return List.of(executionContext.getTenantId());
}
// Get the ECS tenant info first, since this comes from mod-users and should work in non-ECS environments
// We can use this for determining if it's an ECS environment, and if so, retrieving the consortium ID and central tenant ID
Map<String, String> ecsTenantInfo = getEcsTenantInfo();
if (!ecsEnabled(ecsTenantInfo)) {
log.info("ECS NOT ENABLED FOR TENANT");
return List.of(executionContext.getTenantId());
}

return getTenants(entityType, ecsTenantInfo);
}

public List<String> getTenantsToQueryForApi(EntityType entityType) {
Map<String, String> ecsTenantInfo = getEcsTenantInfo();
if (SIMPLE_INSTANCES_ID.equals(entityType.getId()) || SIMPLE_HOLDINGS_ID.equals(entityType.getId()) || SIMPLE_ITEMS_ID.equals(entityType.getId())) {
return getTenants(entityType, ecsTenantInfo);
} return List.of(executionContext.getTenantId());
}

private List<String> getTenants(EntityType entityType, Map<String, String> ecsTenantInfo) {
List<String> tenantsToQuery = new ArrayList<>();
String centralTenantId = getCentralTenantId(ecsTenantInfo);
if (!executionContext.getTenantId().equals(centralTenantId)) {
log.debug("Tenant {} is not central tenant. Running intra-tenant query.", executionContext.getTenantId());
Expand All @@ -52,7 +69,6 @@ public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossT
return List.of(executionContext.getTenantId());
}

List<String> tenantsToQuery = new ArrayList<>();
tenantsToQuery.add(centralTenantId);
List<Map<String, String>> userTenantMaps = getUserTenants(ecsTenantInfo.get("consortiumId"), executionContext.getUserId().toString());
for (var userMap : userTenantMaps) {
Expand Down
34 changes: 21 additions & 13 deletions src/main/java/org/folio/fqm/service/EntityTypeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.fql.model.field.FqlField;
import org.folio.fql.service.FqlValidationService;
import org.folio.fqm.client.SimpleHttpClient;
Expand All @@ -28,6 +29,7 @@

@Service
@RequiredArgsConstructor
@Log4j2
public class EntityTypeService {

private static final int COLUMN_VALUE_DEFAULT_PAGE_SIZE = 1000;
Expand Down Expand Up @@ -125,7 +127,9 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
}

if (field.getValueSourceApi() != null) {
return getFieldValuesFromApi(field, searchText);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQueryForApi(entityType);
log.info("Tenants to query: {}", tenantsToQuery);
return getFieldValuesFromApi(field, searchText, tenantsToQuery);
}

if (field.getSource() != null) {
Expand Down Expand Up @@ -173,20 +177,24 @@ private ColumnValues getFieldValuesFromEntityTypeDefinition(Field field, String
return new ColumnValues().content(filteredValues);
}

private ColumnValues getFieldValuesFromApi(Field field, String searchText) {
String rawJson = fieldValueClient.get(field.getValueSourceApi().getPath(), Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)));
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());

List<ValueWithLabel> results = new ArrayList<>(values.size());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
results.add(new ValueWithLabel().value(value).label(label));
private ColumnValues getFieldValuesFromApi(Field field, String searchText, List<String> tenantsToQuery) {
Set<ValueWithLabel> resultSet = new HashSet<>();
for (String tenant : tenantsToQuery) {
String rawJson = fieldValueClient.get(field.getValueSourceApi().getPath(), Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)), tenant);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());

// List<ValueWithLabel> results = ArrayList<>(values.size());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
resultSet.add(new ValueWithLabel().value(value).label(label));
}
}
}
List<ValueWithLabel> results = new ArrayList<>(resultSet);
results.sort(Comparator.comparing(ValueWithLabel::getLabel, String.CASE_INSENSITIVE_ORDER));
return new ColumnValues().content(results);
}
Expand Down
11 changes: 11 additions & 0 deletions src/test/java/org/folio/fqm/service/CrossTenantQueryServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ void shouldGetListOfTenantsToQuery() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
assertEquals(expectedTenants, actualTenants);
Expand All @@ -122,6 +123,8 @@ void shouldRunIntraTenantQueryForNonCentralTenant() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId); // Central is tenant_01
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(executionContext.getTenantId()).thenReturn("tenant_02"); // Central is tenant_01
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
assertEquals(expectedTenants, actualTenants);
}
Expand All @@ -132,6 +135,8 @@ void shouldRunIntraTenantQueryIfExceptionIsThrown() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
when(executionContext.getTenantId()).thenReturn("tenant_01");
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
assertEquals(expectedTenants, actualTenants);
}
Expand All @@ -143,6 +148,7 @@ void shouldReturnTenantIdOnlyIfUserTenantsApiThrowsException() {

when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
assertEquals(expectedTenants, actualTenants);
Expand All @@ -169,6 +175,8 @@ void shouldNotQueryTenantIfUserLacksTenantPermissions() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

doNothing().when(permissionsService).verifyUserHasNecessaryPermissions("tenant_02", entityType, true);
doThrow(MissingPermissionsException.class).when(permissionsService).verifyUserHasNecessaryPermissions("tenant_03", entityType, true);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
Expand All @@ -185,13 +193,15 @@ void shouldQueryCentralTenantForSharedCompositeInstances() {

when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(instanceEntityType, false);
assertEquals(expectedTenants, actualTenants);
}

@Test
void shouldGetCentralTenantId() {
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
String expectedId = "tenant_01";
when(executionContext.getTenantId()).thenReturn(expectedId);
when(userTenantService.getUserTenantsResponse(expectedId)).thenReturn(ECS_TENANT_INFO);
Expand All @@ -204,6 +214,7 @@ void shouldHandleErrorWhenGettingCentralTenantId() {
String tenantId = "tenant_01";
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
assertNull(crossTenantQueryService.getCentralTenantId());
}

Expand Down
1 change: 1 addition & 0 deletions src/test/java/org/folio/fqm/service/EntityTypeFlatteningServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,7 @@ void shouldFlattenComplexEntityType() {
}
return entityType;
});

when(executionContext.getTenantId()).thenReturn("tenant_01");
when(userTenantService.getUserTenantsResponse("tenant_01")).thenReturn("{'totalRecords': 0}");

Expand Down
11 changes: 11 additions & 0 deletions src/test/java/org/folio/fqm/utils/IdStreamerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,13 @@ void shouldFetchIdStreamForFql() {
List<String[]> ids = idsWithCancelCallback.ids();
ids.forEach(idSet -> actualIds.add(Arrays.asList(idSet)));
};

when(localizationService.localizeEntityType(any(EntityType.class), anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0));
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(NON_ECS_USER_TENANT_JSON);
when(executionContext.getTenantId()).thenReturn("tenant_01");
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(NON_ECS_USER_TENANT_JSON);

int idsCount = idStreamer.streamIdsInBatch(
IdStreamerTestDataProvider.TEST_ENTITY_TYPE_DEFINITION,
true,
Expand All @@ -161,6 +165,7 @@ void shouldUseAdditionalEcsConditionsInEcsEnvironment() {
when(localizationService.localizeEntityType(any(EntityType.class), anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0));
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("consortia/0e88ed41-eadb-44c3-a7a7-f6572bbe06fc/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
when(executionContext.getTenantId()).thenReturn("tenant_01");

idStreamer.streamIdsInBatch(
Expand All @@ -185,10 +190,13 @@ void shouldUseUnionAllForCrossTenantQuery() {
List<String[]> ids = idsWithCancelCallback.ids();
ids.forEach(idSet -> actualIds.add(Arrays.asList(idSet)));
};

when(localizationService.localizeEntityType(any(EntityType.class), anyBoolean())).thenAnswer(invocation -> invocation.getArgument(0));
when(executionContext.getTenantId()).thenReturn("tenant_01");
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("consortia/0e88ed41-eadb-44c3-a7a7-f6572bbe06fc/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

int idsCount = idStreamer.streamIdsInBatch(
IdStreamerTestDataProvider.TEST_ENTITY_TYPE_DEFINITION,
true,
Expand All @@ -211,9 +219,12 @@ void shouldHandleGroupByFields() {
List<String[]> ids = idsWithCancelCallback.ids();
ids.forEach(idSet -> actualIds.add(Arrays.asList(idSet)));
};

when(localizationService.localizeEntityType(any(EntityType.class), anyBoolean())).thenReturn(TEST_GROUP_BY_ENTITY_TYPE_DEFINITION);
when(executionContext.getTenantId()).thenReturn("tenant_01");
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(NON_ECS_USER_TENANT_JSON);
when(ecsClient.get(eq("user-tenants"), anyMap())).thenReturn(NON_ECS_USER_TENANT_JSON);

int idsCount = idStreamer.streamIdsInBatch(
IdStreamerTestDataProvider.TEST_GROUP_BY_ENTITY_TYPE_DEFINITION,
true,
Expand Down
Loading