Skip to content

Commit

Permalink
MODFQMMGR-468: Aggregate Tenant locations across all tenants (#450)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvsharp
bvsharp authored Oct 10, 2024
1 parent d6489fb commit 3081807
Show file tree
Hide file tree
Showing 10 changed files with 117 additions and 47 deletions.
12 changes: 12 additions & 0 deletions src/main/java/org/folio/fqm/client/SimpleHttpClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;

import java.util.Map;

Expand All @@ -21,4 +22,15 @@ public interface SimpleHttpClient {
*/
@GetMapping(value = "/{path}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams);

/**
* Retrieve arbitrary data from a FOLIO API endpoint for the specified tenant.
*
* @param path - the path of the API endpoint
* @param queryParams - a map of query parameters to pass to the API endpoint
* @param tenant - FOLIO tenant from which to retrieve data
* @return the body of the response (JSON)
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams, @RequestHeader("X-Okapi-Tenant") String tenant);
}
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/repository/IdStreamer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public int streamIdsInBatch(EntityType entityType,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
boolean ecsEnabled = crossTenantQueryService.ecsEnabled();
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return this.streamIdsInBatch(entityType, sortResults, fql, batchSize, idsConsumer, tenantsToQuery, ecsEnabled);
}

Expand Down
30 changes: 26 additions & 4 deletions src/main/java/org/folio/fqm/service/CrossTenantQueryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,35 @@ public class CrossTenantQueryService {

private static final String COMPOSITE_INSTANCES_ID = "6b08439b-4f8e-4468-8046-ea620f5cfb74";
private static final String SIMPLE_INSTANCES_ID = "8fc4a9d2-7ccf-4233-afb8-796911839862";
private static final String SIMPLE_INSTANCE_STATUS_ID = "9c239bfd-198f-4013-bbc4-4551c0cbdeaa";
private static final String SIMPLE_INSTANCE_TYPE_ID = "af44e2e0-12e0-4eec-b80d-49feb33a866c";
private static final List<String> INSTANCE_RELATED_ENTITIES = List.of(SIMPLE_INSTANCES_ID, COMPOSITE_INSTANCES_ID, SIMPLE_INSTANCE_STATUS_ID, SIMPLE_INSTANCE_TYPE_ID);

public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossTenantQuery) {
if (!forceCrossTenantQuery
&& !Boolean.TRUE.equals(entityType.getCrossTenantQueriesEnabled())
/**
* Retrieve list of tenants to run query against.
* @param entityType Entity type definition
* @return List of tenants to query
*/
public List<String> getTenantsToQuery(EntityType entityType) {
if (!Boolean.TRUE.equals(entityType.getCrossTenantQueriesEnabled())
&& !COMPOSITE_INSTANCES_ID.equals(entityType.getId())) {
return List.of(executionContext.getTenantId());
}
return getTenants(entityType);
}

/**
* Retrieve list of tenants to retrieve column values from. This method skips the cross-tenant query check, since the
* column values API uses simple entity type definitions, which don't have cross-tenant queries enabled.
* method skips the cross-tenant query check
* @param entityType Entity type definition
* @return List of tenants to query
*/
public List<String> getTenantsToQueryForColumnValues(EntityType entityType) {
return getTenants(entityType);
}

private List<String> getTenants(EntityType entityType) {
// Get the ECS tenant info first, since this comes from mod-users and should work in non-ECS environments
// We can use this for determining if it's an ECS environment, and if so, retrieving the consortium ID and central tenant ID
Map<String, String> ecsTenantInfo = getEcsTenantInfo();
Expand All @@ -46,7 +68,7 @@ public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossT
// The Instances entity type is required to retrieve shared instances from the central tenant when
// running queries from member tenants. This means that if we are running a query for Instances, we need to
// query the current tenant (for local records) as well as the central tenant (for shared records).
if (COMPOSITE_INSTANCES_ID.equals(entityType.getId()) || SIMPLE_INSTANCES_ID.equals(entityType.getId())) {
if (INSTANCE_RELATED_ENTITIES.contains(entityType.getId())) {
return List.of(executionContext.getTenantId(), centralTenantId);
}
return List.of(executionContext.getTenantId());
Expand Down
57 changes: 35 additions & 22 deletions src/main/java/org/folio/fqm/service/EntityTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.codehaus.plexus.util.StringUtils;
Expand All @@ -22,7 +23,6 @@
import org.folio.querytool.domain.dto.EntityTypeColumn;
import org.folio.querytool.domain.dto.Field;
import org.folio.querytool.domain.dto.SourceColumn;
import org.folio.querytool.domain.dto.ValueSourceApi;
import org.folio.querytool.domain.dto.ValueWithLabel;
import org.springframework.lang.Nullable;
import org.springframework.stereotype.Service;
Expand Down Expand Up @@ -140,8 +140,9 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
return getFieldValuesFromEntityTypeDefinition(field, searchText);
}

List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
if (field.getValueSourceApi() != null) {
return getFieldValuesFromApi(field, searchText);
return getFieldValuesFromApi(field, searchText, tenantsToQuery);
}

if (field.getSource() != null) {
Expand All @@ -160,7 +161,7 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
return getTenantIds(entityType);
}
case "languages" -> {
return getLanguages(searchText);
return getLanguages(searchText, tenantsToQuery);
}
default -> {
throw new InvalidEntityTypeDefinitionException("Unhandled source name \"" + field.getSource().getName() + "\" for the FQM value source type in column \"" + fieldName + '"', entityType);
Expand All @@ -173,7 +174,7 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
}

private ColumnValues getTenantIds(EntityType entityType) {
List<String> tenants = crossTenantQueryService.getTenantsToQuery(entityType, true);
List<String> tenants = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
List<ValueWithLabel> tenantValues = tenants
.stream()
.map(tenant -> new ValueWithLabel().value(tenant).label(tenant))
Expand All @@ -192,22 +193,28 @@ private ColumnValues getFieldValuesFromEntityTypeDefinition(Field field, String
return new ColumnValues().content(filteredValues);
}

private ColumnValues getFieldValuesFromApi(Field field, String searchText) {
Map<String, String> queryParams = new HashMap<>(Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)));
ValueSourceApi valueSourceApi = field.getValueSourceApi();
String rawJson = simpleHttpClient.get(valueSourceApi.getPath(), queryParams);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());

List<ValueWithLabel> results = new ArrayList<>(values.size());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
results.add(new ValueWithLabel().value(value).label(label));
private ColumnValues getFieldValuesFromApi(Field field, String searchText, List<String> tenantsToQuery) {
Set<ValueWithLabel> resultSet = new HashSet<>();
for (String tenantId : tenantsToQuery) {
try {
String rawJson = simpleHttpClient.get(field.getValueSourceApi().getPath(), Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)), tenantId);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
resultSet.add(new ValueWithLabel().value(value).label(label));
}
}
} catch (FeignException.Unauthorized e) {
log.error("Failed to get column values from {} tenant due to exception: {}", tenantId, e.getMessage());
}
}


List<ValueWithLabel> results = new ArrayList<>(resultSet);
results.sort(Comparator.comparing(ValueWithLabel::getLabel, String.CASE_INSENSITIVE_ORDER));
return new ColumnValues().content(results);
}
Expand Down Expand Up @@ -243,15 +250,21 @@ private static ColumnValues getCurrencyValues() {
return new ColumnValues().content(currencies);
}

private ColumnValues getLanguages(String searchText) {
private ColumnValues getLanguages(String searchText, List<String> tenantsToQuery) {
Map<String, String> queryParams = Map.of(
"facet", "languages",
"query", "id=*",
"limit", "1000"
);
String rawJson = simpleHttpClient.get("search/instances/facets", queryParams);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read("$.facets.languages.values.*.id");

Set<String> valueSet = new HashSet<>();
for (String tenant : tenantsToQuery) {
String rawJson = simpleHttpClient.get("search/instances/facets", queryParams, tenant);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read("$.facets.languages.values.*.id");
valueSet.addAll(values);
}
List<String> values = new ArrayList<>(valueSet);

List<ValueWithLabel> results = new ArrayList<>();
ObjectMapper mapper =
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/folio/fqm/service/QueryManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,15 +191,15 @@ public List<Map<String, Object>> getContents(UUID entityTypeId, List<String> fie
fields.add(colName);
}
});
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetService.getResultSet(entityTypeId, fields, ids, tenantsToQuery);
}

private List<Map<String, Object>> getContents(UUID queryId, UUID entityTypeId, List<String> fields, boolean includeResults, int offset, int limit) {
if (includeResults) {
EntityType entityType = entityTypeService.getEntityTypeDefinition(entityTypeId, true, false);
List<List<String>> resultIds = queryResultsRepository.getQueryResultIds(queryId, offset, limit);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetService.getResultSet(entityTypeId, fields, resultIds, tenantsToQuery);
}
return List.of();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/service/QueryProcessorService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void getIdsInBatch(FqlQueryWithContext fqlQueryWithContext,
public List<Map<String, Object>> processQuery(EntityType entityType, String fqlQuery, List<String> fields, List<String> afterId, Integer limit) {
Fql fql = fqlService.getFql(fqlQuery);
boolean ecsEnabled = crossTenantQueryService.ecsEnabled();
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetRepository.getResultSetSync(
UUID.fromString(entityType.getId()),
fql,
Expand Down
30 changes: 22 additions & 8 deletions src/test/java/org/folio/fqm/service/CrossTenantQueryServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
class CrossTenantQueryServiceTest {

private static final EntityType entityType = new EntityType()
.id(UUID.randomUUID().toString())
.crossTenantQueriesEnabled(true);

@Mock
Expand Down Expand Up @@ -102,7 +103,7 @@ void shouldGetListOfTenantsToQuery() {
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -112,7 +113,7 @@ void shouldRunIntraTenantQueryForNonInstanceEntityTypes() {

List<String> expectedTenants = List.of("tenant_01");
when(executionContext.getTenantId()).thenReturn("tenant_01");
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(nonEcsEntityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(nonEcsEntityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -122,7 +123,7 @@ void shouldRunIntraTenantQueryForNonCentralTenant() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId); // Central is tenant_01
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -132,7 +133,7 @@ void shouldRunIntraTenantQueryIfExceptionIsThrown() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -144,7 +145,7 @@ void shouldReturnTenantIdOnlyIfUserTenantsApiThrowsException() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -156,7 +157,7 @@ void shouldAttemptCrossTenantQueryIfForceParamIsTrue() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, true);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
verify(userTenantService, times(1)).getUserTenantsResponse(tenantId);
assertEquals(expectedTenants, actualTenants);
}
Expand All @@ -171,7 +172,7 @@ void shouldNotQueryTenantIfUserLacksTenantPermissions() {
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
doNothing().when(permissionsService).verifyUserHasNecessaryPermissions("tenant_02", entityType, true);
doThrow(MissingPermissionsException.class).when(permissionsService).verifyUserHasNecessaryPermissions("tenant_03", entityType, true);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -186,7 +187,20 @@ void shouldQueryCentralTenantForSharedCompositeInstances() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(instanceEntityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(instanceEntityType);
assertEquals(expectedTenants, actualTenants);
}

@Test
void shouldGetListOfTenantsToQueryForColumnValues() {
String tenantId = "tenant_01";
List<String> expectedTenants = List.of("tenant_01", "tenant_02", "tenant_03");

when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand Down
Loading

0 comments on commit 3081807

Please sign in to comment.