Skip to content

Commit

Permalink
fix: Improve handling of allOf schemas with single child
Browse files Browse the repository at this point in the history
  • Loading branch information
@en-milie
en-milie committed Sep 27, 2024
1 parent 7dbb368 commit 0235bf2
Show file tree
Hide file tree
Showing 4 changed files with 187 additions and 42 deletions.
95 changes: 55 additions & 40 deletions src/main/java/com/endava/cats/model/generator/OpenAPIModelGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.module.SimpleModule;
import io.github.ludovicianul.prettylogger.PrettyLogger;
import io.github.ludovicianul.prettylogger.PrettyLoggerFactory;
Expand Down Expand Up @@ -59,7 +60,6 @@ public class OpenAPIModelGenerator {
private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final BigDecimal MAX = new BigDecimal("99999999999");
private final PrettyLogger logger = PrettyLoggerFactory.getLogger(OpenAPIModelGenerator.class);
private final Set<Schema<?>> catsGeneratedExamples = new HashSet<>();
private final Random random;
private final boolean useExamples;
private final CatsGlobalContext globalContext;
Expand Down Expand Up @@ -91,11 +91,22 @@ public OpenAPIModelGenerator(CatsGlobalContext catsGlobalContext, ValidDataForma
this.callStackCounter = new HashMap<>();
this.useDefaults = useDefaults;
this.arraySize = arraySize;

configureDepthAwareJacksonMapper(selfReferenceDepth);
configureDefaultJacksonMapper();
}

private void configureDefaultJacksonMapper() {
simpleObjectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
simpleObjectMapper.disable(SerializationFeature.INDENT_OUTPUT);
}

private void configureDepthAwareJacksonMapper(int selfReferenceDepth) {
SimpleModule module = new SimpleModule();
customDepthMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); // Exclude null values
module.addSerializer(Object.class, new DepthLimitingSerializer());
customDepthMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
module.addSerializer(Object.class, new DepthLimitingSerializer(selfReferenceDepth));
customDepthMapper.registerModule(module);
simpleObjectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); // Exclude null values
customDepthMapper.disable(SerializationFeature.INDENT_OUTPUT);
}


Expand All @@ -119,7 +130,6 @@ public Map<String, String> generate(String modelName) {
example = writeObjectAsString(exampleObject);
globalContext.recordError("Generate sample it's too large to be processed in memory. CATS used a limiting depth serializer which might not include all expected fields. Re-run CATS with a smaller --selfReferenceDepth value, like --selfReferenceDepth 2");
}
resetCatsGeneratedExamples();
if (example != null) {
kv.put(EXAMPLE, example);
return Map.copyOf(kv);
Expand All @@ -131,12 +141,6 @@ public Map<String, String> generate(String modelName) {
return Collections.emptyMap();
}

private void resetCatsGeneratedExamples() {
for (Schema<?> schema : catsGeneratedExamples) {
schema.setExample(null);
}
}

private String writeObjectAsString(Object exampleObject) {
try {
StringWriter stringWriter = new StringWriter();
Expand Down Expand Up @@ -173,7 +177,7 @@ private <T> Object resolvePropertyToExample(String propertyName, Schema<T> prope

if (generatedValueFromFormat != null) {
return generatedValueFromFormat;
} else if (getExample(propertySchema) != null && canUseExamples(propertySchema)) {
} else if (getExample(propertySchema) != null && useExamples) {
logger.trace("Example set in swagger spec, returning example: '{}'", propertySchema.getExample());
return this.formatExampleIfNeeded(propertySchema);
} else if (CatsModelUtils.isStringSchema(propertySchema)) {
Expand Down Expand Up @@ -271,10 +275,6 @@ private Object resolveProperties(Schema<?> schema) {
return getExample(schema);
}

private <T> boolean canUseExamples(Schema<T> property) {
return useExamples || catsGeneratedExamples.contains(property);
}

private <T> Object getExampleForObjectSchema(Schema<T> property) {
Object example = getExample(property);
if (example != null) {
Expand Down Expand Up @@ -507,8 +507,6 @@ private void processSchemaProperties(String name, Schema schema, Map<String, Obj
processInnerSchema(name, schema, values, propertyName, innerSchema);
}
currentProperty = previousPropertyValue;
// schema.setExample(values);
catsGeneratedExamples.add(schema);
}

private void processInnerSchema(String name, Schema schema, Map<String, Object> values, Object
Expand Down Expand Up @@ -615,7 +613,7 @@ private void populateWithComposedSchema(Map<String, Object> values, String prope
if (innerAllOff) {
values.clear();
values.put(newKey, finalMap);
} else {
} else if (!finalMap.isEmpty()) {
values.put(propertyName, finalMap);
}
values = values.entrySet()
Expand Down Expand Up @@ -680,11 +678,19 @@ private List<Schema> excludeNullSchemas(List<Schema> xxxOfSchemas) {
}

private void createMergedSchema(String schemaName, List<Schema> allOfSchema) {
Collection<Schema> allOfSchemasSanitized = allOfSchema
.stream()
.filter(CatsModelUtils::isNotEmptySchema)
.toList();
if (allOfSchemasSanitized.size() == 1) {
return;
}

Schema<?> newSchema = new Schema<>();
newSchema.properties(new LinkedHashMap<>());
newSchema.required(new ArrayList<>());

for (Schema<?> schema : allOfSchema) {
for (Schema<?> schema : allOfSchemasSanitized) {
if (schema.get$ref() != null) {
schema = globalContext.getSchemaFromReference(schema.get$ref());
}
Expand All @@ -710,29 +716,38 @@ private void mapDiscriminator(Schema<?> composedSchema, List<Schema> anyOf) {
}

private void addXXXOfExamples(Map<String, Object> values, Object propertyName, Collection<Schema> allOf, String of) {
Set<String> storedSchemaRefs = new HashSet<>();
int i = 0;

for (Schema allOfSchema : allOf) {
String fullSchemaRef = allOfSchema.get$ref();
String schemaRef;

Schema schemaToExample = allOfSchema;
if (fullSchemaRef != null) {
schemaRef = CatsModelUtils.getSimpleRef(fullSchemaRef);
schemaToExample = this.globalContext.getSchemaFromReference(fullSchemaRef);
} else {
schemaRef = schemaToExample.getType();
Collection<Schema> allOfSchemasSanitized = allOf
.stream()
.filter(CatsModelUtils::isNotEmptySchema)
.toList();

if (allOfSchemasSanitized.size() == 1) {
values.put(propertyName.toString(), resolveModelToExample(propertyName.toString(), allOfSchemasSanitized.iterator().next()));
} else {
Set<String> storedSchemaRefs = new HashSet<>();
int i = 0;

for (Schema allOfSchema : allOfSchemasSanitized) {
String fullSchemaRef = allOfSchema.get$ref();
String schemaRef;

Schema schemaToExample = allOfSchema;
if (fullSchemaRef != null) {
schemaRef = CatsModelUtils.getSimpleRef(fullSchemaRef);
schemaToExample = this.globalContext.getSchemaFromReference(fullSchemaRef);
} else {
schemaRef = schemaToExample.getType();

if (storedSchemaRefs.contains(schemaRef)) {
schemaRef = schemaRef + ++i;
if (storedSchemaRefs.contains(schemaRef)) {
schemaRef = schemaRef + ++i;
}
fullSchemaRef = "#" + schemaRef;
storedSchemaRefs.add(schemaRef);
}
fullSchemaRef = "#" + schemaRef;
storedSchemaRefs.add(schemaRef);
String propertyKey = propertyName.toString() + "_" + schemaRef;
String keyToStore = currentProperty.contains("#") ? currentProperty.substring(currentProperty.lastIndexOf("#") + 1) : currentProperty;
values.put(keyToStore + of + fullSchemaRef, resolveModelToExample(propertyKey, schemaToExample));
}
String propertyKey = propertyName.toString() + "_" + schemaRef;
String keyToStore = currentProperty.contains("#") ? currentProperty.substring(currentProperty.lastIndexOf("#") + 1) : currentProperty;
values.put(keyToStore + of + fullSchemaRef, resolveModelToExample(propertyKey, schemaToExample));
}
}
}
16 changes: 15 additions & 1 deletion src/main/java/com/endava/cats/util/CatsModelUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.swagger.v3.parser.util.SchemaTypeUtil;
import org.apache.commons.lang3.StringUtils;
import org.openapitools.codegen.utils.ModelUtils;
import org.springframework.util.CollectionUtils;

import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -173,7 +174,7 @@ public static boolean isComplexRegex(Schema<?> schema) {
* @return true if the field name contains a complex regex, false otherwise
*/
public static boolean isUri(String pattern, String lowerField) {
return (lowerField.contains("url") || lowerField.contains("uri")) && ("http://www.test.com".matches(pattern) || "https://www.test.com".matches(pattern));
return (lowerField.contains("url") || lowerField.contains("uri") || lowerField.contains("link")) && ("http://www.test.com".matches(pattern) || "https://www.test.com".matches(pattern));
}

/**
Expand All @@ -197,4 +198,17 @@ public static boolean isEmail(String pattern, String lowerField) {
public static boolean isPassword(String pattern, String lowerField) {
return lowerField.contains("password") && "catsISc00l?!useIt#".matches(pattern);
}

public static boolean isNotEmptySchema(Schema schema) {
return schema != null &&
(StringUtils.isNotBlank(schema.get$ref()) ||
StringUtils.isNotBlank(schema.getType()) ||
!CollectionUtils.isEmpty(schema.getTypes()) ||
!CollectionUtils.isEmpty(schema.getProperties()) ||
!CollectionUtils.isEmpty(schema.getAllOf()) ||
!CollectionUtils.isEmpty(schema.getAnyOf()) ||
!CollectionUtils.isEmpty(schema.getOneOf()) ||
schema.getItems() != null ||
!CollectionUtils.isEmpty(schema.getRequired()));
}
}
45 changes: 44 additions & 1 deletion src/test/java/com/endava/cats/factory/FuzzingDataFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
import io.quarkus.test.junit.QuarkusTest;
import io.swagger.parser.OpenAPIParser;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.Operation;
import io.swagger.v3.oas.models.PathItem;
import io.swagger.v3.oas.models.examples.Example;
import io.swagger.v3.oas.models.headers.Header;
import io.swagger.v3.oas.models.media.Content;
import io.swagger.v3.oas.models.media.MediaType;
import io.swagger.v3.oas.models.media.Schema;
import io.swagger.v3.oas.models.parameters.Parameter;
import io.swagger.v3.oas.models.parameters.RequestBody;
import io.swagger.v3.parser.core.models.ParseOptions;
import jakarta.inject.Inject;
import org.assertj.core.api.Assertions;
Expand Down Expand Up @@ -403,7 +406,6 @@ void shouldGenerateRequestWhenContentTypeFormUrlEncoded() throws Exception {
Assertions.assertThat(firstData.getPayload()).contains("code", "message");
}


@Test
void shouldGeneratePayloadsWithCrossPathsReferences() throws Exception {
Mockito.when(processingArguments.getSelfReferenceDepth()).thenReturn(5);
Expand Down Expand Up @@ -719,6 +721,21 @@ void shouldGetAllFieldsWhenSchemaDoesNotExist() throws Exception {
Assertions.assertThat(fields).hasSize(4);
}

@Test
void shouldResolveWhenOpenApiHasMalformedAllOfSchemas() throws Exception {
List<FuzzingData> dataList = setupFuzzingData("/arns", "src/test/resources/petstore.yml");
Assertions.assertThat(dataList).hasSize(1);
String payload = dataList.getFirst().getPayload();
int parametersArraySize = Integer.parseInt(JsonUtils.getVariableFromJson(payload, "$.Parameters.StringParameters.length()").toString());
Object arn = JsonUtils.getVariableFromJson(payload, "$.SourceEntity.Arn");
Object name = JsonUtils.getVariableFromJson(payload, "$.Name");

Assertions.assertThat(parametersArraySize).isEqualTo(2);
Assertions.assertThat(arn).isNotEqualTo("NOT_SET");
Assertions.assertThat(name).isNotEqualTo("NOT_SET");
}


@Test
void shouldDetectCyclicDependenciesWhenPropertiesNamesDontMatch() throws Exception {
Mockito.when(processingArguments.getSelfReferenceDepth()).thenReturn(1);
Expand Down Expand Up @@ -803,4 +820,30 @@ void shouldExtractMultiLevelExamples() {

Assertions.assertThat(examples).hasSize(2).containsExactly("example2", "\"catsIsCool\"");
}

@Test
void shouldHaveContentWhenContentNotNull() {
Operation operation = new Operation();
RequestBody requestBody = new RequestBody();
requestBody.setContent(new Content());
operation.setRequestBody(requestBody);

Assertions.assertThat(FuzzingDataFactory.hasContent(operation)).isTrue();
}

@Test
void shouldNotHaveContentWhenContentIsNull() {
Operation operation = new Operation();
RequestBody requestBody = new RequestBody();
operation.setRequestBody(requestBody);

Assertions.assertThat(FuzzingDataFactory.hasContent(operation)).isFalse();
}

@Test
void shouldNotHaveContentWhenRequestBodyIsNull() {
Operation operation = new Operation();

Assertions.assertThat(FuzzingDataFactory.hasContent(operation)).isFalse();
}
}
73 changes: 73 additions & 0 deletions src/test/resources/petstore.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,48 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Pet'
/arns:
post:
description: Create containers
operationId: createPetContainer
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- Name
properties:
Name:
description: 'A descriptive name for the analysis that you''re creating. This name displays for the analysis in the Amazon QuickSight console. '
type: string
minLength: 1
maxLength: 2048
Parameters:
description: A list of Amazon QuickSight parameters and the list's override values.
type: object
properties:
StringParameters:
allOf:
- $ref: '#/components/schemas/StringParameterList'
- description: The parameters that have a data type of string.
SourceEntity:
description: The source entity of an analysis.
type: object
properties:
SourceTemplate:
allOf:
- $ref: '#/components/schemas/AnalysisSourceTemplate'
- description: The source template for the source entity of the analysis.

responses:
'200':
description: pet response
content:
application/json:
schema:
$ref: '#/components/schemas/Pet'
/pet-types-rec:
post:
description: Creates a new pet in the store. Duplicates are allowed
Expand Down Expand Up @@ -294,10 +336,41 @@ components:
description: "Per container information. Key: container name."
type: object
type: object
StringParameterList:
type: array
items:
$ref: '#/components/schemas/StringParameter'
maxItems: 100
StringParameter:
type: object
required:
- Name
- Values
properties:
Name:
allOf:
- $ref: '#/components/schemas/NonEmptyString'
- description: A display name for a string parameter.
NonEmptyString:
type: string
pattern: .*\S.*
Arn:
type: string
Pets:
type: array
items:
$ref: "#/components/schemas/Pet"
AnalysisSourceTemplate:
type: object
required:
- DataSetReferences
- Arn
properties:
Arn:
allOf:
- $ref: '#/components/schemas/Arn'
- description: The Amazon Resource Name (ARN) of the source template of an analysis.
description: The source template of an analysis.
Pet:
allOf:
- $ref: '#/components/schemas/NewPet'
Expand Down

0 comments on commit 0235bf2

Please sign in to comment.