Skip to content

Commit

Permalink
Fixes neo4j-contrib#2015: Various bugs with Neo4j 4.3 indexes (neo4j-…
Browse files Browse the repository at this point in the history 
…contrib#2262)
  • Loading branch information
@vga91
vga91 committed Mar 23, 2022
1 parent a15a991 commit 5e36360
Show file tree
Hide file tree
Showing 9 changed files with 227 additions and 47 deletions.
8 changes: 8 additions & 0 deletions core/src/main/java/apoc/export/util/NodesAndRelsSubGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
return tx.schema().getIndexes(label);
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
if (!types.contains(type.name())) {
return Collections.emptyList();
}
return tx.schema().getIndexes(type);
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return types.stream()
Expand Down
34 changes: 21 additions & 13 deletions core/src/main/java/apoc/meta/Meta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,37 +628,41 @@ private Map<Set<String>, Map<String, MetaResult>> collectMetaData(SubGraph graph

Set<RelationshipType> types = Iterables.asSet(graph.getAllRelationshipTypesInUse());
Map<String, Iterable<ConstraintDefinition>> relConstraints = new HashMap<>(20);
Map<String, Set<String>> relIndexes = new HashMap<>();
for (RelationshipType type : graph.getAllRelationshipTypesInUse()) {
metaData.put(Set.of(Types.RELATIONSHIP.name(), type.name()), new LinkedHashMap<>(10));
relConstraints.put(type.name(),graph.getConstraints(type));
relIndexes.put(type.name(), getIndexedProperties(graph.getIndexes(type)));
}
for (Label label : graph.getAllLabelsInUse()) {
Map<String,MetaResult> nodeMeta = new LinkedHashMap<>(50);
String labelName = label.name();
// workaround in case of duplicated keys
metaData.put(Set.of(Types.NODE.name(), labelName), nodeMeta);
Iterable<ConstraintDefinition> constraints = graph.getConstraints(label);
Set<String> indexed = new LinkedHashSet<>();
for (IndexDefinition index : graph.getIndexes(label)) {
for (String prop : index.getPropertyKeys()) {
indexed.add(prop);
}
}
Set<String> indexed = getIndexedProperties(graph.getIndexes(label));
long labelCount = graph.countsForNode(label);
long sample = getSampleForLabelCount(labelCount, config.getSample());
Iterator<Node> nodes = graph.findNodes(label);
int count = 1;
while (nodes.hasNext()) {
Node node = nodes.next();
if(count++ % sample == 0) {
addRelationships(metaData, nodeMeta, labelName, node, relConstraints, types);
addRelationships(metaData, nodeMeta, labelName, node, relConstraints, types, relIndexes);
addProperties(nodeMeta, labelName, constraints, indexed, node, node);
}
}
}
return metaData;
}

private Set<String> getIndexedProperties(Iterable<IndexDefinition> indexes) {
return Iterables.stream(indexes)
.map(IndexDefinition::getPropertyKeys)
.flatMap(Iterables::stream)
.collect(Collectors.toSet());
}

private Map<String, Long> getLabelCountStore() {
List<String> labels = Iterables.stream(tx.getAllLabelsInUse()).map(label -> label.name()).collect(Collectors.toList());
TokenRead tokenRead = kernelTx.tokenRead();
Expand Down Expand Up @@ -774,7 +778,8 @@ private Map<String, Object> collectRelationshipsMetaData(MetaStats metaStats, Ma
entityProperties.put(entityDataKey, MapUtil.map(
"type", metaResult.type,
"array", metaResult.array,
"existence", metaResult.existence));
"existence", metaResult.existence,
"indexed", metaResult.index));
}
}
if (isRelationship) {
Expand Down Expand Up @@ -807,7 +812,9 @@ private void addRelationships(Map<Set<String>, Map<String, MetaResult>> metaData
String labelName,
Node node,
Map<String, Iterable<ConstraintDefinition>> relConstraints,
Set<RelationshipType> types) {
Set<RelationshipType> types,
Map<String, Set<String >> relIndexes
) {
StreamSupport.stream(node.getRelationshipTypes().spliterator(), false)
.filter(type -> types.contains(type))
.forEach(type -> {
Expand All @@ -818,17 +825,19 @@ private void addRelationships(Map<Set<String>, Map<String, MetaResult>> metaData
// workaround in case of duplicated keys

Iterable<ConstraintDefinition> constraints = relConstraints.get(typeName);
Set<String> indexes = relIndexes.get(typeName);
if (!nodeMeta.containsKey(typeName)) nodeMeta.put(typeName, new MetaResult(labelName,typeName));
// int in = node.getDegree(type, Direction.INCOMING);

Map<String, MetaResult> typeMeta = metaData.get(Set.of(typeName, Types.RELATIONSHIP.name()));
if (!typeMeta.containsKey(labelName)) typeMeta.put(labelName,new MetaResult(typeName,labelName));
MetaResult relMeta = nodeMeta.get(typeName);
addOtherNodeInfo(node, labelName, out, type, relMeta , typeMeta, constraints);
addOtherNodeInfo(node, labelName, out, type, relMeta , typeMeta, constraints, indexes);
});
}

private void addOtherNodeInfo(Node node, String labelName, int out, RelationshipType type, MetaResult relMeta, Map<String, MetaResult> typeMeta, Iterable<ConstraintDefinition> relConstraints) {
private void addOtherNodeInfo(Node node, String labelName, int out, RelationshipType type, MetaResult relMeta, Map<String, MetaResult> typeMeta,
Iterable<ConstraintDefinition> relConstraints, Set<String> indexes) {
MetaResult relNodeMeta = typeMeta.get(labelName);
relMeta.elementType(Types.of(node).name());
for (Relationship rel : node.getRelationships(Direction.OUTGOING, type)) {
Expand All @@ -837,8 +846,7 @@ private void addOtherNodeInfo(Node node, String labelName, int out, Relationship
int in = endNode.getDegree(type, Direction.INCOMING);
relMeta.inc().other(labels).rel(out , in);
relNodeMeta.inc().other(labels).rel(out,in);
final String typeName = type.name();
addProperties(typeMeta, typeName, relConstraints, Collections.emptySet(), rel, node);
addProperties(typeMeta, type.name(), relConstraints, indexes, rel, node);
relNodeMeta.elementType(Types.RELATIONSHIP.name());
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...oc/result/ConstraintRelationshipInfo.java → ...sult/IndexConstraintRelationshipInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
/**
* Created by alberto.delazzari on 04/07/17.
*/
public class ConstraintRelationshipInfo {
public class IndexConstraintRelationshipInfo {

public final String name;

Expand All @@ -15,7 +15,7 @@ public class ConstraintRelationshipInfo {

public final String status;

public ConstraintRelationshipInfo(String name, String type, List<String> properties, String status) {
public IndexConstraintRelationshipInfo(String name, String type, List<String> properties, String status) {
this.name = name;
this.type = type;
this.properties = properties;
Expand Down
92 changes: 71 additions & 21 deletions core/src/main/java/apoc/schema/Schemas.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package apoc.schema;

import apoc.Pools;
import apoc.result.AssertSchemaResult;
import apoc.result.ConstraintRelationshipInfo;
import apoc.result.IndexConstraintNodeInfo;
import org.antlr.v4.runtime.atn.SemanticContext;
import apoc.result.IndexConstraintRelationshipInfo;
import org.apache.commons.lang3.StringUtils;
import org.neo4j.common.EntityType;
import org.neo4j.common.TokenNameLookup;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
Expand All @@ -25,15 +26,24 @@
import org.neo4j.internal.schema.IndexDescriptor;
import org.neo4j.kernel.api.KernelTransaction;
import org.neo4j.kernel.api.Statement;
import org.neo4j.procedure.*;
import org.neo4j.procedure.Context;
import org.neo4j.procedure.Description;
import org.neo4j.procedure.Mode;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;
import org.neo4j.procedure.UserFunction;
import org.neo4j.token.api.TokenConstants;

import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static org.neo4j.graphdb.Label.label;
import static org.neo4j.internal.schema.SchemaUserDescription.TOKEN_LABEL;
import static org.neo4j.internal.schema.SchemaUserDescription.TOKEN_REL_TYPE;

public class Schemas {
@Context
Expand Down Expand Up @@ -61,8 +71,8 @@ public Stream<IndexConstraintNodeInfo> nodes(@Name(value = "config",defaultValue

@Procedure(value = "apoc.schema.relationships", mode = Mode.SCHEMA)
@Description("CALL apoc.schema.relationships([config]) yield name, startLabel, type, endLabel, properties, status")
public Stream<ConstraintRelationshipInfo> relationships(@Name(value = "config",defaultValue = "{}") Map<String,Object> config) {
return constraintsForRelationship(config);
public Stream<IndexConstraintRelationshipInfo> relationships(@Name(value = "config",defaultValue = "{}") Map<String,Object> config) {
return indexesAndConstraintsForRelationships(config);
}

@UserFunction(value = "apoc.schema.node.indexExists")
Expand Down Expand Up @@ -303,17 +313,15 @@ private Stream<IndexConstraintNodeInfo> indexesAndConstraintsForNode(Map<String,

Iterator<IndexDescriptor> allIndex = schemaRead.indexesGetAll();

indexesIterator = StreamSupport.stream(
Spliterators.spliteratorUnknownSize(allIndex, Spliterator.ORDERED),
false)
.filter(index -> !index.isTokenIndex())
.filter(index -> Arrays.stream(index.schema().getEntityTokenIds()).noneMatch(id -> {
indexesIterator = getIndexesFromSchema(allIndex,
index -> index.schema().entityType().equals(EntityType.NODE)
&& Arrays.stream(index.schema().getEntityTokenIds()).noneMatch(id -> {
try {
return excludeLabels.contains(tokenRead.nodeLabelName(id));
} catch (LabelNotFoundKernelException e) {
return false;
}
})).collect(Collectors.toList());
}));

Iterable<ConstraintDescriptor> allConstraints = () -> schemaRead.constraintsGetAll();
constraintsIterator = StreamSupport.stream(allConstraints.spliterator(),false)
Expand Down Expand Up @@ -357,12 +365,17 @@ private Stream<IndexConstraintNodeInfo> indexesAndConstraintsForNode(Map<String,
}
}

private List<IndexDescriptor> getIndexesFromSchema(Iterator<IndexDescriptor> allIndex, Predicate<IndexDescriptor> indexDescriptorPredicate) {
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(allIndex, Spliterator.ORDERED), false)
.filter(indexDescriptorPredicate).collect(Collectors.toList());
}

/**
* Collects constraints for relationships
*
* @return
*/
private Stream<ConstraintRelationshipInfo> constraintsForRelationship(Map<String,Object> config) {
private Stream<IndexConstraintRelationshipInfo> indexesAndConstraintsForRelationships(Map<String,Object> config) {
Schema schema = tx.schema();

SchemaConfig schemaConfig = new SchemaConfig(config);
Expand All @@ -371,28 +384,46 @@ private Stream<ConstraintRelationshipInfo> constraintsForRelationship(Map<String

try ( Statement ignore = ktx.acquireStatement() ) {
TokenRead tokenRead = ktx.tokenRead();
SchemaRead schemaRead = ktx.schemaRead();
Iterable<ConstraintDefinition> constraintsIterator;
Iterable<IndexDescriptor> indexesIterator;

if(!includeRelationships.isEmpty()) {
constraintsIterator = includeRelationships.stream()
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != -1)
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != TokenConstants.NO_TOKEN)
.flatMap(type -> {
Iterable<ConstraintDefinition> constraintsForType = schema.getConstraints(RelationshipType.withName(type));
return StreamSupport.stream(constraintsForType.spliterator(), false);
})
.collect(Collectors.toList());

indexesIterator = includeRelationships.stream()
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != TokenConstants.NO_TOKEN)
.flatMap(type -> {
Iterable<IndexDescriptor> indexesForRelType = () -> schemaRead.indexesGetForRelationshipType(tokenRead.relationshipType(type));
return StreamSupport.stream(indexesForRelType.spliterator(), false);
})
.collect(Collectors.toList());
} else {
Iterable<ConstraintDefinition> allConstraints = schema.getConstraints();
constraintsIterator = StreamSupport.stream(allConstraints.spliterator(),false)
.filter(index -> !excludeRelationships.contains(index.getRelationshipType().name()))
.collect(Collectors.toList());

Iterator<IndexDescriptor> allIndex = schemaRead.indexesGetAll();
indexesIterator = getIndexesFromSchema(allIndex, index -> index.schema().entityType().equals(EntityType.RELATIONSHIP)
&& Arrays.stream(index.schema().getEntityTokenIds())
.noneMatch(id -> excludeRelationships.contains(tokenRead.relationshipTypeGetName(id))));
}

Stream<ConstraintRelationshipInfo> constraintRelationshipInfoStream = StreamSupport.stream(constraintsIterator.spliterator(), false)
Stream<IndexConstraintRelationshipInfo> constraintRelationshipInfoStream = StreamSupport.stream(constraintsIterator.spliterator(), false)
.filter(constraintDefinition -> constraintDefinition.isConstraintType(ConstraintType.RELATIONSHIP_PROPERTY_EXISTENCE))
.map(this::relationshipInfoFromConstraintDefinition);

return constraintRelationshipInfoStream;
Stream<IndexConstraintRelationshipInfo> indexRelationshipInfoStream = StreamSupport.stream(indexesIterator.spliterator(), false)
.map(index -> relationshipInfoFromIndexDescription(index, tokenRead));

return Stream.of(constraintRelationshipInfoStream, indexRelationshipInfoStream).flatMap(e -> e);
}
}

Expand Down Expand Up @@ -432,14 +463,16 @@ private IndexConstraintNodeInfo nodeInfoFromConstraintDescriptor(ConstraintDescr
*/
private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor indexDescriptor, SchemaRead schemaRead, TokenNameLookup tokens){
int[] labelIds = indexDescriptor.schema().getEntityTokenIds();
if (labelIds.length != 1) throw new IllegalStateException("Index with more than one label");
String labelName = tokens.labelGetName(labelIds[0]);
int length = labelIds.length;
if (length > 1) throw new IllegalStateException("Index with more than one label");
// to handle LOOKUP indexes
String labelName = length == 0 ? TOKEN_LABEL : tokens.labelGetName(labelIds[0]);
List<String> properties = new ArrayList<>();
Arrays.stream(indexDescriptor.schema().getPropertyIds()).forEach((i) -> properties.add(tokens.propertyKeyGetName(i)));
try {
return new IndexConstraintNodeInfo(
// Pretty print for index name
String.format(":%s(%s)", labelName, StringUtils.join(properties, ",")),
getSchemaInfoName(labelName, properties),
labelName,
properties,
schemaRead.indexGetState(indexDescriptor).toString(),
Expand All @@ -453,7 +486,7 @@ private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor inde
} catch(IndexNotFoundKernelException e) {
return new IndexConstraintNodeInfo(
// Pretty print for index name
String.format(":%s(%s)", labelName, StringUtils.join(properties, ",")),
getSchemaInfoName(labelName, properties),
labelName,
properties,
"NOT_FOUND",
Expand All @@ -465,18 +498,35 @@ private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor inde
}
}

private IndexConstraintRelationshipInfo relationshipInfoFromIndexDescription(IndexDescriptor indexDescriptor, TokenNameLookup tokens) {
int[] relIds = indexDescriptor.schema().getEntityTokenIds();
int length = relIds.length;
if (length > 1) throw new IllegalStateException("Index with more than one rel type");
// to handle LOOKUP indexes
String relName = length == 0 ? TOKEN_REL_TYPE : tokens.relationshipTypeGetName(relIds[0]);
final List<String> properties = Arrays.stream(indexDescriptor.schema().getPropertyIds())
.mapToObj(tokens::propertyKeyGetName)
.collect(Collectors.toList());
return new IndexConstraintRelationshipInfo(getSchemaInfoName(relName, properties), relName, properties, "");
}

/**
* Constraint info from ConstraintDefinition for relationships
*
* @param constraintDefinition
* @return
*/
private ConstraintRelationshipInfo relationshipInfoFromConstraintDefinition(ConstraintDefinition constraintDefinition) {
return new ConstraintRelationshipInfo(
private IndexConstraintRelationshipInfo relationshipInfoFromConstraintDefinition(ConstraintDefinition constraintDefinition) {
return new IndexConstraintRelationshipInfo(
String.format("CONSTRAINT %s", constraintDefinition.toString()),
constraintDefinition.getConstraintType().name(),
Iterables.asList(constraintDefinition.getPropertyKeys()),
""
);
}

private String getSchemaInfoName(Object labelOrType, List<String> properties) {
final String labelOrTypeAsString = labelOrType instanceof String ? (String) labelOrType : StringUtils.join(labelOrType, ",");
return String.format(":%s(%s)", labelOrTypeAsString, StringUtils.join(properties, ","));
}
}
8 changes: 8 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/CypherResultSubGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,14 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
.collect(Collectors.toSet());
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
return indexes.stream()
.filter(idx -> StreamSupport.stream(idx.getRelationshipTypes().spliterator(), false)
.anyMatch(relType -> relType.name().equals(type.name())))
.collect(Collectors.toSet());
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return Collections.unmodifiableCollection(types);
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/DatabaseSubGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
return transaction.schema().getIndexes(label);
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
return transaction.schema().getIndexes(type);
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return transaction.getAllRelationshipTypesInUse();
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/SubGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public interface SubGraph

Iterable<IndexDefinition> getIndexes(Label label);

Iterable<IndexDefinition> getIndexes(RelationshipType label);

Iterable<RelationshipType> getAllRelationshipTypesInUse();

Iterable<Label> getAllLabelsInUse();
Expand Down
Loading

0 comments on commit 5e36360

Please sign in to comment.