Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes #2015: Various bugs with Neo4j 4.3 indexes #2262

Merged
merged 2 commits into from
Nov 10, 2021
Merged

Fixes #2015: Various bugs with Neo4j 4.3 indexes #2262

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f7fdff8
Fixes #2015: Various bugs with Neo4j 4.3 indexes
vga91 Jun 18, 2021
b653632
added name() equality check
vga91 Nov 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/src/main/java/apoc/export/util/NodesAndRelsSubGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
return tx.schema().getIndexes(label);
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
if (!types.contains(type.name())) {
return Collections.emptyList();
}
return tx.schema().getIndexes(type);
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return types.stream()
Expand Down
33 changes: 21 additions & 12 deletions core/src/main/java/apoc/meta/Meta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -610,36 +610,40 @@ private Map<String, Map<String, MetaResult>> collectMetaData(SubGraph graph, Met

Set<RelationshipType> types = Iterables.asSet(graph.getAllRelationshipTypesInUse());
Map<String, Iterable<ConstraintDefinition>> relConstraints = new HashMap<>(20);
Map<String, Set<String>> relIndexes = new HashMap<>();
for (RelationshipType type : graph.getAllRelationshipTypesInUse()) {
metaData.put(type.name(), new LinkedHashMap<>(10));
relConstraints.put(type.name(),graph.getConstraints(type));
relIndexes.put(type.name(), getIndexedProperties(graph.getIndexes(type)));
}
for (Label label : graph.getAllLabelsInUse()) {
Map<String,MetaResult> nodeMeta = new LinkedHashMap<>(50);
String labelName = label.name();
metaData.put(labelName, nodeMeta);
Iterable<ConstraintDefinition> constraints = graph.getConstraints(label);
Set<String> indexed = new LinkedHashSet<>();
for (IndexDefinition index : graph.getIndexes(label)) {
for (String prop : index.getPropertyKeys()) {
indexed.add(prop);
}
}
Set<String> indexed = getIndexedProperties(graph.getIndexes(label));
long labelCount = graph.countsForNode(label);
long sample = getSampleForLabelCount(labelCount, config.getSample());
Iterator<Node> nodes = graph.findNodes(label);
int count = 1;
while (nodes.hasNext()) {
Node node = nodes.next();
if(count++ % sample == 0) {
addRelationships(metaData, nodeMeta, labelName, node, relConstraints, types);
addRelationships(metaData, nodeMeta, labelName, node, relConstraints, types, relIndexes);
addProperties(nodeMeta, labelName, constraints, indexed, node, node);
}
}
}
return metaData;
}

private Set<String> getIndexedProperties(Iterable<IndexDefinition> indexes) {
return Iterables.stream(indexes)
.map(IndexDefinition::getPropertyKeys)
.flatMap(Iterables::stream)
.collect(Collectors.toSet());
}

private Map<String, Long> getLabelCountStore() {
List<String> labels = Iterables.stream(tx.getAllLabelsInUse()).map( Label::name ).collect( Collectors.toList());
TokenRead tokenRead = kernelTx.tokenRead();
Expand Down Expand Up @@ -754,7 +758,8 @@ private Map<String, Object> collectRelationshipsMetaData(MetaStats metaStats, Ma
entityProperties.put(entityDataKey, MapUtil.map(
"type", metaResult.type,
"array", metaResult.array,
"existence", metaResult.existence));
"existence", metaResult.existence,
"indexed", metaResult.index));
}
}
if (isRelationship) {
Expand Down Expand Up @@ -782,7 +787,9 @@ private void addRelationships(Map<String, Map<String, MetaResult>> metaData,
String labelName,
Node node,
Map<String, Iterable<ConstraintDefinition>> relConstraints,
Set<RelationshipType> types) {
Set<RelationshipType> types,
Map<String, Set<String >> relIndexes
) {
StreamSupport.stream(node.getRelationshipTypes().spliterator(), false)
.filter(type -> types.contains(type))
.forEach(type -> {
Expand All @@ -792,17 +799,19 @@ private void addRelationships(Map<String, Map<String, MetaResult>> metaData,
String typeName = type.name();

Iterable<ConstraintDefinition> constraints = relConstraints.get(typeName);
Set<String> indexes = relIndexes.get(typeName);
if (!nodeMeta.containsKey(typeName)) nodeMeta.put(typeName, new MetaResult(labelName,typeName));
// int in = node.getDegree(type, Direction.INCOMING);

Map<String, MetaResult> typeMeta = metaData.get(typeName);
if (!typeMeta.containsKey(labelName)) typeMeta.put(labelName,new MetaResult(typeName,labelName));
MetaResult relMeta = nodeMeta.get(typeName);
addOtherNodeInfo(node, labelName, out, type, relMeta , typeMeta, constraints);
addOtherNodeInfo(node, labelName, out, type, relMeta , typeMeta, constraints, indexes);
});
}

private void addOtherNodeInfo(Node node, String labelName, int out, RelationshipType type, MetaResult relMeta, Map<String, MetaResult> typeMeta, Iterable<ConstraintDefinition> relConstraints) {
private void addOtherNodeInfo(Node node, String labelName, int out, RelationshipType type, MetaResult relMeta, Map<String, MetaResult> typeMeta,
Iterable<ConstraintDefinition> relConstraints, Set<String> indexes) {
MetaResult relNodeMeta = typeMeta.get(labelName);
relMeta.elementType(Types.of(node).name());
for (Relationship rel : node.getRelationships(Direction.OUTGOING, type)) {
Expand All @@ -811,7 +820,7 @@ private void addOtherNodeInfo(Node node, String labelName, int out, Relationship
int in = endNode.getDegree(type, Direction.INCOMING);
relMeta.inc().other(labels).rel(out , in);
relNodeMeta.inc().other(labels).rel(out,in);
addProperties(typeMeta, type.name(), relConstraints, Collections.emptySet(), rel, node);
addProperties(typeMeta, type.name(), relConstraints, indexes, rel, node);
relNodeMeta.elementType(Types.RELATIONSHIP.name());
}
}
Expand Down
4 changes: 2 additions & 2 deletions ...oc/result/ConstraintRelationshipInfo.java → ...sult/IndexConstraintRelationshipInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
/**
* Created by alberto.delazzari on 04/07/17.
*/
public class ConstraintRelationshipInfo {
public class IndexConstraintRelationshipInfo {

public final String name;

Expand All @@ -15,7 +15,7 @@ public class ConstraintRelationshipInfo {

public final String status;

public ConstraintRelationshipInfo(String name, String type, List<String> properties, String status) {
public IndexConstraintRelationshipInfo(String name, String type, List<String> properties, String status) {
this.name = name;
this.type = type;
this.properties = properties;
Expand Down
92 changes: 71 additions & 21 deletions core/src/main/java/apoc/schema/Schemas.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package apoc.schema;

import apoc.Pools;
import apoc.result.AssertSchemaResult;
import apoc.result.ConstraintRelationshipInfo;
import apoc.result.IndexConstraintNodeInfo;
import org.antlr.v4.runtime.atn.SemanticContext;
import apoc.result.IndexConstraintRelationshipInfo;
import org.apache.commons.lang3.StringUtils;
import org.neo4j.common.EntityType;
import org.neo4j.common.TokenNameLookup;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
Expand All @@ -25,15 +26,24 @@
import org.neo4j.internal.schema.IndexDescriptor;
import org.neo4j.kernel.api.KernelTransaction;
import org.neo4j.kernel.api.Statement;
import org.neo4j.procedure.*;
import org.neo4j.procedure.Context;
import org.neo4j.procedure.Description;
import org.neo4j.procedure.Mode;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;
import org.neo4j.procedure.UserFunction;
import org.neo4j.token.api.TokenConstants;

import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static org.neo4j.graphdb.Label.label;
import static org.neo4j.internal.schema.SchemaUserDescription.TOKEN_LABEL;
import static org.neo4j.internal.schema.SchemaUserDescription.TOKEN_REL_TYPE;

public class Schemas {
@Context
Expand Down Expand Up @@ -61,8 +71,8 @@ public Stream<IndexConstraintNodeInfo> nodes(@Name(value = "config",defaultValue

@Procedure(value = "apoc.schema.relationships", mode = Mode.SCHEMA)
@Description("CALL apoc.schema.relationships([config]) yield name, startLabel, type, endLabel, properties, status")
public Stream<ConstraintRelationshipInfo> relationships(@Name(value = "config",defaultValue = "{}") Map<String,Object> config) {
return constraintsForRelationship(config);
public Stream<IndexConstraintRelationshipInfo> relationships(@Name(value = "config",defaultValue = "{}") Map<String,Object> config) {
return indexesAndConstraintsForRelationships(config);
}

@UserFunction(value = "apoc.schema.node.indexExists")
Expand Down Expand Up @@ -303,17 +313,15 @@ private Stream<IndexConstraintNodeInfo> indexesAndConstraintsForNode(Map<String,

Iterator<IndexDescriptor> allIndex = schemaRead.indexesGetAll();

indexesIterator = StreamSupport.stream(
Spliterators.spliteratorUnknownSize(allIndex, Spliterator.ORDERED),
false)
.filter(index -> !index.isTokenIndex())
.filter(index -> Arrays.stream(index.schema().getEntityTokenIds()).noneMatch(id -> {
indexesIterator = getIndexesFromSchema(allIndex,
index -> index.schema().entityType().equals(EntityType.NODE)
&& Arrays.stream(index.schema().getEntityTokenIds()).noneMatch(id -> {
try {
return excludeLabels.contains(tokenRead.nodeLabelName(id));
} catch (LabelNotFoundKernelException e) {
return false;
}
})).collect(Collectors.toList());
}));

Iterable<ConstraintDescriptor> allConstraints = () -> schemaRead.constraintsGetAll();
constraintsIterator = StreamSupport.stream(allConstraints.spliterator(),false)
Expand Down Expand Up @@ -357,12 +365,17 @@ private Stream<IndexConstraintNodeInfo> indexesAndConstraintsForNode(Map<String,
}
}

private List<IndexDescriptor> getIndexesFromSchema(Iterator<IndexDescriptor> allIndex, Predicate<IndexDescriptor> indexDescriptorPredicate) {
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(allIndex, Spliterator.ORDERED), false)
.filter(indexDescriptorPredicate).collect(Collectors.toList());
}

/**
* Collects constraints for relationships
*
* @return
*/
private Stream<ConstraintRelationshipInfo> constraintsForRelationship(Map<String,Object> config) {
private Stream<IndexConstraintRelationshipInfo> indexesAndConstraintsForRelationships(Map<String,Object> config) {
Schema schema = tx.schema();

SchemaConfig schemaConfig = new SchemaConfig(config);
Expand All @@ -371,28 +384,46 @@ private Stream<ConstraintRelationshipInfo> constraintsForRelationship(Map<String

try ( Statement ignore = ktx.acquireStatement() ) {
TokenRead tokenRead = ktx.tokenRead();
SchemaRead schemaRead = ktx.schemaRead();
Iterable<ConstraintDefinition> constraintsIterator;
Iterable<IndexDescriptor> indexesIterator;

if(!includeRelationships.isEmpty()) {
constraintsIterator = includeRelationships.stream()
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != -1)
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != TokenConstants.NO_TOKEN)
.flatMap(type -> {
Iterable<ConstraintDefinition> constraintsForType = schema.getConstraints(RelationshipType.withName(type));
return StreamSupport.stream(constraintsForType.spliterator(), false);
})
.collect(Collectors.toList());

indexesIterator = includeRelationships.stream()
.filter(type -> !excludeRelationships.contains(type) && tokenRead.relationshipType(type) != TokenConstants.NO_TOKEN)
.flatMap(type -> {
Iterable<IndexDescriptor> indexesForRelType = () -> schemaRead.indexesGetForRelationshipType(tokenRead.relationshipType(type));
return StreamSupport.stream(indexesForRelType.spliterator(), false);
})
.collect(Collectors.toList());
} else {
Iterable<ConstraintDefinition> allConstraints = schema.getConstraints();
constraintsIterator = StreamSupport.stream(allConstraints.spliterator(),false)
.filter(index -> !excludeRelationships.contains(index.getRelationshipType().name()))
.collect(Collectors.toList());

Iterator<IndexDescriptor> allIndex = schemaRead.indexesGetAll();
indexesIterator = getIndexesFromSchema(allIndex, index -> index.schema().entityType().equals(EntityType.RELATIONSHIP)
&& Arrays.stream(index.schema().getEntityTokenIds())
.noneMatch(id -> excludeRelationships.contains(tokenRead.relationshipTypeGetName(id))));
}

Stream<ConstraintRelationshipInfo> constraintRelationshipInfoStream = StreamSupport.stream(constraintsIterator.spliterator(), false)
Stream<IndexConstraintRelationshipInfo> constraintRelationshipInfoStream = StreamSupport.stream(constraintsIterator.spliterator(), false)
.filter(constraintDefinition -> constraintDefinition.isConstraintType(ConstraintType.RELATIONSHIP_PROPERTY_EXISTENCE))
.map(this::relationshipInfoFromConstraintDefinition);

return constraintRelationshipInfoStream;
Stream<IndexConstraintRelationshipInfo> indexRelationshipInfoStream = StreamSupport.stream(indexesIterator.spliterator(), false)
.map(index -> relationshipInfoFromIndexDescription(index, tokenRead));

return Stream.of(constraintRelationshipInfoStream, indexRelationshipInfoStream).flatMap(e -> e);
}
}

Expand Down Expand Up @@ -432,14 +463,16 @@ private IndexConstraintNodeInfo nodeInfoFromConstraintDescriptor(ConstraintDescr
*/
private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor indexDescriptor, SchemaRead schemaRead, TokenNameLookup tokens){
int[] labelIds = indexDescriptor.schema().getEntityTokenIds();
if (labelIds.length != 1) throw new IllegalStateException("Index with more than one label");
String labelName = tokens.labelGetName(labelIds[0]);
int length = labelIds.length;
if (length > 1) throw new IllegalStateException("Index with more than one label");
// to handle LOOKUP indexes
String labelName = length == 0 ? TOKEN_LABEL : tokens.labelGetName(labelIds[0]);
List<String> properties = new ArrayList<>();
Arrays.stream(indexDescriptor.schema().getPropertyIds()).forEach((i) -> properties.add(tokens.propertyKeyGetName(i)));
try {
return new IndexConstraintNodeInfo(
// Pretty print for index name
String.format(":%s(%s)", labelName, StringUtils.join(properties, ",")),
getSchemaInfoName(labelName, properties),
labelName,
properties,
schemaRead.indexGetState(indexDescriptor).toString(),
Expand All @@ -453,7 +486,7 @@ private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor inde
} catch(IndexNotFoundKernelException e) {
return new IndexConstraintNodeInfo(
// Pretty print for index name
String.format(":%s(%s)", labelName, StringUtils.join(properties, ",")),
getSchemaInfoName(labelName, properties),
labelName,
properties,
"NOT_FOUND",
Expand All @@ -465,18 +498,35 @@ private IndexConstraintNodeInfo nodeInfoFromIndexDefinition(IndexDescriptor inde
}
}

private IndexConstraintRelationshipInfo relationshipInfoFromIndexDescription(IndexDescriptor indexDescriptor, TokenNameLookup tokens) {
int[] relIds = indexDescriptor.schema().getEntityTokenIds();
int length = relIds.length;
if (length > 1) throw new IllegalStateException("Index with more than one rel type");
// to handle LOOKUP indexes
String relName = length == 0 ? TOKEN_REL_TYPE : tokens.relationshipTypeGetName(relIds[0]);
final List<String> properties = Arrays.stream(indexDescriptor.schema().getPropertyIds())
.mapToObj(tokens::propertyKeyGetName)
.collect(Collectors.toList());
return new IndexConstraintRelationshipInfo(getSchemaInfoName(relName, properties), relName, properties, "");
}

/**
* Constraint info from ConstraintDefinition for relationships
*
* @param constraintDefinition
* @return
*/
private ConstraintRelationshipInfo relationshipInfoFromConstraintDefinition(ConstraintDefinition constraintDefinition) {
return new ConstraintRelationshipInfo(
private IndexConstraintRelationshipInfo relationshipInfoFromConstraintDefinition(ConstraintDefinition constraintDefinition) {
return new IndexConstraintRelationshipInfo(
String.format("CONSTRAINT %s", constraintDefinition.toString()),
constraintDefinition.getConstraintType().name(),
Iterables.asList(constraintDefinition.getPropertyKeys()),
""
);
}

private String getSchemaInfoName(Object labelOrType, List<String> properties) {
final String labelOrTypeAsString = labelOrType instanceof String ? (String) labelOrType : StringUtils.join(labelOrType, ",");
return String.format(":%s(%s)", labelOrTypeAsString, StringUtils.join(properties, ","));
}
}
7 changes: 7 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/CypherResultSubGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,13 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
.collect(Collectors.toSet());
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
return indexes.stream()
.filter(idx -> StreamSupport.stream(idx.getRelationshipTypes().spliterator(), false).anyMatch(lb -> lb.equals(type)))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should probably do equals on the name() to be sure, because RelationshipType is an interface so there is no symmetric equals

.collect(Collectors.toSet());
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return Collections.unmodifiableCollection(types);
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/DatabaseSubGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ public Iterable<IndexDefinition> getIndexes(Label label) {
return transaction.schema().getIndexes(label);
}

@Override
public Iterable<IndexDefinition> getIndexes(RelationshipType type) {
return transaction.schema().getIndexes(type);
}

@Override
public Iterable<RelationshipType> getAllRelationshipTypesInUse() {
return transaction.getAllRelationshipTypesInUse();
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/neo4j/cypher/export/SubGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public interface SubGraph

Iterable<IndexDefinition> getIndexes(Label label);

Iterable<IndexDefinition> getIndexes(RelationshipType label);

Iterable<RelationshipType> getAllRelationshipTypesInUse();

Iterable<Label> getAllLabelsInUse();
Expand Down
Loading